From eb612d7c8082157aebb483bbd2c22f5dbf22e4e1 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Mon, 16 Nov 2020 08:03:19 +0100 Subject: [PATCH 01/35] use the proposed compressed-blobs REAPI --- .../execution/v2/remote_execution.pb.go | 552 +++++++++++------- 1 file changed, 328 insertions(+), 224 deletions(-) diff --git a/genproto/build/bazel/remote/execution/v2/remote_execution.pb.go b/genproto/build/bazel/remote/execution/v2/remote_execution.pb.go index 70c5ad223..8cec9561c 100755 --- a/genproto/build/bazel/remote/execution/v2/remote_execution.pb.go +++ b/genproto/build/bazel/remote/execution/v2/remote_execution.pb.go @@ -8,6 +8,7 @@ import ( fmt "fmt" semver "github.com/buchgr/bazel-remote/genproto/build/bazel/semver" proto "github.com/golang/protobuf/proto" + any "github.com/golang/protobuf/ptypes/any" duration "github.com/golang/protobuf/ptypes/duration" timestamp "github.com/golang/protobuf/ptypes/timestamp" wrappers "github.com/golang/protobuf/ptypes/wrappers" @@ -159,6 +160,34 @@ func (SymlinkAbsolutePathStrategy_Value) EnumDescriptor() ([]byte, []int) { return fileDescriptor_c43847ba40caac95, []int{39, 0} } +type Compressor_Value int32 + +const ( + // No compression. Servers and clients MUST always support this, and do + // not need to advertise it. + Compressor_IDENTITY Compressor_Value = 0 + // Zstandard compression. + Compressor_ZSTD Compressor_Value = 1 +) + +var Compressor_Value_name = map[int32]string{ + 0: "IDENTITY", + 1: "ZSTD", +} + +var Compressor_Value_value = map[string]int32{ + "IDENTITY": 0, + "ZSTD": 1, +} + +func (x Compressor_Value) String() string { + return proto.EnumName(Compressor_Value_name, int32(x)) +} + +func (Compressor_Value) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_c43847ba40caac95, []int{40, 0} +} + // An `Action` captures all the information about an execution which is required // to reproduce it. // @@ -1237,9 +1266,13 @@ type ExecutedActionMetadata struct { OutputUploadStartTimestamp *timestamp.Timestamp `protobuf:"bytes,9,opt,name=output_upload_start_timestamp,json=outputUploadStartTimestamp,proto3" json:"output_upload_start_timestamp,omitempty"` // When the worker finished uploading action outputs. OutputUploadCompletedTimestamp *timestamp.Timestamp `protobuf:"bytes,10,opt,name=output_upload_completed_timestamp,json=outputUploadCompletedTimestamp,proto3" json:"output_upload_completed_timestamp,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // Details that are specific to the kind of worker used. For example, + // on POSIX-like systems this could contain a message with + // getrusage(2) statistics. + AuxiliaryMetadata []*any.Any `protobuf:"bytes,11,rep,name=auxiliary_metadata,json=auxiliaryMetadata,proto3" json:"auxiliary_metadata,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ExecutedActionMetadata) Reset() { *m = ExecutedActionMetadata{} } @@ -1337,6 +1370,13 @@ func (m *ExecutedActionMetadata) GetOutputUploadCompletedTimestamp() *timestamp. return nil } +func (m *ExecutedActionMetadata) GetAuxiliaryMetadata() []*any.Any { + if m != nil { + return m.AuxiliaryMetadata + } + return nil +} + // An ActionResult represents the result of an // [Action][build.bazel.remote.execution.v2.Action] being run. // @@ -3442,6 +3482,38 @@ func (m *SymlinkAbsolutePathStrategy) XXX_DiscardUnknown() { var xxx_messageInfo_SymlinkAbsolutePathStrategy proto.InternalMessageInfo +// Compression formats which may be supported. +type Compressor struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Compressor) Reset() { *m = Compressor{} } +func (m *Compressor) String() string { return proto.CompactTextString(m) } +func (*Compressor) ProtoMessage() {} +func (*Compressor) Descriptor() ([]byte, []int) { + return fileDescriptor_c43847ba40caac95, []int{40} +} + +func (m *Compressor) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Compressor.Unmarshal(m, b) +} +func (m *Compressor) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Compressor.Marshal(b, m, deterministic) +} +func (m *Compressor) XXX_Merge(src proto.Message) { + xxx_messageInfo_Compressor.Merge(m, src) +} +func (m *Compressor) XXX_Size() int { + return xxx_messageInfo_Compressor.Size(m) +} +func (m *Compressor) XXX_DiscardUnknown() { + xxx_messageInfo_Compressor.DiscardUnknown(m) +} + +var xxx_messageInfo_Compressor proto.InternalMessageInfo + // Capabilities of the remote cache system. type CacheCapabilities struct { // All the digest functions supported by the remote cache. @@ -3458,16 +3530,23 @@ type CacheCapabilities struct { MaxBatchTotalSizeBytes int64 `protobuf:"varint,4,opt,name=max_batch_total_size_bytes,json=maxBatchTotalSizeBytes,proto3" json:"max_batch_total_size_bytes,omitempty"` // Whether absolute symlink targets are supported. SymlinkAbsolutePathStrategy SymlinkAbsolutePathStrategy_Value `protobuf:"varint,5,opt,name=symlink_absolute_path_strategy,json=symlinkAbsolutePathStrategy,proto3,enum=build.bazel.remote.execution.v2.SymlinkAbsolutePathStrategy_Value" json:"symlink_absolute_path_strategy,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // Compressors supported by the "compressed-blobs" bytestream resources. + // Servers MUST support identity/no-compression, even if it is not listed + // here. + // + // Note that this does not imply which if any compressors are supported by + // the server at the gRPC level. + SupportedCompressor []Compressor_Value `protobuf:"varint,6,rep,packed,name=supported_compressor,json=supportedCompressor,proto3,enum=build.bazel.remote.execution.v2.Compressor_Value" json:"supported_compressor,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *CacheCapabilities) Reset() { *m = CacheCapabilities{} } func (m *CacheCapabilities) String() string { return proto.CompactTextString(m) } func (*CacheCapabilities) ProtoMessage() {} func (*CacheCapabilities) Descriptor() ([]byte, []int) { - return fileDescriptor_c43847ba40caac95, []int{40} + return fileDescriptor_c43847ba40caac95, []int{41} } func (m *CacheCapabilities) XXX_Unmarshal(b []byte) error { @@ -3523,6 +3602,13 @@ func (m *CacheCapabilities) GetSymlinkAbsolutePathStrategy() SymlinkAbsolutePath return SymlinkAbsolutePathStrategy_UNKNOWN } +func (m *CacheCapabilities) GetSupportedCompressor() []Compressor_Value { + if m != nil { + return m.SupportedCompressor + } + return nil +} + // Capabilities of the remote execution system. type ExecutionCapabilities struct { // Remote execution may only support a single digest function. @@ -3542,7 +3628,7 @@ func (m *ExecutionCapabilities) Reset() { *m = ExecutionCapabilities{} } func (m *ExecutionCapabilities) String() string { return proto.CompactTextString(m) } func (*ExecutionCapabilities) ProtoMessage() {} func (*ExecutionCapabilities) Descriptor() ([]byte, []int) { - return fileDescriptor_c43847ba40caac95, []int{41} + return fileDescriptor_c43847ba40caac95, []int{42} } func (m *ExecutionCapabilities) XXX_Unmarshal(b []byte) error { @@ -3606,7 +3692,7 @@ func (m *ToolDetails) Reset() { *m = ToolDetails{} } func (m *ToolDetails) String() string { return proto.CompactTextString(m) } func (*ToolDetails) ProtoMessage() {} func (*ToolDetails) Descriptor() ([]byte, []int) { - return fileDescriptor_c43847ba40caac95, []int{42} + return fileDescriptor_c43847ba40caac95, []int{43} } func (m *ToolDetails) XXX_Unmarshal(b []byte) error { @@ -3675,7 +3761,7 @@ func (m *RequestMetadata) Reset() { *m = RequestMetadata{} } func (m *RequestMetadata) String() string { return proto.CompactTextString(m) } func (*RequestMetadata) ProtoMessage() {} func (*RequestMetadata) Descriptor() ([]byte, []int) { - return fileDescriptor_c43847ba40caac95, []int{43} + return fileDescriptor_c43847ba40caac95, []int{44} } func (m *RequestMetadata) XXX_Unmarshal(b []byte) error { @@ -3728,6 +3814,7 @@ func init() { proto.RegisterEnum("build.bazel.remote.execution.v2.ExecutionStage_Value", ExecutionStage_Value_name, ExecutionStage_Value_value) proto.RegisterEnum("build.bazel.remote.execution.v2.DigestFunction_Value", DigestFunction_Value_name, DigestFunction_Value_value) proto.RegisterEnum("build.bazel.remote.execution.v2.SymlinkAbsolutePathStrategy_Value", SymlinkAbsolutePathStrategy_Value_name, SymlinkAbsolutePathStrategy_Value_value) + proto.RegisterEnum("build.bazel.remote.execution.v2.Compressor_Value", Compressor_Value_name, Compressor_Value_value) proto.RegisterType((*Action)(nil), "build.bazel.remote.execution.v2.Action") proto.RegisterType((*Command)(nil), "build.bazel.remote.execution.v2.Command") proto.RegisterType((*Command_EnvironmentVariable)(nil), "build.bazel.remote.execution.v2.Command.EnvironmentVariable") @@ -3775,6 +3862,7 @@ func init() { proto.RegisterType((*PriorityCapabilities)(nil), "build.bazel.remote.execution.v2.PriorityCapabilities") proto.RegisterType((*PriorityCapabilities_PriorityRange)(nil), "build.bazel.remote.execution.v2.PriorityCapabilities.PriorityRange") proto.RegisterType((*SymlinkAbsolutePathStrategy)(nil), "build.bazel.remote.execution.v2.SymlinkAbsolutePathStrategy") + proto.RegisterType((*Compressor)(nil), "build.bazel.remote.execution.v2.Compressor") proto.RegisterType((*CacheCapabilities)(nil), "build.bazel.remote.execution.v2.CacheCapabilities") proto.RegisterType((*ExecutionCapabilities)(nil), "build.bazel.remote.execution.v2.ExecutionCapabilities") proto.RegisterType((*ToolDetails)(nil), "build.bazel.remote.execution.v2.ToolDetails") @@ -3786,220 +3874,226 @@ func init() { } var fileDescriptor_c43847ba40caac95 = []byte{ - // 3397 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x5b, 0x4d, 0x70, 0x1b, 0xd7, - 0x91, 0xde, 0x01, 0x40, 0xfc, 0x34, 0x40, 0x02, 0x7a, 0xa6, 0x28, 0x08, 0x92, 0x2c, 0x69, 0x5c, - 0xf6, 0x72, 0x29, 0x0b, 0x90, 0x20, 0x4b, 0x96, 0xb9, 0x6b, 0x7b, 0xf9, 0x03, 0x59, 0x92, 0xf9, - 0xb7, 0x03, 0x92, 0x92, 0x77, 0xbd, 0x1a, 0x0f, 0x31, 0x4f, 0xe0, 0xac, 0x80, 0x19, 0x68, 0xe6, - 0x81, 0x14, 0xe5, 0x52, 0x6d, 0xd5, 0x56, 0x79, 0xbd, 0xb5, 0x5b, 0xe5, 0xad, 0xad, 0xcd, 0x25, - 0xf1, 0x2d, 0xc7, 0x54, 0x8e, 0xb9, 0xe4, 0xef, 0x92, 0x93, 0x53, 0xc9, 0x29, 0xa9, 0x24, 0xc7, - 0x5c, 0x72, 0xc9, 0xcf, 0x35, 0x97, 0x54, 0x0e, 0xa9, 0xf7, 0x33, 0x7f, 0xc0, 0x50, 0x03, 0x50, - 0x74, 0x92, 0x13, 0x67, 0xfa, 0xbd, 0xfe, 0xba, 0x5f, 0xbf, 0x7e, 0xdd, 0xfd, 0x7a, 0x40, 0xb8, - 0xb1, 0xd3, 0x37, 0x3a, 0x7a, 0x6d, 0x47, 0x7b, 0x8a, 0x3b, 0x35, 0x1b, 0x77, 0x2d, 0x82, 0x6b, - 0xf8, 0x09, 0x6e, 0xf5, 0x89, 0x61, 0x99, 0xb5, 0xbd, 0xba, 0xa0, 0xa9, 0x1e, 0xad, 0xda, 0xb3, - 0x2d, 0x62, 0xa1, 0xf3, 0x8c, 0xaf, 0xca, 0xf8, 0xaa, 0x7c, 0x4e, 0xd5, 0x9f, 0xb3, 0x57, 0xaf, - 0x9c, 0x0f, 0x02, 0x3b, 0xb8, 0xbb, 0x87, 0x6d, 0xf1, 0x87, 0x23, 0x54, 0xce, 0xb6, 0x2d, 0xab, - 0xdd, 0xc1, 0x35, 0xad, 0x67, 0xd4, 0x34, 0xd3, 0xb4, 0x88, 0x46, 0x59, 0x1d, 0x31, 0xfa, 0x8a, - 0x18, 0xed, 0x58, 0x66, 0xdb, 0xee, 0x9b, 0xa6, 0x61, 0xb6, 0x6b, 0x56, 0x0f, 0xdb, 0xa1, 0x49, - 0x2f, 0x8b, 0x49, 0xec, 0x6d, 0xa7, 0xff, 0xb0, 0xa6, 0xf7, 0xf9, 0x04, 0x31, 0x7e, 0x7e, 0x70, - 0x9c, 0x18, 0x5d, 0xec, 0x10, 0xad, 0xdb, 0x3b, 0x0c, 0x60, 0xdf, 0xd6, 0x7a, 0x3d, 0x6c, 0xbb, - 0x02, 0x4e, 0x89, 0x71, 0xbb, 0xd7, 0xaa, 0x39, 0x44, 0x23, 0x7d, 0x31, 0x20, 0xff, 0x36, 0x01, - 0xe9, 0x85, 0x16, 0x15, 0x85, 0xd6, 0x60, 0xaa, 0x65, 0x75, 0xbb, 0x9a, 0xa9, 0xab, 0xba, 0xd1, - 0xc6, 0x0e, 0x29, 0x4b, 0x17, 0xa4, 0xd9, 0x7c, 0xfd, 0x6f, 0xab, 0x31, 0x26, 0xaa, 0x2e, 0xb3, - 0xe9, 0xca, 0xa4, 0x60, 0xe7, 0xaf, 0xa8, 0x09, 0x27, 0x0c, 0xb3, 0xd7, 0x27, 0xaa, 0x6d, 0x59, - 0xc4, 0x85, 0x4c, 0x8c, 0x07, 0x59, 0x64, 0x08, 0x8a, 0x65, 0x11, 0x01, 0x7a, 0x0d, 0x32, 0x74, - 0xed, 0x56, 0x9f, 0x94, 0xd3, 0x0c, 0xea, 0x74, 0x95, 0x2f, 0xad, 0xea, 0x2e, 0xbd, 0xba, 0x2c, - 0x6c, 0xa7, 0xb8, 0x33, 0xd1, 0x05, 0x28, 0xe8, 0x96, 0x6a, 0x5a, 0x44, 0x6d, 0x69, 0xad, 0x5d, - 0x5c, 0xce, 0x5c, 0x90, 0x66, 0xb3, 0x0a, 0xe8, 0xd6, 0x9a, 0x45, 0x96, 0x28, 0x05, 0x21, 0x48, - 0x39, 0x5a, 0x87, 0x94, 0x73, 0x17, 0xa4, 0xd9, 0x82, 0xc2, 0x9e, 0x51, 0x03, 0xb2, 0xbd, 0x8e, - 0x46, 0x1e, 0x5a, 0x76, 0xb7, 0x0c, 0x4c, 0xd6, 0xdf, 0xc5, 0xaa, 0xbd, 0x21, 0x18, 0x14, 0x8f, - 0xf5, 0x6e, 0x2a, 0x9b, 0x2c, 0xa5, 0xef, 0xa6, 0xb2, 0xd9, 0x52, 0x4e, 0xfe, 0x43, 0x12, 0x32, - 0x4b, 0xdc, 0x48, 0xe8, 0x2c, 0xe4, 0x34, 0xbb, 0xdd, 0xef, 0x62, 0x93, 0x38, 0x65, 0xe9, 0x42, - 0x72, 0x36, 0xa7, 0xf8, 0x04, 0xf4, 0x18, 0x4e, 0x62, 0x73, 0xcf, 0xb0, 0x2d, 0x93, 0xbe, 0xab, - 0x7b, 0x9a, 0x6d, 0x68, 0x3b, 0x1d, 0xec, 0x94, 0x13, 0x17, 0x92, 0xb3, 0xf9, 0xfa, 0x3f, 0xc4, - 0x6a, 0x22, 0xc4, 0x54, 0x1b, 0x3e, 0xca, 0xb6, 0x00, 0x51, 0xa6, 0xf1, 0x30, 0xd1, 0x41, 0x17, - 0xa1, 0x60, 0xf5, 0x09, 0xdd, 0xb0, 0x87, 0x06, 0x95, 0x94, 0x64, 0x3a, 0xe5, 0x39, 0xed, 0x16, - 0x25, 0xa1, 0xcb, 0x80, 0xc4, 0x14, 0xdd, 0xb0, 0x71, 0x8b, 0x58, 0xb6, 0x81, 0x9d, 0x72, 0x8a, - 0x4d, 0x3c, 0xc1, 0x47, 0x96, 0xfd, 0x81, 0x00, 0x62, 0x4f, 0x23, 0xbb, 0x4e, 0x39, 0x13, 0x44, - 0xdc, 0xa0, 0xa4, 0x90, 0x91, 0x27, 0x8e, 0x6c, 0x64, 0x74, 0x09, 0x4e, 0xec, 0x5b, 0xf6, 0x23, - 0xc3, 0x6c, 0x7b, 0x9a, 0x1d, 0x30, 0x07, 0xc9, 0x29, 0x25, 0x31, 0xe0, 0x2a, 0x76, 0x80, 0xde, - 0x80, 0x19, 0xa1, 0x96, 0x69, 0xe9, 0x58, 0xed, 0xd9, 0xf4, 0x3c, 0x12, 0xba, 0x92, 0x2c, 0x53, - 0x70, 0x9a, 0x8f, 0xae, 0x59, 0x3a, 0xde, 0xf0, 0xc6, 0x2a, 0xef, 0xc2, 0x4b, 0x11, 0xb6, 0xa4, - 0x9e, 0x63, 0x6a, 0x5d, 0xcc, 0xce, 0x4a, 0x4e, 0x61, 0xcf, 0x68, 0x1a, 0x26, 0xf6, 0xb4, 0x4e, - 0x1f, 0x33, 0x6f, 0xcf, 0x29, 0xfc, 0x45, 0xfe, 0x8a, 0x04, 0x59, 0x57, 0x75, 0xa4, 0x00, 0x04, - 0xe4, 0x4a, 0x6c, 0x53, 0xeb, 0x23, 0xaf, 0xbc, 0x2a, 0xf4, 0x3a, 0x50, 0x02, 0x28, 0x95, 0x37, - 0x20, 0xeb, 0xd2, 0xc7, 0x50, 0xeb, 0x8b, 0x04, 0xe4, 0x7c, 0xdb, 0xbc, 0x0b, 0x13, 0x7c, 0xf7, - 0xb9, 0x4a, 0xf1, 0x9b, 0x41, 0x1d, 0x83, 0x5a, 0x4a, 0xe1, 0x7c, 0x68, 0x03, 0xf2, 0x41, 0xdf, - 0xe0, 0xee, 0x5a, 0x1d, 0xe1, 0xbc, 0x0b, 0x0d, 0x18, 0x56, 0x10, 0x02, 0xdd, 0x86, 0xac, 0x73, - 0xd0, 0xed, 0x18, 0xe6, 0x23, 0xee, 0x93, 0xf9, 0xfa, 0xeb, 0xb1, 0x70, 0x4d, 0xce, 0xc0, 0xc0, - 0x3c, 0x6e, 0x74, 0x1f, 0x8a, 0x83, 0x3b, 0xce, 0x7d, 0xae, 0x16, 0x0b, 0x18, 0x76, 0x06, 0x65, - 0xca, 0x0c, 0xbd, 0xdf, 0x4d, 0x65, 0x53, 0xa5, 0x09, 0xf9, 0x26, 0x14, 0x02, 0xf3, 0xc6, 0xd9, - 0x84, 0x1f, 0x4b, 0x30, 0x15, 0x16, 0x81, 0x56, 0x23, 0x3c, 0xe4, 0xf2, 0x38, 0x7a, 0x86, 0x9c, - 0x03, 0x5d, 0x81, 0x89, 0x2e, 0x8d, 0x87, 0x22, 0x02, 0x57, 0x86, 0xc2, 0xe6, 0xa6, 0x9b, 0x52, - 0x14, 0x3e, 0x11, 0xbd, 0x05, 0xb9, 0xbe, 0x69, 0x3c, 0x51, 0xbb, 0x96, 0x8e, 0xcb, 0x49, 0xc6, - 0x75, 0x76, 0x88, 0x6b, 0xeb, 0x8e, 0x49, 0xae, 0xd5, 0xb7, 0xe9, 0x22, 0x94, 0x2c, 0x9d, 0xbe, - 0x6a, 0xe9, 0x58, 0xfe, 0x8d, 0x04, 0x59, 0xd7, 0x31, 0x22, 0xad, 0xf0, 0x2e, 0xa4, 0x8f, 0x96, - 0x10, 0x04, 0x1b, 0x7a, 0x05, 0x26, 0x0d, 0x47, 0x24, 0x73, 0x7a, 0x0e, 0xcb, 0x29, 0x16, 0xd3, - 0x0b, 0x86, 0xd3, 0xf0, 0x68, 0x51, 0xfb, 0x9d, 0x3e, 0xae, 0xfd, 0x4e, 0x96, 0x52, 0x77, 0x53, - 0xd9, 0x89, 0x52, 0x5a, 0xd6, 0x61, 0x32, 0xe4, 0xbd, 0x5f, 0xca, 0x82, 0xe5, 0xaf, 0x4a, 0x90, - 0x0f, 0x78, 0x75, 0xa4, 0x90, 0x19, 0x48, 0x13, 0xcd, 0x6e, 0x63, 0x22, 0x9c, 0x4b, 0xbc, 0x45, - 0xd9, 0x21, 0x75, 0x8c, 0x76, 0x90, 0xff, 0x1e, 0xd2, 0x22, 0x3d, 0x23, 0x48, 0xed, 0x6a, 0xce, - 0xae, 0xab, 0x15, 0x7d, 0x46, 0xe7, 0x00, 0x1c, 0xe3, 0x29, 0x56, 0x77, 0x0e, 0x08, 0x0b, 0x08, - 0xd2, 0x6c, 0x52, 0xc9, 0x51, 0xca, 0x22, 0x25, 0xc8, 0xbf, 0x4c, 0xc3, 0x0c, 0xdf, 0x33, 0xac, - 0xf3, 0x4a, 0x64, 0x15, 0x13, 0x4d, 0xd7, 0x88, 0x46, 0xd7, 0x43, 0x83, 0x37, 0xb6, 0x05, 0x9e, - 0x78, 0x43, 0x0d, 0x28, 0x3d, 0xee, 0xe3, 0x3e, 0xd6, 0x55, 0xaf, 0x0e, 0x1a, 0xc1, 0xad, 0x8b, - 0x9c, 0xc7, 0x23, 0xa0, 0x0d, 0x98, 0xe1, 0x80, 0xaa, 0x43, 0x34, 0x9b, 0x04, 0xc0, 0x92, 0xb1, - 0x60, 0xd3, 0x9c, 0xb3, 0x49, 0x19, 0x7d, 0xc4, 0xfb, 0x50, 0x11, 0x88, 0x2d, 0xab, 0xdb, 0xeb, - 0x60, 0x12, 0x52, 0x31, 0x15, 0x8b, 0x5a, 0xe6, 0xdc, 0x4b, 0x2e, 0xb3, 0x8f, 0xfc, 0x01, 0x9c, - 0xe1, 0xc5, 0xd4, 0x43, 0x4c, 0x5a, 0xbb, 0x43, 0x0a, 0x4f, 0xc4, 0x43, 0x33, 0xf6, 0x5b, 0x94, - 0x7b, 0x40, 0x69, 0x0d, 0xce, 0x07, 0xa1, 0xa3, 0x34, 0x4f, 0xc7, 0xc2, 0x9f, 0xf5, 0xe1, 0x23, - 0xb4, 0xdf, 0x86, 0xd3, 0x9e, 0x57, 0x0d, 0xe9, 0x9e, 0x89, 0x05, 0x3f, 0xe5, 0x31, 0x0f, 0xa8, - 0xfe, 0x00, 0xce, 0xf9, 0xb8, 0x51, 0x8a, 0x67, 0x63, 0xb1, 0xcf, 0x78, 0x00, 0x11, 0x7a, 0xff, - 0x2b, 0x9c, 0x13, 0x95, 0x42, 0xbf, 0xd7, 0xb1, 0x34, 0x7d, 0x48, 0xf7, 0x5c, 0x2c, 0x7e, 0x85, - 0x03, 0x6c, 0x31, 0xfe, 0x01, 0xf5, 0x31, 0x5c, 0x0c, 0xc3, 0x47, 0x2d, 0x01, 0x62, 0x45, 0xbc, - 0x1c, 0x14, 0x31, 0xbc, 0x0a, 0xf9, 0x47, 0x69, 0x28, 0xf0, 0x93, 0xa5, 0x60, 0xa7, 0xdf, 0x21, - 0x68, 0x6d, 0xa0, 0xd2, 0xe3, 0x49, 0xfa, 0x52, 0x6c, 0x30, 0x58, 0xf7, 0x4a, 0xc1, 0x70, 0x59, - 0xf8, 0x11, 0x4c, 0x07, 0xf0, 0x54, 0x2f, 0x5b, 0xc3, 0x88, 0xc9, 0x9f, 0xe3, 0x8a, 0xe8, 0xa6, - 0x20, 0x1f, 0xba, 0xe9, 0x66, 0xee, 0x7b, 0x50, 0x14, 0x12, 0x3c, 0xf0, 0xc2, 0x91, 0xc0, 0xa7, - 0xac, 0xe0, 0xab, 0x83, 0xd4, 0xc8, 0x8a, 0x96, 0x97, 0x19, 0x57, 0x46, 0xc4, 0xf6, 0xa2, 0x7f, - 0x54, 0x0d, 0xfc, 0x6f, 0x70, 0x7a, 0x40, 0xc0, 0x81, 0xbf, 0x86, 0xfc, 0x91, 0xd6, 0x70, 0x2a, - 0x2c, 0xe5, 0xc0, 0x5b, 0xcc, 0x19, 0xc8, 0xe1, 0x27, 0x06, 0x51, 0x5b, 0x34, 0x63, 0xd3, 0x68, - 0x33, 0xa1, 0x64, 0x29, 0x61, 0x89, 0x26, 0x0c, 0x1a, 0x86, 0x89, 0x6e, 0xd1, 0xfb, 0x98, 0xb6, - 0xcf, 0x02, 0x46, 0x41, 0xc9, 0x71, 0x8a, 0xa2, 0xed, 0xa3, 0x15, 0x98, 0x14, 0xc3, 0x22, 0x4f, - 0xa5, 0xc7, 0xcb, 0x53, 0x05, 0xce, 0x2d, 0xf2, 0x00, 0x17, 0x86, 0x6d, 0x9b, 0x09, 0xcb, 0x78, - 0xc2, 0xb0, 0x6d, 0xfb, 0xc2, 0xe8, 0xb0, 0x10, 0x96, 0x1d, 0x5f, 0x18, 0xb6, 0x6d, 0x21, 0xec, - 0x21, 0x20, 0x3f, 0x0a, 0x74, 0x45, 0xf2, 0x10, 0x47, 0xf3, 0xcd, 0x58, 0xc8, 0xe8, 0xdc, 0xa3, - 0x9c, 0xf0, 0x26, 0xb9, 0xa4, 0xbb, 0xa9, 0xac, 0x54, 0x4a, 0xc8, 0xff, 0x95, 0x00, 0xf0, 0x0f, - 0x02, 0xcd, 0x78, 0xf4, 0x72, 0xe3, 0x66, 0x3c, 0xfa, 0xfc, 0x67, 0xaa, 0x6e, 0x2a, 0x90, 0x6d, - 0x59, 0x26, 0x61, 0xf7, 0x47, 0xbe, 0x9d, 0xde, 0x7b, 0x54, 0xc6, 0xcf, 0x1c, 0x6f, 0xe5, 0x93, - 0x2e, 0x65, 0xe4, 0xcf, 0x24, 0x48, 0x6d, 0xda, 0x18, 0xa3, 0x77, 0x20, 0x45, 0x2f, 0xf9, 0xa2, - 0x61, 0x30, 0x37, 0x7a, 0xb5, 0xaf, 0x30, 0x3e, 0x74, 0x0b, 0xb2, 0xad, 0x5d, 0xa3, 0xa3, 0xdb, - 0xd8, 0x14, 0xc1, 0x68, 0x1c, 0x0c, 0x8f, 0x57, 0xee, 0x43, 0x71, 0xe0, 0x48, 0x46, 0xee, 0xcf, - 0x6d, 0xc8, 0x13, 0x1b, 0x63, 0xd7, 0xf9, 0x92, 0xe3, 0x6d, 0x12, 0x50, 0x5e, 0xfe, 0x7c, 0x37, - 0x95, 0x4d, 0x94, 0x92, 0xf2, 0xe7, 0x12, 0x4c, 0x86, 0x8e, 0x68, 0xa4, 0xd4, 0xbf, 0x54, 0x75, - 0x76, 0x19, 0x8a, 0x0d, 0x97, 0x69, 0xc3, 0xea, 0x18, 0xad, 0x03, 0xea, 0x3a, 0x3d, 0xdb, 0xb0, - 0x6c, 0x83, 0x1c, 0x30, 0x15, 0x27, 0x14, 0xef, 0x5d, 0xbe, 0x02, 0x88, 0xa7, 0x09, 0x87, 0xb5, - 0x46, 0x46, 0xe0, 0xf8, 0x24, 0x09, 0x53, 0xe2, 0x14, 0x29, 0xf8, 0x71, 0xdf, 0x75, 0x60, 0xd3, - 0x21, 0x9a, 0xd9, 0xc2, 0x6a, 0xa0, 0x4c, 0x2d, 0xb8, 0xc4, 0x35, 0x5a, 0xae, 0xce, 0xc1, 0x09, - 0xe7, 0x91, 0xd1, 0xe3, 0x4d, 0x19, 0xb5, 0x63, 0x59, 0x8f, 0xfa, 0xbc, 0xf4, 0xca, 0x2a, 0x45, - 0x3a, 0xc0, 0xe4, 0xaf, 0x30, 0x32, 0x8d, 0x18, 0x1a, 0x3b, 0xa0, 0x47, 0x0d, 0x4f, 0x9c, 0x5b, - 0x44, 0x8c, 0x7f, 0x81, 0x92, 0x1f, 0x31, 0x7a, 0x6c, 0x85, 0xe2, 0x7c, 0x5c, 0x19, 0x31, 0x5e, - 0x78, 0xb6, 0x54, 0x8a, 0x78, 0xc0, 0xb8, 0x18, 0xa6, 0x6d, 0x6e, 0x40, 0xb1, 0x32, 0x21, 0x80, - 0xc7, 0xb8, 0x6b, 0xb1, 0x02, 0x86, 0xad, 0xaf, 0x20, 0x7b, 0x88, 0xc6, 0x5d, 0x8f, 0x5f, 0x3c, - 0xc5, 0x45, 0xe4, 0x31, 0x64, 0x56, 0xac, 0x36, 0x8b, 0x4a, 0x7e, 0x04, 0x92, 0x8e, 0x16, 0x81, - 0x5e, 0x85, 0xa9, 0xdd, 0x7e, 0x57, 0x33, 0x55, 0x1b, 0x6b, 0x3a, 0x0b, 0x41, 0x09, 0xb6, 0x31, - 0x93, 0x8c, 0xaa, 0x08, 0xa2, 0xfc, 0xbf, 0x49, 0xd7, 0xb9, 0xb0, 0x82, 0x9d, 0x9e, 0x65, 0x3a, - 0x18, 0x35, 0x20, 0xcd, 0xd5, 0x15, 0xb2, 0xe3, 0x2f, 0xad, 0xc1, 0xe2, 0x44, 0x11, 0xcc, 0xd4, - 0x85, 0x98, 0xf9, 0x74, 0x55, 0xa0, 0x71, 0x05, 0x0a, 0x9c, 0x28, 0x2a, 0x99, 0x39, 0x48, 0xf3, - 0x76, 0xa6, 0x38, 0xc4, 0xc8, 0x2d, 0x93, 0xec, 0x5e, 0xab, 0xda, 0x64, 0x23, 0x8a, 0x98, 0x81, - 0x34, 0xc8, 0x3b, 0xd8, 0xde, 0xc3, 0xb6, 0xda, 0xb1, 0xda, 0xbc, 0x6b, 0x95, 0xaf, 0xff, 0xe3, - 0xa8, 0xf9, 0xc1, 0x5d, 0x5e, 0xb5, 0xc9, 0x30, 0x56, 0xac, 0xb6, 0xd3, 0x30, 0x89, 0x7d, 0xa0, - 0x80, 0xe3, 0x11, 0x50, 0x19, 0x32, 0x5d, 0xec, 0x38, 0x5a, 0x1b, 0xb3, 0x88, 0x9c, 0x53, 0xdc, - 0xd7, 0x4a, 0x1b, 0x8a, 0x03, 0x8c, 0xa8, 0x04, 0xc9, 0x47, 0xf8, 0x40, 0x9c, 0x0c, 0xfa, 0x88, - 0xde, 0x09, 0xf6, 0x06, 0xf2, 0xf5, 0xd9, 0x58, 0xdd, 0xc4, 0x76, 0x8b, 0x2e, 0xc2, 0x7c, 0xe2, - 0xa6, 0x24, 0x6b, 0xee, 0x59, 0xe4, 0xd5, 0x72, 0x1b, 0xcb, 0xeb, 0x30, 0xc1, 0xee, 0xe7, 0x28, - 0x0f, 0x99, 0xad, 0xb5, 0xf7, 0xd7, 0xd6, 0xef, 0xad, 0x95, 0xfe, 0x06, 0x15, 0x21, 0xbf, 0xb4, - 0xb0, 0x74, 0xbb, 0xa1, 0x2e, 0xdd, 0x6e, 0x2c, 0xbd, 0x5f, 0x92, 0x10, 0x40, 0xfa, 0x9f, 0xb6, - 0x1a, 0x5b, 0x8d, 0xe5, 0x52, 0x02, 0x4d, 0x42, 0xae, 0x71, 0xbf, 0xb1, 0xb4, 0xb5, 0x79, 0x67, - 0xed, 0xbd, 0x52, 0x92, 0xbe, 0x2e, 0xad, 0xaf, 0x6e, 0xac, 0x34, 0x36, 0x1b, 0xcb, 0xa5, 0x94, - 0xfc, 0x7f, 0x09, 0x28, 0x0b, 0xab, 0xac, 0xbb, 0x9d, 0x6c, 0xef, 0xce, 0xf6, 0x3e, 0x4c, 0x38, - 0x54, 0x2c, 0x5b, 0xd7, 0x54, 0xfd, 0xfa, 0xe8, 0xe7, 0x89, 0x69, 0x5b, 0xe5, 0xad, 0x04, 0x8e, - 0x31, 0x7c, 0xea, 0x13, 0x2f, 0x72, 0xea, 0x5f, 0x07, 0x24, 0x4a, 0x1c, 0x87, 0xd8, 0x58, 0xeb, - 0xf2, 0xc8, 0x94, 0xe4, 0x5d, 0x42, 0x3e, 0xd2, 0x64, 0x03, 0x2c, 0x3a, 0xf1, 0xd9, 0xb4, 0x46, - 0x09, 0xce, 0x4e, 0x79, 0xb3, 0xb1, 0x6d, 0xfb, 0xb3, 0xe5, 0x39, 0x98, 0xbe, 0xa7, 0x19, 0xc4, - 0x5b, 0x8c, 0x1b, 0x08, 0x23, 0xae, 0xe9, 0xf2, 0x27, 0x09, 0x98, 0x79, 0x0f, 0x93, 0x90, 0xd7, - 0x8f, 0x13, 0x37, 0x8f, 0xd7, 0x2a, 0x4c, 0x64, 0xc7, 0x30, 0xb1, 0xca, 0x4d, 0x20, 0x22, 0x70, - 0x81, 0x13, 0x9b, 0x8c, 0x16, 0x9e, 0x84, 0x6d, 0xdb, 0x2b, 0x48, 0xdc, 0x49, 0xd8, 0xb6, 0x51, - 0x15, 0x5e, 0x12, 0x93, 0x42, 0xb7, 0x8b, 0x09, 0xde, 0x1e, 0xe6, 0x43, 0x7e, 0xe5, 0xe4, 0xc8, - 0x3f, 0x4c, 0xc0, 0xe9, 0xad, 0x9e, 0xae, 0x11, 0xfc, 0x57, 0x62, 0x0a, 0xc5, 0x43, 0x13, 0x21, - 0x27, 0x79, 0x94, 0x00, 0x26, 0x30, 0x45, 0x84, 0x3a, 0x2c, 0x1b, 0xa4, 0x8e, 0x35, 0x1b, 0xc8, - 0xff, 0x2d, 0xc1, 0xa9, 0x5b, 0x86, 0xa9, 0xaf, 0x1a, 0x8e, 0x63, 0x98, 0xed, 0xc5, 0x8e, 0xb5, - 0xe3, 0x8c, 0x65, 0xc9, 0xbb, 0x50, 0xd8, 0xe9, 0x58, 0x3b, 0xc2, 0x8e, 0xee, 0x9d, 0x70, 0x64, - 0x43, 0xe6, 0x29, 0x33, 0x7f, 0x76, 0xe4, 0x3e, 0x94, 0x87, 0x75, 0x11, 0xd9, 0xe1, 0x03, 0x98, - 0xee, 0x72, 0xba, 0xfa, 0x22, 0xf2, 0x50, 0xd7, 0x07, 0x77, 0xc5, 0xfe, 0x51, 0x82, 0x53, 0x8b, - 0x1a, 0x69, 0xed, 0x72, 0xa7, 0x1a, 0xdf, 0x06, 0x1f, 0x42, 0xd6, 0xe6, 0xf3, 0x5d, 0x7d, 0xe2, - 0xd3, 0xc3, 0x21, 0x02, 0xab, 0xe2, 0xaf, 0xe2, 0x21, 0x56, 0x1e, 0x40, 0xc6, 0xd5, 0xe6, 0x85, - 0xd3, 0x33, 0x82, 0x14, 0xbb, 0xe4, 0x24, 0xf8, 0xf7, 0x2a, 0xfa, 0x2c, 0xff, 0x5e, 0x82, 0xf2, - 0xb0, 0x36, 0xc2, 0xec, 0x1f, 0x41, 0xce, 0x16, 0xcf, 0x6e, 0x33, 0x79, 0xf1, 0x08, 0x6b, 0x13, - 0x39, 0xd0, 0x7d, 0x50, 0x7c, 0xd0, 0xca, 0x3e, 0x64, 0x3d, 0x69, 0x2f, 0xbc, 0x3e, 0x3f, 0xaf, - 0x27, 0xe2, 0xf2, 0xba, 0xfc, 0xef, 0x70, 0x92, 0x29, 0x4a, 0x8b, 0x92, 0xf1, 0xf7, 0x7c, 0x01, - 0x32, 0x47, 0x74, 0x41, 0x97, 0x4f, 0xfe, 0xcf, 0x04, 0xcc, 0x0c, 0x6a, 0x20, 0x0c, 0xf1, 0x60, - 0xd8, 0xec, 0x23, 0xba, 0xd4, 0x10, 0x56, 0xa4, 0xd1, 0xff, 0x47, 0x3a, 0x4e, 0xab, 0x47, 0x78, - 0xd5, 0x38, 0x15, 0x96, 0xfc, 0x5d, 0x09, 0xa6, 0xde, 0xc3, 0x84, 0x5e, 0x09, 0xc7, 0xda, 0x83, - 0xdb, 0x90, 0x7f, 0x81, 0x6f, 0xc4, 0x60, 0xfb, 0x9f, 0x87, 0xcf, 0x40, 0xae, 0xa7, 0xb5, 0xb1, - 0xea, 0x18, 0x4f, 0x79, 0x66, 0xa7, 0xf7, 0x14, 0xad, 0x8d, 0x9b, 0xc6, 0x53, 0xd6, 0x01, 0x61, - 0x83, 0xc4, 0x7a, 0x84, 0x4d, 0x91, 0xc9, 0xd9, 0xf4, 0x4d, 0x4a, 0x90, 0x3f, 0x95, 0xa0, 0xe8, - 0x69, 0x2f, 0x4c, 0xba, 0x12, 0xfe, 0x9a, 0x25, 0x8d, 0x7d, 0x37, 0x0d, 0x7d, 0xc9, 0x7a, 0x0d, - 0x8a, 0x26, 0x7e, 0x42, 0xd4, 0x80, 0x16, 0xfc, 0x2a, 0x38, 0x49, 0xc9, 0x1b, 0x9e, 0x26, 0x6f, - 0xb3, 0xfa, 0x60, 0x49, 0xeb, 0x69, 0x3b, 0x46, 0xc7, 0x60, 0x57, 0xbb, 0x31, 0xcc, 0x29, 0xff, - 0x20, 0x09, 0x88, 0x17, 0x9b, 0x41, 0x08, 0xa4, 0x01, 0xe2, 0x19, 0xa8, 0x15, 0xa0, 0x0a, 0x57, - 0x89, 0xff, 0xf4, 0xc8, 0x92, 0x4d, 0x48, 0xa5, 0x13, 0xad, 0x41, 0x12, 0xea, 0xc2, 0x4c, 0xa0, - 0x1f, 0x1b, 0x14, 0xc3, 0xf7, 0xf4, 0xc6, 0xe8, 0xd5, 0x60, 0x48, 0xd4, 0x49, 0x1c, 0x45, 0x46, - 0x1b, 0x30, 0xa3, 0xe3, 0x9e, 0x8d, 0x5b, 0x1a, 0xc1, 0xba, 0xaa, 0xf5, 0x0c, 0x75, 0x0f, 0xdb, - 0x8e, 0x61, 0x99, 0x5e, 0x03, 0x3f, 0x28, 0x4e, 0xfc, 0x68, 0xa3, 0x89, 0xbb, 0xdb, 0xd8, 0x56, - 0xa6, 0x7d, 0xce, 0x85, 0x9e, 0xb1, 0xcd, 0xf9, 0xd0, 0x22, 0x14, 0x3b, 0xd6, 0x7e, 0x08, 0x2a, - 0x15, 0x0b, 0x35, 0xd9, 0xb1, 0xf6, 0x03, 0x18, 0xcb, 0x50, 0xda, 0x35, 0xda, 0xbb, 0x21, 0x90, - 0x89, 0x58, 0x90, 0x29, 0xca, 0xe3, 0xa3, 0xc8, 0x0e, 0x4c, 0x71, 0x9f, 0xbe, 0xd5, 0x37, 0x59, - 0x3d, 0x21, 0x6b, 0x91, 0x75, 0x3c, 0x40, 0xba, 0x79, 0x7b, 0xa1, 0x7e, 0xfd, 0x46, 0x49, 0x42, - 0x59, 0x48, 0x35, 0x6f, 0x2f, 0x5c, 0x2d, 0x25, 0x50, 0x06, 0x92, 0xab, 0xcb, 0xd7, 0x4b, 0x49, - 0xfa, 0xb0, 0xdd, 0x5c, 0x2f, 0xa5, 0xc4, 0xbc, 0x6b, 0x37, 0xdf, 0x28, 0x4d, 0x88, 0xe7, 0xeb, - 0x57, 0xeb, 0xa5, 0x34, 0x05, 0x5b, 0xdd, 0x52, 0x56, 0xb7, 0x94, 0x6b, 0xa5, 0x8c, 0x7c, 0x0b, - 0xce, 0xf1, 0x52, 0x86, 0xed, 0x36, 0x8f, 0xfc, 0x21, 0x8b, 0xbf, 0x0a, 0x53, 0x7d, 0x46, 0x55, - 0xb1, 0x49, 0x2f, 0x80, 0x3a, 0xf3, 0x9f, 0xac, 0x32, 0xc9, 0xa9, 0x0d, 0x4e, 0x94, 0x7f, 0x22, - 0xc1, 0xf4, 0x86, 0x68, 0x0f, 0x84, 0xf8, 0x5b, 0x00, 0xa2, 0x6d, 0xe0, 0x1f, 0xa7, 0xa5, 0xf8, - 0xcf, 0xde, 0x11, 0x50, 0x1e, 0x51, 0xd1, 0xcc, 0x36, 0x56, 0x02, 0xb0, 0x95, 0x2d, 0x98, 0x0c, - 0x0d, 0xa2, 0x8b, 0x50, 0xe8, 0x1a, 0xa6, 0x3a, 0xd0, 0xc0, 0xc8, 0x77, 0x0d, 0xd3, 0x9d, 0xc7, - 0xa6, 0x68, 0x4f, 0xfc, 0x29, 0x09, 0x31, 0x45, 0x7b, 0xe2, 0x4e, 0x91, 0x37, 0xe0, 0x8c, 0x68, - 0xef, 0x2c, 0xec, 0x38, 0x56, 0xa7, 0x4f, 0xf0, 0x86, 0x46, 0x76, 0x9b, 0xc4, 0xd6, 0x08, 0x6e, - 0x1f, 0xc8, 0x57, 0x23, 0xb7, 0x67, 0x0a, 0x60, 0xf9, 0x4e, 0x73, 0x61, 0x65, 0x65, 0xfd, 0x5e, - 0x63, 0xb9, 0x24, 0xd1, 0x41, 0xf7, 0x25, 0x21, 0x7f, 0x27, 0x05, 0x27, 0x86, 0xce, 0x15, 0x7a, - 0x00, 0x45, 0x1e, 0x08, 0xd5, 0x87, 0x62, 0xeb, 0x99, 0xa1, 0x46, 0xb9, 0x4b, 0x85, 0x3d, 0x46, - 0xdc, 0xa5, 0xa6, 0xf4, 0x10, 0x15, 0x7d, 0x2a, 0xc1, 0x05, 0x51, 0xe6, 0xf2, 0x78, 0x20, 0x76, - 0x34, 0xe2, 0xbc, 0xbe, 0x33, 0x62, 0xe5, 0x7b, 0x88, 0xbb, 0x28, 0xe7, 0xb4, 0xe7, 0x7a, 0x53, - 0x1f, 0xce, 0x88, 0x9a, 0x58, 0xd8, 0x38, 0xac, 0x03, 0x3f, 0xc4, 0xd7, 0x8f, 0xe4, 0x1e, 0xca, - 0x69, 0x86, 0x1c, 0xe9, 0x84, 0xf3, 0x50, 0xa1, 0x7b, 0xbd, 0x43, 0xd3, 0xac, 0x4a, 0x2c, 0xa2, - 0x75, 0xd4, 0xc0, 0x07, 0xca, 0x14, 0xfb, 0x40, 0x39, 0xd3, 0xd5, 0x9e, 0xb0, 0x3c, 0xbc, 0x49, - 0xc7, 0x9b, 0xee, 0xd7, 0x4a, 0x6a, 0xbc, 0x97, 0x45, 0xfb, 0x5e, 0xd5, 0x84, 0x1b, 0xb0, 0x5f, - 0xb7, 0xd0, 0x5b, 0x22, 0x73, 0x04, 0x76, 0xd6, 0xa7, 0x46, 0xa8, 0xae, 0x9e, 0xe3, 0x4c, 0x62, - 0xe7, 0xce, 0x38, 0xcf, 0xf1, 0xb7, 0x9f, 0x25, 0xe0, 0x64, 0x64, 0xb4, 0x8c, 0x76, 0x20, 0xe9, - 0xf8, 0x1c, 0xe8, 0x22, 0x14, 0x28, 0x93, 0x17, 0x02, 0x78, 0x63, 0x26, 0x4f, 0x69, 0x22, 0x00, - 0xa0, 0x67, 0x70, 0x3e, 0xd0, 0x60, 0x3b, 0xfe, 0xdd, 0xf5, 0x3f, 0xfb, 0x1d, 0xb2, 0xc3, 0xa7, - 0x9d, 0x7e, 0xaf, 0x67, 0xd9, 0x34, 0x2f, 0x0c, 0x37, 0x57, 0xe9, 0x7d, 0xf4, 0x94, 0x37, 0x21, - 0xdc, 0x44, 0x95, 0x57, 0x21, 0xbf, 0x69, 0x59, 0x9d, 0x65, 0x4c, 0x34, 0xa3, 0xc3, 0xbe, 0xa9, - 0x10, 0xcb, 0xea, 0x04, 0xb3, 0x6d, 0x96, 0x12, 0x58, 0xe1, 0x72, 0x11, 0x0a, 0x6c, 0xd0, 0x0d, - 0xf3, 0x3c, 0x9b, 0xe7, 0x29, 0xcd, 0x8d, 0xe3, 0xbf, 0x96, 0xa0, 0x28, 0xb2, 0xb7, 0xd7, 0x23, - 0x59, 0x17, 0x6c, 0x3a, 0x97, 0x21, 0x72, 0x70, 0xfc, 0xaf, 0x5a, 0x02, 0x7a, 0x71, 0x21, 0x01, - 0x25, 0xc5, 0x89, 0x36, 0x74, 0xa1, 0x44, 0x96, 0x13, 0xee, 0xe8, 0xe8, 0x75, 0x40, 0x4c, 0x9a, - 0x61, 0xee, 0x59, 0x2d, 0xcd, 0x9d, 0x25, 0xda, 0x1e, 0x74, 0xe4, 0x8e, 0x37, 0x70, 0x47, 0xa7, - 0xa6, 0x6b, 0x59, 0xb6, 0x8d, 0x3b, 0x2c, 0xa7, 0xfa, 0x3c, 0x0e, 0x65, 0xe2, 0x35, 0xd3, 0x29, - 0x7f, 0x82, 0xcf, 0xea, 0xdc, 0xd1, 0xeb, 0xdf, 0x4e, 0x40, 0xce, 0x73, 0x49, 0xf4, 0x99, 0x04, - 0x19, 0xd1, 0x26, 0x42, 0xb5, 0xd1, 0xdb, 0x6c, 0xcc, 0x54, 0x95, 0x73, 0x6e, 0x99, 0x19, 0xf8, - 0xdd, 0x64, 0xd5, 0xeb, 0x36, 0xc9, 0x57, 0xff, 0xe3, 0xa7, 0xbf, 0xfa, 0xff, 0xc4, 0x25, 0xf9, - 0xb5, 0xda, 0x5e, 0xbd, 0xf6, 0x71, 0xa8, 0x24, 0x7a, 0x7b, 0x6e, 0xee, 0x59, 0x8d, 0x2f, 0xde, - 0x99, 0xe7, 0x22, 0xf0, 0xbc, 0x34, 0x77, 0x45, 0x42, 0x5f, 0x93, 0x60, 0x32, 0xd4, 0xa4, 0x41, - 0xf1, 0xde, 0x17, 0xd5, 0xd4, 0x19, 0x4f, 0x39, 0xa6, 0x93, 0xff, 0x8b, 0xcf, 0xda, 0xdc, 0xdc, - 0xb3, 0xf9, 0xfd, 0x20, 0x2a, 0x53, 0xae, 0xfe, 0xf3, 0x24, 0xe4, 0x03, 0xc1, 0x14, 0xfd, 0x82, - 0x57, 0xa3, 0xa1, 0xef, 0xb6, 0xf1, 0x1f, 0xb3, 0xa2, 0xdb, 0x4a, 0x95, 0xf1, 0x3a, 0x18, 0xf2, - 0x87, 0x6c, 0x01, 0xdb, 0x68, 0xf3, 0xb9, 0xd6, 0x15, 0x9d, 0x89, 0xda, 0xc7, 0xa1, 0x0e, 0x4c, - 0x75, 0x57, 0x73, 0x76, 0x9f, 0x0d, 0x12, 0xfd, 0xf8, 0xfa, 0x0c, 0xfd, 0x4e, 0x02, 0x34, 0xdc, - 0xf6, 0x41, 0xf3, 0xb1, 0x3a, 0x1e, 0xda, 0x2b, 0x1a, 0x77, 0x7d, 0x8f, 0xd8, 0xfa, 0x70, 0xe5, - 0x4b, 0x59, 0xdf, 0x7c, 0xb8, 0x87, 0x54, 0xff, 0x3c, 0x0d, 0xa7, 0x97, 0xf8, 0x77, 0xb9, 0x05, - 0x5d, 0xb7, 0xb1, 0xe3, 0xd0, 0x10, 0xd9, 0x24, 0x96, 0xad, 0xb5, 0x31, 0xfa, 0x9e, 0x04, 0xa5, - 0xc1, 0x5e, 0x09, 0xba, 0x39, 0xc2, 0xaf, 0xee, 0x22, 0x5b, 0x3d, 0x95, 0xb7, 0x8e, 0xc0, 0xc9, - 0xaf, 0x3a, 0xf2, 0x35, 0x66, 0x94, 0xcb, 0xf2, 0xec, 0x21, 0x46, 0xd9, 0xa1, 0xb3, 0xe7, 0x1f, - 0xfa, 0xec, 0xf3, 0xd2, 0x1c, 0x53, 0x7f, 0xb0, 0x4b, 0x30, 0x82, 0xfa, 0x87, 0x34, 0x4d, 0x46, - 0x50, 0xff, 0xb0, 0x96, 0xc4, 0x88, 0xea, 0xef, 0xf8, 0xec, 0x54, 0xfd, 0x6f, 0x49, 0x30, 0x15, - 0xbe, 0x6d, 0xa3, 0x1b, 0x63, 0x5f, 0xcf, 0xb9, 0xea, 0x6f, 0x1e, 0xf1, 0x5a, 0x1f, 0x1b, 0xca, - 0x02, 0x8a, 0x53, 0x66, 0xaa, 0xf6, 0x17, 0x12, 0x64, 0xc4, 0x4d, 0x75, 0x84, 0xc8, 0x1a, 0xbe, - 0x91, 0x57, 0xae, 0x8c, 0xce, 0x20, 0x34, 0xbc, 0xcf, 0x34, 0x54, 0xd0, 0xc6, 0xf3, 0x34, 0xac, - 0x7d, 0x1c, 0xb8, 0xc2, 0xbb, 0x87, 0x24, 0x48, 0x0a, 0x1e, 0x91, 0x36, 0x97, 0x70, 0x45, 0xaa, - 0x7f, 0x5f, 0x82, 0x42, 0x28, 0x75, 0x7f, 0x93, 0xc7, 0xbd, 0x10, 0x6d, 0xa4, 0xb8, 0x17, 0x71, - 0x5d, 0xae, 0xc4, 0xb7, 0x57, 0x87, 0xef, 0xc9, 0xf2, 0x25, 0xb6, 0xdc, 0x57, 0xd1, 0x2b, 0x87, - 0x2c, 0x37, 0x58, 0xc5, 0x2c, 0xda, 0x10, 0xf7, 0x0f, 0x04, 0x8b, 0xd3, 0x0a, 0x23, 0xfa, 0xdf, - 0x0d, 0x6d, 0x8b, 0x58, 0x1b, 0xd2, 0x3f, 0x17, 0xf9, 0x64, 0x6f, 0xee, 0xd7, 0x13, 0x49, 0xa5, - 0x71, 0xff, 0x1b, 0x89, 0xf3, 0x8b, 0x0c, 0x70, 0x91, 0x01, 0x72, 0x5e, 0xff, 0x5a, 0x5c, 0xdd, - 0xae, 0xef, 0xa4, 0xd9, 0x6f, 0x80, 0xae, 0xfd, 0x29, 0x00, 0x00, 0xff, 0xff, 0xde, 0x45, 0x55, - 0xfb, 0xf3, 0x30, 0x00, 0x00, + // 3491 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x5b, 0xcb, 0x93, 0x1b, 0x57, + 0x57, 0xff, 0x5a, 0xd2, 0xe8, 0x71, 0xa4, 0x19, 0x69, 0x6e, 0xc6, 0x63, 0x8d, 0x6c, 0xc7, 0x76, + 0xa7, 0xf2, 0x31, 0x8c, 0x63, 0xc9, 0x96, 0x63, 0xc7, 0x19, 0x48, 0xc2, 0x3c, 0xe4, 0x78, 0x9c, + 0x79, 0xd1, 0x9a, 0x19, 0x3b, 0x21, 0xb8, 0xd3, 0xa3, 0xbe, 0xd6, 0x34, 0x96, 0xfa, 0xca, 0xdd, + 0xad, 0x79, 0x38, 0xe5, 0xa2, 0x8a, 0xaa, 0x10, 0x0a, 0xaa, 0x42, 0x51, 0xb0, 0x81, 0x6c, 0x28, + 0x8a, 0x15, 0xc5, 0x92, 0x0d, 0x05, 0x6c, 0x58, 0x85, 0x82, 0x15, 0x14, 0xf0, 0x0f, 0xb0, 0xe1, + 0xb1, 0x65, 0x43, 0xb1, 0xa0, 0xee, 0xa3, 0x5f, 0x52, 0x8f, 0x5b, 0x1a, 0x4f, 0x80, 0xd5, 0x74, + 0x9f, 0x7b, 0xcf, 0xef, 0x9c, 0x7b, 0xee, 0xb9, 0xe7, 0x9c, 0x7b, 0x5a, 0x03, 0xf7, 0xf6, 0xfb, + 0x46, 0x47, 0xaf, 0xed, 0x6b, 0x2f, 0x71, 0xa7, 0x66, 0xe1, 0x2e, 0x71, 0x70, 0x0d, 0x1f, 0xe3, + 0x56, 0xdf, 0x31, 0x88, 0x59, 0x3b, 0xac, 0x0b, 0x9a, 0xea, 0xd1, 0xaa, 0x3d, 0x8b, 0x38, 0x04, + 0x5d, 0x65, 0x7c, 0x55, 0xc6, 0x57, 0xe5, 0x73, 0xaa, 0xfe, 0x9c, 0xc3, 0x7a, 0xe5, 0x6a, 0x10, + 0xd8, 0xc6, 0xdd, 0x43, 0x6c, 0x89, 0x3f, 0x1c, 0xa1, 0x72, 0xb9, 0x4d, 0x48, 0xbb, 0x83, 0x6b, + 0x5a, 0xcf, 0xa8, 0x69, 0xa6, 0x49, 0x1c, 0x8d, 0xb2, 0xda, 0x62, 0xf4, 0x1d, 0x31, 0xda, 0x21, + 0x66, 0xdb, 0xea, 0x9b, 0xa6, 0x61, 0xb6, 0x6b, 0xa4, 0x87, 0xad, 0xd0, 0xa4, 0x39, 0x31, 0x89, + 0xbd, 0xed, 0xf7, 0x9f, 0xd5, 0x34, 0xf3, 0x44, 0x0c, 0xbd, 0x3d, 0x38, 0xa4, 0xf7, 0x39, 0xaf, + 0x18, 0xbf, 0x3a, 0x38, 0xee, 0x18, 0x5d, 0x6c, 0x3b, 0x5a, 0xb7, 0x77, 0x1a, 0xc0, 0x91, 0xa5, + 0xf5, 0x7a, 0xd8, 0x72, 0x65, 0x5f, 0x14, 0xe3, 0x56, 0xaf, 0x55, 0xb3, 0x1d, 0xcd, 0xe9, 0x8b, + 0x01, 0xf9, 0xdf, 0x13, 0x90, 0x5e, 0x6a, 0x51, 0x51, 0x68, 0x13, 0xa6, 0x5a, 0xa4, 0xdb, 0xd5, + 0x4c, 0x5d, 0xd5, 0x8d, 0x36, 0xb6, 0x9d, 0xb2, 0x74, 0x4d, 0x9a, 0xcf, 0xd7, 0x7f, 0xa6, 0x1a, + 0x63, 0xbd, 0xea, 0x2a, 0x9b, 0xae, 0x4c, 0x0a, 0x76, 0xfe, 0x8a, 0x9a, 0x30, 0x6d, 0x98, 0xbd, + 0xbe, 0xa3, 0x5a, 0x84, 0x38, 0x2e, 0x64, 0x62, 0x3c, 0xc8, 0x22, 0x43, 0x50, 0x08, 0x71, 0x04, + 0xe8, 0x1d, 0xc8, 0xd0, 0xb5, 0x93, 0xbe, 0x53, 0x4e, 0x33, 0xa8, 0xb9, 0x2a, 0x5f, 0x5a, 0xd5, + 0x5d, 0x7a, 0x75, 0x55, 0xd8, 0x4e, 0x71, 0x67, 0xa2, 0x6b, 0x50, 0xd0, 0x89, 0x6a, 0x12, 0x47, + 0x6d, 0x69, 0xad, 0x03, 0x5c, 0xce, 0x5c, 0x93, 0xe6, 0xb3, 0x0a, 0xe8, 0x64, 0x93, 0x38, 0x2b, + 0x94, 0x82, 0x10, 0xa4, 0x6c, 0xad, 0xe3, 0x94, 0x73, 0xd7, 0xa4, 0xf9, 0x82, 0xc2, 0x9e, 0x51, + 0x03, 0xb2, 0xbd, 0x8e, 0xe6, 0x3c, 0x23, 0x56, 0xb7, 0x0c, 0x4c, 0xd6, 0xcf, 0xc6, 0xaa, 0xbd, + 0x2d, 0x18, 0x14, 0x8f, 0xf5, 0x51, 0x2a, 0x9b, 0x2c, 0xa5, 0x1f, 0xa5, 0xb2, 0xd9, 0x52, 0x4e, + 0xfe, 0xaf, 0x24, 0x64, 0x56, 0xb8, 0x91, 0xd0, 0x65, 0xc8, 0x69, 0x56, 0xbb, 0xdf, 0xc5, 0xa6, + 0x63, 0x97, 0xa5, 0x6b, 0xc9, 0xf9, 0x9c, 0xe2, 0x13, 0xd0, 0x0b, 0xb8, 0x80, 0xcd, 0x43, 0xc3, + 0x22, 0x26, 0x7d, 0x57, 0x0f, 0x35, 0xcb, 0xd0, 0xf6, 0x3b, 0xd8, 0x2e, 0x27, 0xae, 0x25, 0xe7, + 0xf3, 0xf5, 0x9f, 0x8f, 0xd5, 0x44, 0x88, 0xa9, 0x36, 0x7c, 0x94, 0x3d, 0x01, 0xa2, 0xcc, 0xe0, + 0x61, 0xa2, 0x8d, 0xae, 0x43, 0x81, 0xf4, 0x1d, 0xba, 0x61, 0xcf, 0x0c, 0x2a, 0x29, 0xc9, 0x74, + 0xca, 0x73, 0xda, 0x03, 0x4a, 0x42, 0x37, 0x01, 0x89, 0x29, 0xba, 0x61, 0xe1, 0x96, 0x43, 0x2c, + 0x03, 0xdb, 0xe5, 0x14, 0x9b, 0x38, 0xcd, 0x47, 0x56, 0xfd, 0x81, 0x00, 0x62, 0x4f, 0x73, 0x0e, + 0xec, 0x72, 0x26, 0x88, 0xb8, 0x4d, 0x49, 0x21, 0x23, 0x4f, 0x9c, 0xd9, 0xc8, 0xe8, 0x06, 0x4c, + 0x1f, 0x11, 0xeb, 0xb9, 0x61, 0xb6, 0x3d, 0xcd, 0x4e, 0x98, 0x83, 0xe4, 0x94, 0x92, 0x18, 0x70, + 0x15, 0x3b, 0x41, 0xef, 0xc3, 0xac, 0x50, 0xcb, 0x24, 0x3a, 0x56, 0x7b, 0x16, 0x3d, 0xaa, 0x0e, + 0x5d, 0x49, 0x96, 0x29, 0x38, 0xc3, 0x47, 0x37, 0x89, 0x8e, 0xb7, 0xbd, 0xb1, 0xca, 0x27, 0xf0, + 0x56, 0x84, 0x2d, 0xa9, 0xe7, 0x98, 0x5a, 0x17, 0xb3, 0xb3, 0x92, 0x53, 0xd8, 0x33, 0x9a, 0x81, + 0x89, 0x43, 0xad, 0xd3, 0xc7, 0xcc, 0xdb, 0x73, 0x0a, 0x7f, 0x91, 0x7f, 0x4f, 0x82, 0xac, 0xab, + 0x3a, 0x52, 0x00, 0x02, 0x72, 0x25, 0xb6, 0xa9, 0xf5, 0x91, 0x57, 0x5e, 0x15, 0x7a, 0x9d, 0x28, + 0x01, 0x94, 0xca, 0xfb, 0x90, 0x75, 0xe9, 0x63, 0xa8, 0xf5, 0x43, 0x02, 0x72, 0xbe, 0x6d, 0x3e, + 0x81, 0x09, 0xbe, 0xfb, 0x5c, 0xa5, 0xf8, 0xcd, 0xa0, 0x8e, 0x41, 0x2d, 0xa5, 0x70, 0x3e, 0xb4, + 0x0d, 0xf9, 0xa0, 0x6f, 0x70, 0x77, 0xad, 0x8e, 0x70, 0xde, 0x85, 0x06, 0x0c, 0x2b, 0x08, 0x81, + 0x1e, 0x42, 0xd6, 0x3e, 0xe9, 0x76, 0x0c, 0xf3, 0x39, 0xf7, 0xc9, 0x7c, 0xfd, 0xbd, 0x58, 0xb8, + 0x26, 0x67, 0x60, 0x60, 0x1e, 0x37, 0x7a, 0x02, 0xc5, 0xc1, 0x1d, 0xe7, 0x3e, 0x57, 0x8b, 0x05, + 0x0c, 0x3b, 0x83, 0x32, 0x65, 0x86, 0xde, 0x1f, 0xa5, 0xb2, 0xa9, 0xd2, 0x84, 0x7c, 0x1f, 0x0a, + 0x81, 0x79, 0xe3, 0x6c, 0xc2, 0xdf, 0x49, 0x30, 0x15, 0x16, 0x81, 0x36, 0x22, 0x3c, 0xe4, 0xe6, + 0x38, 0x7a, 0x86, 0x9c, 0x03, 0xdd, 0x82, 0x89, 0x2e, 0x8d, 0x87, 0x22, 0x02, 0x57, 0x86, 0xc2, + 0xe6, 0x8e, 0x9b, 0x52, 0x14, 0x3e, 0x11, 0x7d, 0x08, 0xb9, 0xbe, 0x69, 0x1c, 0xab, 0x5d, 0xa2, + 0xe3, 0x72, 0x92, 0x71, 0x5d, 0x1e, 0xe2, 0xda, 0x5d, 0x33, 0x9d, 0x3b, 0xf5, 0x3d, 0xba, 0x08, + 0x25, 0x4b, 0xa7, 0x6f, 0x10, 0x1d, 0xcb, 0xff, 0x26, 0x41, 0xd6, 0x75, 0x8c, 0x48, 0x2b, 0x7c, + 0x02, 0xe9, 0xb3, 0x25, 0x04, 0xc1, 0x86, 0xde, 0x81, 0x49, 0xc3, 0x16, 0x79, 0x9e, 0x9e, 0xc3, + 0x72, 0x8a, 0xc5, 0xf4, 0x82, 0x61, 0x37, 0x3c, 0x5a, 0xd4, 0x7e, 0xa7, 0xcf, 0x6b, 0xbf, 0x93, + 0xa5, 0xd4, 0xa3, 0x54, 0x76, 0xa2, 0x94, 0x96, 0x75, 0x98, 0x0c, 0x79, 0xef, 0x8f, 0xb2, 0x60, + 0xf9, 0xf7, 0x25, 0xc8, 0x07, 0xbc, 0x3a, 0x52, 0xc8, 0x2c, 0xa4, 0x1d, 0xcd, 0x6a, 0x63, 0x47, + 0x38, 0x97, 0x78, 0x8b, 0xb2, 0x43, 0xea, 0x1c, 0xed, 0x20, 0xff, 0x1c, 0xa4, 0x45, 0x7a, 0x46, + 0x90, 0x3a, 0xd0, 0xec, 0x03, 0x57, 0x2b, 0xfa, 0x8c, 0xae, 0x00, 0xd8, 0xc6, 0x4b, 0xac, 0xee, + 0x9f, 0x38, 0x2c, 0x20, 0x48, 0xf3, 0x49, 0x25, 0x47, 0x29, 0xcb, 0x94, 0x20, 0xff, 0x71, 0x06, + 0x66, 0xf9, 0x9e, 0x61, 0x9d, 0x57, 0x22, 0x1b, 0xd8, 0xd1, 0x74, 0xcd, 0xd1, 0xe8, 0x7a, 0x68, + 0xf0, 0xc6, 0x96, 0xc0, 0x13, 0x6f, 0xa8, 0x01, 0xa5, 0x17, 0x7d, 0xdc, 0xc7, 0xba, 0xea, 0xd5, + 0x41, 0x23, 0xb8, 0x75, 0x91, 0xf3, 0x78, 0x04, 0xb4, 0x0d, 0xb3, 0x1c, 0x50, 0xb5, 0x1d, 0xcd, + 0x72, 0x02, 0x60, 0xc9, 0x58, 0xb0, 0x19, 0xce, 0xd9, 0xa4, 0x8c, 0x3e, 0xe2, 0x13, 0xa8, 0x08, + 0xc4, 0x16, 0xe9, 0xf6, 0x3a, 0xd8, 0x09, 0xa9, 0x98, 0x8a, 0x45, 0x2d, 0x73, 0xee, 0x15, 0x97, + 0xd9, 0x47, 0xfe, 0x1c, 0x2e, 0xf1, 0x62, 0xea, 0x19, 0x76, 0x5a, 0x07, 0x43, 0x0a, 0x4f, 0xc4, + 0x43, 0x33, 0xf6, 0x07, 0x94, 0x7b, 0x40, 0x69, 0x0d, 0xae, 0x06, 0xa1, 0xa3, 0x34, 0x4f, 0xc7, + 0xc2, 0x5f, 0xf6, 0xe1, 0x23, 0xb4, 0xdf, 0x83, 0x39, 0xcf, 0xab, 0x86, 0x74, 0xcf, 0xc4, 0x82, + 0x5f, 0xf4, 0x98, 0x07, 0x54, 0x7f, 0x0a, 0x57, 0x7c, 0xdc, 0x28, 0xc5, 0xb3, 0xb1, 0xd8, 0x97, + 0x3c, 0x80, 0x08, 0xbd, 0x7f, 0x19, 0xae, 0x88, 0x4a, 0xa1, 0xdf, 0xeb, 0x10, 0x4d, 0x1f, 0xd2, + 0x3d, 0x17, 0x8b, 0x5f, 0xe1, 0x00, 0xbb, 0x8c, 0x7f, 0x40, 0x7d, 0x0c, 0xd7, 0xc3, 0xf0, 0x51, + 0x4b, 0x80, 0x58, 0x11, 0x6f, 0x07, 0x45, 0x44, 0xac, 0x62, 0x05, 0x90, 0xd6, 0x3f, 0x36, 0x3a, + 0x86, 0x66, 0x9d, 0xa8, 0x5d, 0x71, 0xb8, 0xca, 0x79, 0x96, 0x51, 0x66, 0x86, 0x70, 0x97, 0xcc, + 0x13, 0x65, 0xda, 0x9b, 0xef, 0x9e, 0x45, 0xf9, 0x6f, 0xd3, 0x50, 0xe0, 0xc7, 0x53, 0xc1, 0x76, + 0xbf, 0xe3, 0xa0, 0xcd, 0x81, 0x72, 0x91, 0x67, 0xfa, 0x1b, 0xb1, 0x11, 0x65, 0xcb, 0xab, 0x27, + 0xc3, 0xb5, 0xe5, 0x57, 0x30, 0x13, 0xc0, 0x53, 0xbd, 0x94, 0x0f, 0x23, 0x56, 0x10, 0x1c, 0x57, + 0x84, 0x48, 0x05, 0xf9, 0xd0, 0x4d, 0x37, 0xfd, 0x3f, 0x86, 0xa2, 0x90, 0xe0, 0x81, 0x17, 0xce, + 0x04, 0x3e, 0x45, 0x82, 0xaf, 0x36, 0x52, 0x23, 0xcb, 0x62, 0x5e, 0xab, 0xdc, 0x1a, 0x11, 0xdb, + 0x4b, 0x21, 0x51, 0x85, 0xf4, 0xaf, 0xc0, 0xdc, 0x80, 0x80, 0x13, 0x7f, 0x0d, 0xf9, 0x33, 0xad, + 0xe1, 0x62, 0x58, 0xca, 0x89, 0xb7, 0x98, 0x4b, 0x90, 0xc3, 0xc7, 0x86, 0xa3, 0xb6, 0x68, 0xda, + 0xa7, 0x21, 0x6b, 0x42, 0xc9, 0x52, 0xc2, 0x0a, 0xcd, 0x3a, 0x34, 0x96, 0x3b, 0x3a, 0xa1, 0x97, + 0x3a, 0xed, 0x88, 0x45, 0x9d, 0x82, 0x92, 0xe3, 0x14, 0x45, 0x3b, 0x42, 0xeb, 0x30, 0x29, 0x86, + 0x45, 0xb2, 0x4b, 0x8f, 0x97, 0xec, 0x0a, 0x9c, 0x5b, 0x24, 0x13, 0x2e, 0x0c, 0x5b, 0x16, 0x13, + 0x96, 0xf1, 0x84, 0x61, 0xcb, 0xf2, 0x85, 0xd1, 0x61, 0x21, 0x2c, 0x3b, 0xbe, 0x30, 0x6c, 0x59, + 0x42, 0xd8, 0x33, 0x40, 0x7e, 0x28, 0xf1, 0x0e, 0x09, 0x3f, 0xdf, 0x1f, 0xc4, 0x42, 0x46, 0x27, + 0x30, 0x65, 0xda, 0x9b, 0xe4, 0x92, 0x1e, 0xa5, 0xb2, 0x52, 0x29, 0x21, 0xff, 0x46, 0x02, 0xc0, + 0x3f, 0x08, 0x34, 0x6d, 0xd2, 0x1b, 0x92, 0x9b, 0x36, 0xe9, 0xf3, 0xff, 0x52, 0x89, 0x54, 0x81, + 0x6c, 0x8b, 0x98, 0x0e, 0xbb, 0x84, 0xf2, 0xed, 0xf4, 0xde, 0xa3, 0xca, 0x86, 0xcc, 0xf9, 0x96, + 0x4f, 0xe9, 0x52, 0x46, 0xfe, 0x4e, 0x82, 0xd4, 0x8e, 0x85, 0x31, 0xfa, 0x18, 0x52, 0x16, 0x21, + 0x6e, 0xd7, 0x61, 0x61, 0xf4, 0x2b, 0x83, 0xc2, 0xf8, 0xd0, 0x03, 0xc8, 0xb6, 0x0e, 0x8c, 0x8e, + 0x6e, 0x61, 0x53, 0x04, 0xa3, 0x71, 0x30, 0x3c, 0x5e, 0xb9, 0x0f, 0xc5, 0x81, 0x23, 0x19, 0xb9, + 0x3f, 0x0f, 0x21, 0xef, 0x58, 0x18, 0xbb, 0xce, 0x97, 0x1c, 0x6f, 0x93, 0x80, 0xf2, 0xf2, 0xe7, + 0x47, 0xa9, 0x6c, 0xa2, 0x94, 0x94, 0xbf, 0x97, 0x60, 0x32, 0x74, 0x44, 0x23, 0xa5, 0xfe, 0x5f, + 0x95, 0x78, 0x37, 0xa1, 0xd8, 0x70, 0x99, 0xb6, 0x49, 0xc7, 0x68, 0x9d, 0x50, 0xd7, 0xe9, 0x59, + 0x06, 0xb1, 0x0c, 0xe7, 0x84, 0xa9, 0x38, 0xa1, 0x78, 0xef, 0xf2, 0x2d, 0x40, 0x3c, 0x4d, 0xd8, + 0xac, 0xbf, 0x32, 0x02, 0xc7, 0x37, 0x49, 0x98, 0x12, 0xa7, 0x48, 0xc1, 0x2f, 0xfa, 0xae, 0x03, + 0x9b, 0xb6, 0xa3, 0x99, 0x2d, 0xac, 0x06, 0x6a, 0xdd, 0x82, 0x4b, 0xdc, 0xa4, 0x35, 0xef, 0x02, + 0x4c, 0xdb, 0xcf, 0x8d, 0x1e, 0xef, 0xec, 0xa8, 0x1d, 0x42, 0x9e, 0xf7, 0x79, 0xfd, 0x96, 0x55, + 0x8a, 0x74, 0x80, 0xc9, 0x5f, 0x67, 0x64, 0x1a, 0x31, 0x34, 0x76, 0x40, 0xcf, 0x1a, 0x9e, 0x38, + 0xb7, 0x88, 0x18, 0xbf, 0x04, 0x25, 0x3f, 0x62, 0xf4, 0xd8, 0x0a, 0xc5, 0xf9, 0xb8, 0x35, 0x62, + 0xbc, 0xf0, 0x6c, 0xa9, 0x14, 0xf1, 0x80, 0x71, 0x31, 0xcc, 0x58, 0xdc, 0x80, 0x62, 0x65, 0x42, + 0x00, 0x8f, 0x71, 0x77, 0x62, 0x05, 0x0c, 0x5b, 0x5f, 0x41, 0xd6, 0x10, 0x8d, 0xbb, 0x1e, 0xbf, + 0xbd, 0x8a, 0xdb, 0xcc, 0x0b, 0xc8, 0xac, 0x93, 0x36, 0x8b, 0x4a, 0x7e, 0x04, 0x92, 0xce, 0x16, + 0x81, 0xde, 0x85, 0xa9, 0x83, 0x7e, 0x57, 0x33, 0x55, 0x0b, 0x6b, 0x3a, 0x0b, 0x41, 0x09, 0xb6, + 0x31, 0x93, 0x8c, 0xaa, 0x08, 0xa2, 0xfc, 0xdb, 0x49, 0xd7, 0xb9, 0xb0, 0x82, 0xed, 0x1e, 0x31, + 0x6d, 0x8c, 0x1a, 0x90, 0xe6, 0xea, 0x0a, 0xd9, 0xf1, 0x37, 0xdf, 0x60, 0x71, 0xa2, 0x08, 0x66, + 0xea, 0x42, 0xcc, 0x7c, 0xba, 0x2a, 0xd0, 0xb8, 0x02, 0x05, 0x4e, 0x14, 0x95, 0xcc, 0x02, 0xa4, + 0x79, 0x4f, 0x54, 0x1c, 0x62, 0xe4, 0xd6, 0x44, 0x56, 0xaf, 0x55, 0x6d, 0xb2, 0x11, 0x45, 0xcc, + 0x40, 0x1a, 0xe4, 0x6d, 0x6c, 0x1d, 0x62, 0x4b, 0xed, 0x90, 0x36, 0x6f, 0x7d, 0xe5, 0xeb, 0xbf, + 0x30, 0x6a, 0x7e, 0x70, 0x97, 0x57, 0x6d, 0x32, 0x8c, 0x75, 0xd2, 0xb6, 0x1b, 0xa6, 0x63, 0x9d, + 0x28, 0x60, 0x7b, 0x04, 0x54, 0x86, 0x4c, 0x17, 0xdb, 0xb6, 0xd6, 0xc6, 0x2c, 0x22, 0xe7, 0x14, + 0xf7, 0xb5, 0xd2, 0x86, 0xe2, 0x00, 0x23, 0x2a, 0x41, 0xf2, 0x39, 0x3e, 0x11, 0x27, 0x83, 0x3e, + 0xa2, 0x8f, 0x83, 0x0d, 0x86, 0x7c, 0x7d, 0x3e, 0x56, 0x37, 0xb1, 0xdd, 0xa2, 0x15, 0xb1, 0x98, + 0xb8, 0x2f, 0xc9, 0x9a, 0x7b, 0x16, 0x79, 0xc9, 0xdd, 0xc6, 0xf2, 0x16, 0x4c, 0xb0, 0x4b, 0x3e, + 0xca, 0x43, 0x66, 0x77, 0xf3, 0xb3, 0xcd, 0xad, 0xc7, 0x9b, 0xa5, 0x9f, 0xa0, 0x22, 0xe4, 0x57, + 0x96, 0x56, 0x1e, 0x36, 0xd4, 0x95, 0x87, 0x8d, 0x95, 0xcf, 0x4a, 0x12, 0x02, 0x48, 0xff, 0xe2, + 0x6e, 0x63, 0xb7, 0xb1, 0x5a, 0x4a, 0xa0, 0x49, 0xc8, 0x35, 0x9e, 0x34, 0x56, 0x76, 0x77, 0xd6, + 0x36, 0x3f, 0x2d, 0x25, 0xe9, 0xeb, 0xca, 0xd6, 0xc6, 0xf6, 0x7a, 0x63, 0xa7, 0xb1, 0x5a, 0x4a, + 0xc9, 0xbf, 0x93, 0x80, 0xb2, 0xb0, 0xca, 0x96, 0xdb, 0x29, 0xf7, 0x2e, 0x7e, 0x9f, 0xc1, 0x84, + 0x4d, 0xc5, 0xb2, 0x75, 0x4d, 0xd5, 0xef, 0x8e, 0x7e, 0x9e, 0x98, 0xb6, 0x55, 0xde, 0x8f, 0xe0, + 0x18, 0xc3, 0xa7, 0x3e, 0xf1, 0x26, 0xa7, 0xfe, 0x3d, 0x40, 0xa2, 0xc4, 0xb1, 0x1d, 0x0b, 0x6b, + 0x5d, 0x1e, 0x99, 0x92, 0xbc, 0xd5, 0xc8, 0x47, 0x9a, 0x6c, 0x80, 0x45, 0x27, 0x3e, 0x9b, 0xd6, + 0x28, 0xc1, 0xd9, 0x29, 0x6f, 0x36, 0xb6, 0x2c, 0x7f, 0xb6, 0xbc, 0x00, 0x33, 0x8f, 0x35, 0xc3, + 0xf1, 0x16, 0xe3, 0x06, 0xc2, 0x88, 0xbb, 0xbe, 0xfc, 0x4d, 0x02, 0x66, 0x3f, 0xc5, 0x4e, 0xc8, + 0xeb, 0xc7, 0x89, 0x9b, 0xe7, 0x6b, 0x15, 0x26, 0xb2, 0x63, 0x98, 0x58, 0xe5, 0x26, 0x10, 0x11, + 0xb8, 0xc0, 0x89, 0x4d, 0x46, 0x0b, 0x4f, 0xc2, 0x96, 0xe5, 0x15, 0x24, 0xee, 0x24, 0x6c, 0x59, + 0xa8, 0x0a, 0x6f, 0x89, 0x49, 0xa1, 0xdb, 0xc5, 0x04, 0xef, 0x31, 0xf3, 0x21, 0xbf, 0x72, 0xb2, + 0xe5, 0xbf, 0x49, 0xc0, 0xdc, 0x6e, 0x4f, 0xd7, 0x1c, 0xfc, 0xff, 0xc4, 0x14, 0x8a, 0x87, 0x26, + 0x42, 0x4e, 0xf2, 0x2c, 0x01, 0x4c, 0x60, 0x8a, 0x08, 0x75, 0x5a, 0x36, 0x48, 0x9d, 0x6b, 0x36, + 0x90, 0x7f, 0x53, 0x82, 0x8b, 0x0f, 0x0c, 0x53, 0xdf, 0x30, 0x6c, 0xdb, 0x30, 0xdb, 0xcb, 0x1d, + 0xb2, 0x6f, 0x8f, 0x65, 0xc9, 0x47, 0x50, 0xd8, 0xef, 0x90, 0x7d, 0x61, 0x47, 0xf7, 0x4e, 0x38, + 0xb2, 0x21, 0xf3, 0x94, 0x99, 0x3f, 0xdb, 0x72, 0x1f, 0xca, 0xc3, 0xba, 0x88, 0xec, 0xf0, 0x39, + 0xcc, 0x74, 0x39, 0x5d, 0x7d, 0x13, 0x79, 0xa8, 0xeb, 0x83, 0xbb, 0x62, 0xff, 0x5b, 0x82, 0x8b, + 0xcb, 0x9a, 0xd3, 0x3a, 0xe0, 0x4e, 0x35, 0xbe, 0x0d, 0xbe, 0x84, 0xac, 0xc5, 0xe7, 0xbb, 0xfa, + 0xc4, 0xa7, 0x87, 0x53, 0x04, 0x56, 0xc5, 0x5f, 0xc5, 0x43, 0xac, 0x3c, 0x85, 0x8c, 0xab, 0xcd, + 0x1b, 0xa7, 0x67, 0x04, 0x29, 0x76, 0xc9, 0x49, 0xf0, 0x8f, 0x5e, 0xec, 0x9a, 0xff, 0x9f, 0x12, + 0x94, 0x87, 0xb5, 0x11, 0x66, 0xff, 0x0a, 0x72, 0x96, 0x78, 0x76, 0x3b, 0xd2, 0xcb, 0x67, 0x58, + 0x9b, 0xc8, 0x81, 0xee, 0x83, 0xe2, 0x83, 0x56, 0x8e, 0x20, 0xeb, 0x49, 0x7b, 0xe3, 0xf5, 0xf9, + 0x79, 0x3d, 0x11, 0x97, 0xd7, 0xe5, 0x5f, 0x85, 0x0b, 0x4c, 0x51, 0x5a, 0x94, 0x8c, 0xbf, 0xe7, + 0x4b, 0x90, 0x39, 0xa3, 0x0b, 0xba, 0x7c, 0xf2, 0xaf, 0x27, 0x60, 0x76, 0x50, 0x03, 0x61, 0x88, + 0xa7, 0xc3, 0x66, 0x1f, 0xd1, 0xa5, 0x86, 0xb0, 0x22, 0x8d, 0xfe, 0x5b, 0xd2, 0x79, 0x5a, 0x3d, + 0xc2, 0xab, 0xc6, 0xa9, 0xb0, 0xe4, 0xbf, 0x90, 0x60, 0xea, 0x53, 0xec, 0xd0, 0x2b, 0xe1, 0x58, + 0x7b, 0xf0, 0x10, 0xf2, 0x6f, 0xf0, 0xa1, 0x19, 0x2c, 0xff, 0x1b, 0xf3, 0x25, 0xc8, 0xf5, 0xb4, + 0x36, 0x56, 0x6d, 0xe3, 0x25, 0xcf, 0xec, 0xf4, 0x9e, 0xa2, 0xb5, 0x71, 0xd3, 0x78, 0xc9, 0x3a, + 0x20, 0x6c, 0xd0, 0x21, 0xcf, 0xb1, 0x29, 0x32, 0x39, 0x9b, 0xbe, 0x43, 0x09, 0xf2, 0xb7, 0x12, + 0x14, 0x3d, 0xed, 0x85, 0x49, 0xd7, 0xc3, 0x9f, 0xc4, 0xa4, 0xb1, 0xef, 0xa6, 0xa1, 0xcf, 0x61, + 0x3f, 0x85, 0xa2, 0x89, 0x8f, 0x1d, 0x35, 0xa0, 0x05, 0xbf, 0x0a, 0x4e, 0x52, 0xf2, 0xb6, 0xa7, + 0xc9, 0x47, 0xac, 0x3e, 0x58, 0xd1, 0x7a, 0xda, 0xbe, 0xd1, 0x31, 0xd8, 0xd5, 0x6e, 0x0c, 0x73, + 0xca, 0x7f, 0x9d, 0x04, 0xc4, 0x8b, 0xcd, 0x20, 0x04, 0xd2, 0x00, 0xf1, 0x0c, 0xd4, 0x0a, 0x50, + 0x85, 0xab, 0xc4, 0x7f, 0xbf, 0x64, 0xc9, 0x26, 0xa4, 0xd2, 0x74, 0x6b, 0x90, 0x84, 0xba, 0x30, + 0x1b, 0x68, 0xea, 0x06, 0xc5, 0xf0, 0x3d, 0xbd, 0x37, 0x7a, 0x35, 0x18, 0x12, 0x75, 0x01, 0x47, + 0x91, 0xd1, 0x36, 0xcc, 0xea, 0xb8, 0x67, 0xe1, 0x96, 0xe6, 0x60, 0x5d, 0xd5, 0x7a, 0x86, 0x7a, + 0x88, 0x2d, 0xdb, 0x20, 0xa6, 0xf7, 0x15, 0x20, 0x28, 0x4e, 0xfc, 0x28, 0xa4, 0x89, 0xbb, 0x7b, + 0xd8, 0x52, 0x66, 0x7c, 0xce, 0xa5, 0x9e, 0xb1, 0xc7, 0xf9, 0xd0, 0x32, 0x14, 0x3b, 0xe4, 0x28, + 0x04, 0x95, 0x8a, 0x85, 0x9a, 0xec, 0x90, 0xa3, 0x00, 0xc6, 0x2a, 0x94, 0x0e, 0x8c, 0xf6, 0x41, + 0x08, 0x64, 0x22, 0x16, 0x64, 0x8a, 0xf2, 0xf8, 0x28, 0xb2, 0x0d, 0x53, 0xdc, 0xa7, 0x1f, 0xf4, + 0x4d, 0x56, 0x4f, 0xc8, 0x5a, 0x64, 0x1d, 0x0f, 0x90, 0x6e, 0x3e, 0x5c, 0xaa, 0xdf, 0xbd, 0x57, + 0x92, 0x50, 0x16, 0x52, 0xcd, 0x87, 0x4b, 0xb7, 0x4b, 0x09, 0x94, 0x81, 0xe4, 0xc6, 0xea, 0xdd, + 0x52, 0x92, 0x3e, 0xec, 0x35, 0xb7, 0x4a, 0x29, 0x31, 0xef, 0xce, 0xfd, 0xf7, 0x4b, 0x13, 0xe2, + 0xf9, 0xee, 0xed, 0x7a, 0x29, 0x4d, 0xc1, 0x36, 0x76, 0x95, 0x8d, 0x5d, 0xe5, 0x4e, 0x29, 0x23, + 0x3f, 0x80, 0x2b, 0xbc, 0x94, 0x61, 0xbb, 0xcd, 0x23, 0x7f, 0xc8, 0xe2, 0xef, 0xc2, 0x54, 0x9f, + 0x51, 0x55, 0x6c, 0xd2, 0x0b, 0xa0, 0xce, 0xfc, 0x27, 0xab, 0x4c, 0x72, 0x6a, 0x83, 0x13, 0xe5, + 0xbf, 0x97, 0x60, 0x66, 0x5b, 0xb4, 0x07, 0x42, 0xfc, 0x2d, 0x00, 0xd1, 0x36, 0xf0, 0x8f, 0xd3, + 0x4a, 0xfc, 0xb7, 0xf3, 0x08, 0x28, 0x8f, 0xa8, 0x68, 0x66, 0x1b, 0x2b, 0x01, 0xd8, 0xca, 0x2e, + 0x4c, 0x86, 0x06, 0xd1, 0x75, 0x28, 0x74, 0x0d, 0x53, 0x1d, 0x68, 0x60, 0xe4, 0xbb, 0x86, 0xe9, + 0xce, 0x63, 0x53, 0xb4, 0x63, 0x7f, 0x4a, 0x42, 0x4c, 0xd1, 0x8e, 0xdd, 0x29, 0xf2, 0x36, 0x5c, + 0x12, 0xed, 0x9d, 0xa5, 0x7d, 0x9b, 0x74, 0xfa, 0x0e, 0xde, 0xd6, 0x9c, 0x83, 0xa6, 0x63, 0x69, + 0x0e, 0x6e, 0x9f, 0xc8, 0xb7, 0x23, 0xb7, 0x67, 0x0a, 0x60, 0x75, 0xad, 0xb9, 0xb4, 0xbe, 0xbe, + 0xf5, 0xb8, 0xb1, 0x5a, 0x92, 0xe8, 0xa0, 0xfb, 0x92, 0x90, 0x6f, 0x02, 0xac, 0x90, 0x6e, 0xcf, + 0xc2, 0xb6, 0x4d, 0x2c, 0xf9, 0xaa, 0x0b, 0x50, 0x80, 0xec, 0xda, 0x6a, 0x63, 0x73, 0x67, 0x6d, + 0xe7, 0xf3, 0xd2, 0x4f, 0xe8, 0xa6, 0x7e, 0xd1, 0xdc, 0x59, 0x2d, 0x49, 0xf2, 0x1f, 0x4e, 0xc0, + 0xf4, 0xd0, 0x31, 0x44, 0x4f, 0xa1, 0xc8, 0xe3, 0xa6, 0xfa, 0x4c, 0x78, 0x0a, 0xb3, 0xeb, 0x28, + 0x57, 0xaf, 0xb0, 0x83, 0x89, 0xab, 0xd7, 0x94, 0x1e, 0xa2, 0xa2, 0x6f, 0x25, 0xb8, 0x26, 0xaa, + 0x62, 0x1e, 0x3e, 0x84, 0x03, 0x44, 0x1c, 0xef, 0x8f, 0x47, 0x2c, 0x94, 0x4f, 0xf1, 0x2e, 0xe5, + 0x8a, 0xf6, 0x5a, 0xe7, 0xeb, 0xc3, 0x25, 0x51, 0x42, 0x8b, 0x2d, 0x09, 0xeb, 0xc0, 0xcf, 0xfc, + 0xdd, 0x33, 0x79, 0x93, 0x32, 0xc7, 0x90, 0x23, 0x7d, 0x76, 0x11, 0x2a, 0xd4, 0x35, 0xf6, 0x69, + 0x56, 0x56, 0x1d, 0xe2, 0x68, 0x1d, 0x35, 0xf0, 0x51, 0x34, 0xc5, 0x3e, 0x8a, 0xce, 0x76, 0xb5, + 0x63, 0x96, 0xb6, 0x77, 0xe8, 0x78, 0xd3, 0xfd, 0x42, 0x4a, 0x8d, 0xf7, 0xb6, 0xe8, 0xf6, 0xab, + 0x9a, 0xf0, 0x1a, 0xf6, 0x8b, 0x1a, 0x7a, 0xa9, 0x64, 0x7e, 0xc3, 0x42, 0xc3, 0xd4, 0x08, 0xc5, + 0xd8, 0x6b, 0x7c, 0x4f, 0xec, 0xdc, 0x25, 0xfb, 0xf4, 0x29, 0x48, 0x87, 0x19, 0xbb, 0xdf, 0xeb, + 0x11, 0x8b, 0x86, 0xca, 0x96, 0xe7, 0x75, 0xe5, 0x34, 0xf3, 0x95, 0xdb, 0xa3, 0xfc, 0x28, 0x49, + 0xb0, 0x08, 0x69, 0x6f, 0x79, 0x70, 0x01, 0x1f, 0xfe, 0xc7, 0x04, 0x5c, 0x88, 0x0c, 0xe1, 0xd1, + 0x6e, 0x2a, 0x9d, 0x9f, 0x9b, 0x5e, 0x87, 0x02, 0x65, 0xf2, 0xe2, 0x12, 0xef, 0x16, 0xe5, 0x29, + 0x4d, 0x44, 0x25, 0xf4, 0x0a, 0xae, 0x06, 0xba, 0x7e, 0xe7, 0xef, 0x43, 0xfe, 0x07, 0xcd, 0x53, + 0xfc, 0x68, 0xce, 0xdf, 0x81, 0xe1, 0x8e, 0x2f, 0xbd, 0x24, 0x5f, 0xf4, 0x26, 0x84, 0x3b, 0xbb, + 0xf2, 0x06, 0xe4, 0x77, 0x08, 0xe9, 0xac, 0x62, 0x47, 0x33, 0x3a, 0xec, 0x43, 0x8f, 0x43, 0x48, + 0x27, 0x58, 0x02, 0x64, 0x29, 0x81, 0x55, 0x53, 0xd7, 0xa1, 0xc0, 0x06, 0xdd, 0xdc, 0xc3, 0x4b, + 0x8c, 0x3c, 0xa5, 0xb9, 0xc9, 0xe5, 0x5f, 0x25, 0x28, 0x8a, 0x92, 0xc2, 0x6b, 0xdc, 0x6c, 0x09, + 0x36, 0x9d, 0xcb, 0x10, 0x85, 0x41, 0xfc, 0xef, 0x75, 0x02, 0x7a, 0x71, 0x21, 0x01, 0x25, 0x45, + 0xdc, 0x30, 0x74, 0xa1, 0x44, 0x96, 0x13, 0xd6, 0x74, 0xf4, 0x1e, 0x20, 0x26, 0xcd, 0x30, 0x0f, + 0x49, 0x4b, 0x73, 0x67, 0x89, 0x5e, 0x0c, 0x1d, 0x59, 0xf3, 0x06, 0xd6, 0x74, 0x6a, 0xba, 0x16, + 0xb1, 0x2c, 0xdc, 0x61, 0x89, 0xde, 0xe7, 0xb1, 0x29, 0x13, 0x2f, 0xe4, 0x2e, 0xfa, 0x13, 0x7c, + 0x56, 0x7b, 0x4d, 0xaf, 0xff, 0x79, 0x02, 0x72, 0x9e, 0x4b, 0xa2, 0xef, 0x24, 0xc8, 0x88, 0xde, + 0x15, 0xaa, 0x8d, 0xde, 0xfb, 0x63, 0xa6, 0xaa, 0x5c, 0x71, 0x6b, 0xdf, 0xc0, 0x8f, 0x45, 0xab, + 0x5e, 0x0b, 0x4c, 0xbe, 0xfd, 0x6b, 0xff, 0xf0, 0x2f, 0xbf, 0x9b, 0xb8, 0x21, 0xff, 0xb4, 0x76, + 0x58, 0xaf, 0x7d, 0x1d, 0xaa, 0xd3, 0x3e, 0x5a, 0x58, 0x78, 0x55, 0xe3, 0x8b, 0xb7, 0x17, 0xb9, + 0x08, 0xbc, 0x28, 0x2d, 0xdc, 0x92, 0xd0, 0x1f, 0x48, 0x30, 0x19, 0xea, 0x1c, 0xa1, 0x78, 0xef, + 0x8b, 0xea, 0x34, 0x8d, 0xa7, 0x1c, 0xd3, 0xc9, 0xff, 0x99, 0x6b, 0x6d, 0x61, 0xe1, 0xd5, 0xe2, + 0x51, 0x10, 0x95, 0x29, 0x57, 0xff, 0xa7, 0x24, 0xe4, 0x03, 0x21, 0x1b, 0xfd, 0x33, 0x2f, 0x91, + 0x43, 0x1f, 0x93, 0xe3, 0xbf, 0xb0, 0x45, 0xf7, 0xba, 0x2a, 0xe3, 0xb5, 0x55, 0xe4, 0x2f, 0xd9, + 0x02, 0xf6, 0xd0, 0xce, 0x6b, 0xad, 0x2b, 0xda, 0x25, 0xb5, 0xaf, 0x43, 0x6d, 0xa1, 0xea, 0x81, + 0x66, 0x1f, 0xbc, 0x1a, 0x24, 0xfa, 0x51, 0xfc, 0x15, 0xfa, 0x0f, 0x09, 0xd0, 0x70, 0x2f, 0x0a, + 0x2d, 0xc6, 0xea, 0x78, 0x6a, 0x03, 0x6b, 0xdc, 0xf5, 0x3d, 0x67, 0xeb, 0xc3, 0x95, 0x1f, 0x65, + 0x7d, 0x8b, 0xe1, 0xc6, 0x56, 0xfd, 0xfb, 0x34, 0xcc, 0xad, 0xf0, 0x8f, 0x85, 0x4b, 0xba, 0x4e, + 0x83, 0x37, 0x0d, 0x91, 0x4d, 0x87, 0x58, 0x5a, 0x1b, 0xa3, 0xbf, 0x94, 0xa0, 0x34, 0xd8, 0xc0, + 0x41, 0xf7, 0x47, 0xf8, 0x3d, 0x61, 0x64, 0xff, 0xa9, 0xf2, 0xe1, 0x19, 0x38, 0xf9, 0xfd, 0x4b, + 0xbe, 0xc3, 0x8c, 0x72, 0x53, 0x9e, 0x3f, 0xc5, 0x28, 0xfb, 0x74, 0xf6, 0xe2, 0x33, 0x9f, 0x7d, + 0x51, 0x5a, 0x60, 0xea, 0x0f, 0xb6, 0x2e, 0x46, 0x50, 0xff, 0x94, 0x4e, 0xce, 0x08, 0xea, 0x9f, + 0xd6, 0x27, 0x19, 0x51, 0xfd, 0x7d, 0x9f, 0x9d, 0xaa, 0xff, 0x67, 0x12, 0x4c, 0x85, 0x5b, 0x00, + 0xe8, 0xde, 0xd8, 0x3d, 0x03, 0xae, 0xfa, 0x07, 0x67, 0xec, 0x35, 0xc4, 0x86, 0xb2, 0x80, 0xe2, + 0x94, 0x99, 0xaa, 0xfd, 0x83, 0x04, 0x19, 0x71, 0x7d, 0x1e, 0x21, 0xb2, 0x86, 0xdb, 0x04, 0x95, + 0x5b, 0xa3, 0x33, 0x08, 0x0d, 0x9f, 0x30, 0x0d, 0x15, 0xb4, 0xfd, 0x3a, 0x0d, 0x6b, 0x5f, 0x07, + 0xfa, 0x0a, 0xee, 0x21, 0x09, 0x92, 0x82, 0x47, 0xa4, 0xcd, 0x25, 0xdc, 0x92, 0xea, 0x7f, 0x25, + 0x41, 0x21, 0x94, 0xba, 0xff, 0x94, 0xc7, 0xbd, 0x10, 0x6d, 0xa4, 0xb8, 0x17, 0x71, 0x87, 0xaf, + 0xc4, 0xf7, 0x7c, 0x87, 0x2f, 0xef, 0xf2, 0x0d, 0xb6, 0xdc, 0x77, 0xd1, 0x3b, 0xa7, 0x2c, 0x37, + 0x58, 0xc5, 0x2c, 0x5b, 0x10, 0xf7, 0x5f, 0x13, 0xcb, 0x33, 0x0a, 0x23, 0xfa, 0x1f, 0x33, 0x2d, + 0xe2, 0x90, 0x6d, 0xe9, 0x8b, 0x22, 0x9f, 0xec, 0xcd, 0xfd, 0xa3, 0x44, 0x52, 0x69, 0x3c, 0xf9, + 0x93, 0xc4, 0xd5, 0x65, 0x06, 0xb8, 0xcc, 0x00, 0x39, 0xaf, 0x7f, 0x57, 0xaf, 0xee, 0xd5, 0xf7, + 0xd3, 0xec, 0x57, 0x48, 0x77, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0x89, 0x45, 0xb0, 0xed, 0xe8, + 0x31, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -4077,6 +4171,11 @@ type ExecutionClient interface { // where, for each requested blob not present in the CAS, there is a // `Violation` with a `type` of `MISSING` and a `subject` of // `"blobs/{hash}/{size}"` indicating the digest of the missing blob. + // + // The server does not need to guarantee that a call to this method leads to + // at most one execution of the action. The server MAY execute the action + // multiple times, potentially in parallel. These redundant executions MAY + // continue to run, even if the operation is completed. Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (Execution_ExecuteClient, error) // Wait for an execution operation to complete. When the client initially // makes the request, the server immediately responds with the current status @@ -4224,6 +4323,11 @@ type ExecutionServer interface { // where, for each requested blob not present in the CAS, there is a // `Violation` with a `type` of `MISSING` and a `subject` of // `"blobs/{hash}/{size}"` indicating the digest of the missing blob. + // + // The server does not need to guarantee that a call to this method leads to + // at most one execution of the action. The server MAY execute the action + // multiple times, potentially in parallel. These redundant executions MAY + // continue to run, even if the operation is completed. Execute(*ExecuteRequest, Execution_ExecuteServer) error // Wait for an execution operation to complete. When the client initially // makes the request, the server immediately responds with the current status From 712e06e588c4addd53d09ab658596972eb973ae6 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Mon, 16 Nov 2020 08:03:28 +0100 Subject: [PATCH 02/35] [disk] store CAS blobs in compressed form This is the first step towards supporting the compressed-blobs bytestream API. CAS blobs are stored on disk with a header, followed by a sequence of chunks compressed with zstandard. This requires some test refactoring, we can no longer reliably predict cache evictions at the disk cache API boundary (because the size of blobs will change after compression). --- cache/cache.go | 27 +- cache/disk/BUILD.bazel | 2 + cache/disk/casblob/BUILD.bazel | 9 + cache/disk/casblob/casblob.go | 521 ++++++++++++++++++++++++++++++ cache/disk/disk.go | 284 +++++++++++++--- cache/disk/disk_test.go | 151 +++++++-- cache/disk/lru.go | 12 +- cache/disk/lru_test.go | 12 +- cache/httpproxy/BUILD.bazel | 1 + cache/httpproxy/httpproxy.go | 51 ++- cache/httpproxy/httpproxy_test.go | 47 ++- cache/s3proxy/s3proxy.go | 100 ++++-- cache/s3proxy/s3proxy_test.go | 44 ++- go.mod | 1 + main.go | 10 + server/grpc_test.go | 7 +- server/http_test.go | 27 +- utils/testutils.go | 13 - 18 files changed, 1156 insertions(+), 163 deletions(-) create mode 100644 cache/disk/casblob/BUILD.bazel create mode 100644 cache/disk/casblob/casblob.go diff --git a/cache/cache.go b/cache/cache.go index 49c7cc93c..9360e5c62 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -4,7 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "io" - "path/filepath" + "path" ) // EntryKind describes the kind of cache entry @@ -33,6 +33,16 @@ func (e EntryKind) String() string { return "raw" } +func (e EntryKind) DirName() string { + if e == AC { + return "ac" + } + if e == CAS { + return "cas.v2" + } + return "raw" +} + // Logger is designed to be satisfied by log.Logger. type Logger interface { Printf(format string, v ...interface{}) @@ -88,6 +98,17 @@ func TransformActionCacheKey(key, instance string, logger Logger) string { } // Key returns the proper cache key for an entry kind and hash. -func Key(kind EntryKind, hash string) string { - return filepath.Join(kind.String(), hash[:2], hash) +func FileLocation(kind EntryKind, hash string) string { + var kindWithVersion string + if kind == CAS { + kindWithVersion = "cas.v2" + } else { + kindWithVersion = kind.String() + } + + return path.Join(kindWithVersion, hash[:2], hash) +} + +func LookupKey(kind EntryKind, hash string) string { + return kind.String() + "/" + hash } diff --git a/cache/disk/BUILD.bazel b/cache/disk/BUILD.bazel index cb2e706e7..32e6465a5 100644 --- a/cache/disk/BUILD.bazel +++ b/cache/disk/BUILD.bazel @@ -10,6 +10,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//cache:go_default_library", + "//cache/disk/casblob:go_default_library", "//genproto/build/bazel/remote/execution/v2:go_default_library", "//utils/tempfile:go_default_library", "@com_github_djherbis_atime//:go_default_library", @@ -28,6 +29,7 @@ go_test( embed = [":go_default_library"], deps = [ "//cache:go_default_library", + "//cache/disk/casblob:go_default_library", "//cache/httpproxy:go_default_library", "//genproto/build/bazel/remote/execution/v2:go_default_library", "//utils:go_default_library", diff --git a/cache/disk/casblob/BUILD.bazel b/cache/disk/casblob/BUILD.bazel new file mode 100644 index 000000000..567123310 --- /dev/null +++ b/cache/disk/casblob/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["casblob.go"], + importpath = "github.com/buchgr/bazel-remote/cache/disk/casblob", + visibility = ["//visibility:public"], + deps = ["@com_github_klauspost_compress//zstd:go_default_library"], +) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go new file mode 100644 index 000000000..c301a00b7 --- /dev/null +++ b/cache/disk/casblob/casblob.go @@ -0,0 +1,521 @@ +package casblob + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + + "github.com/klauspost/compress/zstd" +) + +type CompressionType uint8 + +const ( + Identity CompressionType = 0 + Zstandard CompressionType = 1 +) + +var zstdLevelOpt = zstd.WithEncoderLevel(zstd.SpeedFastest) +var encoder, _ = zstd.NewWriter(nil, zstdLevelOpt) +var decoder, _ = zstd.NewReader(nil) + +const defaultChunkSize = 1024 * 1024 * 1 // 1M + +const defaultChunkSize = 1024 * 1024 * 1 // 1M + +// CAS blobs are stored with a header followed by chunks of data in either +// compressed or uncompressed format. +type header struct { + // The following data is stored in little-endian format on disk. + uncompressedSize int64 // 8 bytes + compression CompressionType // uint8, 1 byte + chunkSize uint32 // 4 bytes + + // Offsets in the file on disk, of each chunk, with an additional + // entry for the end of the file. + // + // Stored as an int64 number of chunks, followed by their int64 offsets, + // and a final value for the size of the file (header + data). + // 8 bytes + (n+1)*8 bytes. + chunkOffsets []int64 +} + +const chunkTableOffset = 8 + 1 + 4 + 8 + +// Returns the size of the header itself. +func (h *header) size() int64 { + return chunkTableOffset + (int64(len(h.chunkOffsets)) * 8) +} + +// Provides an io.ReadCloser that returns uncompressed data from a cas blob. +type readCloserWrapper struct { + *header + + rdr io.Reader // Read from this, not from decoder or file. + + decoder *zstd.Decoder // Might be nil. + file *os.File +} + +// Read the header and leave f at the start of the data. +func readHeader(f *os.File) (*header, error) { + var err error + var h header + + fileInfo, err := f.Stat() + if err != nil { + return nil, err + } + foundFileSize := fileInfo.Size() + + err = binary.Read(f, binary.LittleEndian, &h.uncompressedSize) + if err != nil { + return nil, err + } + + err = binary.Read(f, binary.LittleEndian, &h.compression) + if err != nil { + return nil, err + } + + err = binary.Read(f, binary.LittleEndian, &h.chunkSize) + if err != nil { + return nil, err + } + + var numOffsets int64 + err = binary.Read(f, binary.LittleEndian, &numOffsets) + if err != nil { + return nil, err + } + + if numOffsets < 2 { + // chunkOffsets has an extra entry to specify the compressed file size. + return nil, fmt.Errorf("internal error: need at least one chunk, found %d", numOffsets-1) + } + + prevOffset := int64(-1) + h.chunkOffsets = make([]int64, numOffsets, numOffsets) + for i := 0; int64(i) < numOffsets; i++ { + err = binary.Read(f, binary.LittleEndian, &h.chunkOffsets[i]) + if err != nil { + return nil, err + } + + if h.chunkOffsets[i] <= prevOffset { + return nil, + fmt.Errorf("offset table values should increase: %d -> %d", + h.chunkOffsets[i], prevOffset) + } + prevOffset = h.chunkOffsets[i] + } + + if prevOffset != foundFileSize { + return nil, + fmt.Errorf("final offset in chunk table %d should be file size %d", + prevOffset, foundFileSize) + } + + return &h, nil +} + +func GetLogicalSize(filename string) (int64, error) { + f, err := os.Open(filename) + if err != nil { + return -1, err + } + defer f.Close() + + hdr, err := readHeader(f) + if err != nil { + return -1, fmt.Errorf("Failed to read %s: %w", filename, err) + } + + return hdr.uncompressedSize, nil +} + +// Closes f if there is an error. Otherwise the caller must Close the returned +// io.ReadCloser. +func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) { + h, err := readHeader(f) + if err != nil { + f.Close() + return nil, err + } + + if expectedSize != -1 && h.uncompressedSize != expectedSize { + return nil, fmt.Errorf("expected a blob of size %d, found %d", + expectedSize, h.uncompressedSize) + } + + if h.compression == Identity { + // Simple case. Assumes that we only have one chunk if the data is + // uncompressed (which makes sense). + + if offset > 0 { + _, err = f.Seek(offset, io.SeekCurrent) + if err != nil { + f.Close() + return nil, err + } + } + + return f, nil + } + + if h.compression != Zstandard { + f.Close() + return nil, + fmt.Errorf("internal error: unsupported compression type %d", + h.compression) + } + + // Find the first relevant chunk. + chunkNum := int64(offset / int64(h.chunkSize)) + remainder := offset % int64(h.chunkSize) + + if chunkNum > 0 { + f.Seek(h.chunkOffsets[chunkNum], io.SeekStart) + } + if remainder == 0 { + z, err := zstd.NewReader(f) // TODO: use a pool, or a chunk decoder. + if err != nil { + f.Close() + return nil, err + } + + return &readCloserWrapper{ + header: h, + rdr: z, + decoder: z, + file: f, + }, nil + } + + compressedFirstChunk := make([]byte, h.chunkOffsets[chunkNum+1]-h.chunkOffsets[chunkNum]) + _, err = io.ReadFull(f, compressedFirstChunk) + if err != nil { + f.Close() + return nil, err + } + + uncompressedFirstChunk, err := decoder.DecodeAll(compressedFirstChunk, nil) + if err != nil { + f.Close() + return nil, err + } + + if chunkNum == int64(len(h.chunkOffsets)-2) { + // Last chunk in the file. + r := bytes.NewReader(uncompressedFirstChunk[remainder:]) + f.Close() + return ioutil.NopCloser(r), nil + } + + z, err := zstd.NewReader(f) // TODO: use a pool. + if err != nil { + f.Close() + return nil, err + } + + br := bytes.NewReader(uncompressedFirstChunk[remainder:]) + + return &readCloserWrapper{ + header: h, + rdr: io.MultiReader(br, z), + decoder: z, + file: f, + }, nil +} + +// Closes f if there is an error. Otherwise the caller must Close the returned +// io.ReadCloser. +func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) { + + h, err := readHeader(f) + if err != nil { + f.Close() + return nil, err + } + + if expectedSize != -1 && h.uncompressedSize != expectedSize { + return nil, fmt.Errorf("expected a blob of size %d, found %d", + expectedSize, h.uncompressedSize) + } + + if h.compression == Identity { + // Simple case. Assumes that we only have one chunk if the data is + // uncompressed (which makes sense). + + if offset > 0 { + _, err = f.Seek(offset, io.SeekCurrent) + if err != nil { + f.Close() + return nil, err + } + } + + pr, pw := io.Pipe() + + // TODO: use a pool + enc, err := zstd.NewWriter(pw, zstd.WithEncoderLevel(zstd.SpeedFastest)) + if err != nil { + f.Close() + return nil, err + } + + go func() { + // Read from the file, write to enc. + + // TODO: consider implementing something with a timeout? + _, err := enc.ReadFrom(f) + if err != nil { + // We can't do anything here except log an error. + log.Println("Error while compressing file:", err) + } + + enc.Close() + f.Close() + }() + + return pr, nil + } + + if h.compression != Zstandard { + f.Close() + return nil, fmt.Errorf("unsupported compression type: %d", + h.compression) + } + + // Find the first relevant chunk. + chunkNum := int64(offset / int64(h.chunkSize)) + remainder := offset % int64(h.chunkSize) + + if chunkNum > 0 { + f.Seek(h.chunkOffsets[chunkNum], io.SeekStart) + } + + if remainder == 0 { + // Simple case- just stream the file from here. + return f, nil + } + + compressedFirstChunk := make([]byte, h.chunkOffsets[chunkNum+1]-h.chunkOffsets[chunkNum]) + _, err = io.ReadFull(f, compressedFirstChunk) + if err != nil { + f.Close() + return nil, err + } + + uncompressedFirstChunk, err := decoder.DecodeAll(compressedFirstChunk, nil) + if err != nil { + f.Close() + return nil, err + } + + chunkToRecompress := uncompressedFirstChunk[remainder:] + recompressedChunk := encoder.EncodeAll(chunkToRecompress, nil) + + br := bytes.NewReader(recompressedChunk) + if chunkNum == int64(len(h.chunkOffsets)-2) { + f.Close() + return ioutil.NopCloser(br), nil + } + + return &readCloserWrapper{ + header: h, + rdr: io.MultiReader(br, f), + file: f, + }, nil +} + +func (h *header) write(f *os.File) error { + var err error + + err = binary.Write(f, binary.LittleEndian, h.uncompressedSize) + if err != nil { + return err + } + + err = binary.Write(f, binary.LittleEndian, h.compression) + if err != nil { + return err + } + + err = binary.Write(f, binary.LittleEndian, h.chunkSize) + if err != nil { + return err + } + + err = binary.Write(f, binary.LittleEndian, int64(len(h.chunkOffsets))) + if err != nil { + return err + } + + return h.writeChunkTable(f) +} + +func (h *header) writeChunkTable(f *os.File) error { + var err error + + for _, o := range h.chunkOffsets { + err = binary.Write(f, binary.LittleEndian, o) + if err != nil { + return err + } + } + + return nil +} + +func (b *readCloserWrapper) Read(p []byte) (int, error) { + return b.rdr.Read(p) +} + +func (b *readCloserWrapper) Close() error { + if b.decoder != nil { + b.decoder.Close() + b.decoder = nil + } + + if b.file == nil { + return nil + } + + f := b.file + b.file = nil + + return f.Close() +} + +// Read from r and write to f, using CompressionType t. +// Return the size on disk or an error if something went wrong. +func WriteAndClose(r io.Reader, f *os.File, t CompressionType, hash string, size int64) (int64, error) { + var err error + defer f.Close() + + if size <= 0 { + return -1, fmt.Errorf("invalid file size: %d", size) + } + + chunkSize := uint32(defaultChunkSize) + + numChunks := int64(1) + remainder := int64(0) + if t == Zstandard { + numChunks = size / int64(chunkSize) + remainder = size % int64(chunkSize) + if remainder > 0 { + numChunks++ + } + } + + numOffsets := numChunks + 1 + h := header{ + uncompressedSize: size, + compression: t, + chunkSize: chunkSize, + chunkOffsets: make([]int64, numOffsets, numOffsets), + } + + h.chunkOffsets[0] = chunkTableOffset + + err = h.write(f) + if err != nil { + return -1, err + } + + fileOffset := h.size() + + var n int64 + + if t == Identity { + hasher := sha256.New() + + n, err = io.Copy(io.MultiWriter(f, hasher), r) + if err != nil { + return -1, err + } + if n != size { + return -1, fmt.Errorf("expected to copy %d bytes, actually copied %d bytes", + size, n) + } + + actualHash := hex.EncodeToString(hasher.Sum(nil)) + if actualHash != hash { + return -1, + fmt.Errorf("checksums don't match. Expected %s, found %s", + hash, actualHash) + } + + return n + fileOffset, f.Close() + } + + // Compress the data in chunks... + + nextChunk := 0 // Index in h.chunkOffsets. + remainingRawData := size + + uncompressedChunk := make([]byte, chunkSize, chunkSize) + + hasher := sha256.New() + + for nextChunk < len(h.chunkOffsets)-1 { + h.chunkOffsets[nextChunk] = fileOffset + nextChunk++ + + chunkEnd := int64(chunkSize) + if remainingRawData <= int64(chunkSize) { + chunkEnd = remainingRawData + } + remainingRawData -= chunkEnd + + _, err = io.ReadFull(r, uncompressedChunk[0:chunkEnd]) + if err != nil { + return -1, err + } + + compressedChunk := encoder.EncodeAll(uncompressedChunk[0:chunkEnd], nil) + + hasher.Write(uncompressedChunk[0:chunkEnd]) + + written, err := f.Write(compressedChunk) + if err != nil { + return -1, err + } + + fileOffset += int64(written) + } + h.chunkOffsets[nextChunk] = fileOffset + + actualHash := hex.EncodeToString(hasher.Sum(nil)) + if actualHash != hash { + return -1, fmt.Errorf("checksums don't match. Expected %s, found %s", + hash, actualHash) + } + + // We know all the chunk offsets now, go back and fill those in. + _, err = f.Seek(chunkTableOffset, io.SeekStart) + if err != nil { + return -1, err + } + + err = h.writeChunkTable(f) + if err != nil { + return -1, err + } + + err = f.Sync() + if err != nil { + return -1, err + } + + return fileOffset, f.Close() +} diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 33390d659..42e60a46f 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -3,20 +3,22 @@ package disk import ( "bytes" "crypto/sha256" - "encoding/hex" "fmt" "io" "io/ioutil" "log" "net/http" "os" + "path" "path/filepath" "regexp" "sort" + "strings" "sync" "syscall" "github.com/buchgr/bazel-remote/cache" + "github.com/buchgr/bazel-remote/cache/disk/casblob" "github.com/buchgr/bazel-remote/utils/tempfile" "github.com/djherbis/atime" @@ -42,7 +44,8 @@ var tfc = tempfile.NewCreator() // lruItem is the type of the values stored in SizedLRU to keep track of items. type lruItem struct { - size int64 + size int64 + sizeOnDisk int64 } // Cache is a filesystem-based LRU cache, with an optional backend proxy. @@ -71,15 +74,15 @@ func New(dir string, maxSizeBytes int64, proxy cache.Proxy) (*Cache, error) { for _, c1 := range hexLetters { for _, c2 := range hexLetters { subDir := string(c1) + string(c2) - err := os.MkdirAll(filepath.Join(dir, cache.CAS.String(), subDir), os.ModePerm) + err := os.MkdirAll(filepath.Join(dir, cache.CAS.DirName(), subDir), os.ModePerm) if err != nil { return nil, err } - err = os.MkdirAll(filepath.Join(dir, cache.AC.String(), subDir), os.ModePerm) + err = os.MkdirAll(filepath.Join(dir, cache.AC.DirName(), subDir), os.ModePerm) if err != nil { return nil, err } - err = os.MkdirAll(filepath.Join(dir, cache.RAW.String(), subDir), os.ModePerm) + err = os.MkdirAll(filepath.Join(dir, cache.RAW.DirName(), subDir), os.ModePerm) if err != nil { return nil, err } @@ -90,7 +93,19 @@ func New(dir string, maxSizeBytes int64, proxy cache.Proxy) (*Cache, error) { // This function is only called while the lock is held // by the current goroutine. onEvict := func(key Key, value lruItem) { - f := filepath.Join(dir, key.(string)) + ks := key.(string) + hash := ks[len(ks)-sha256.Size*2:] + var kind cache.EntryKind = cache.AC + if strings.HasPrefix(ks, "cas") { + kind = cache.CAS + } else if strings.HasPrefix(ks, "ac") { + kind = cache.AC + } else if strings.HasPrefix(ks, "raw") { + kind = cache.RAW + } + + f := filepath.Join(dir, cache.FileLocation(kind, hash)) + err := os.Remove(f) if err != nil { log.Printf("ERROR: failed to remove evicted cache file: %s", f) @@ -116,11 +131,11 @@ func New(dir string, maxSizeBytes int64, proxy cache.Proxy) (*Cache, error) { } func (c *Cache) migrateDirectories() error { - err := migrateDirectory(filepath.Join(c.dir, cache.AC.String())) + err := migrateDirectory(c.dir, cache.AC) if err != nil { return err } - err = migrateDirectory(filepath.Join(c.dir, cache.CAS.String())) + err = migrateDirectory(c.dir, cache.CAS) if err != nil { return err } @@ -128,10 +143,17 @@ func (c *Cache) migrateDirectories() error { return nil } -func migrateDirectory(dir string) error { - log.Printf("Migrating files (if any) to new directory structure: %s\n", dir) +func migrateDirectory(baseDir string, kind cache.EntryKind) error { + sourceDir := path.Join(baseDir, kind.String()) + + _, err := os.Stat(sourceDir) + if os.IsNotExist(err) { + return nil + } + + log.Println("Migrating files (if any) to new directory structure:", sourceDir) - listing, err := ioutil.ReadDir(dir) + listing, err := ioutil.ReadDir(sourceDir) if err != nil { return err } @@ -144,15 +166,26 @@ func migrateDirectory(dir string) error { // hex character pairs. v1DirRegex := regexp.MustCompile("^[a-f0-9]{2}$") + targetDir := path.Join(baseDir, kind.DirName()) + for _, item := range listing { oldName := item.Name() - oldNamePath := filepath.Join(dir, oldName) + oldNamePath := filepath.Join(sourceDir, oldName) if item.IsDir() { if !v1DirRegex.MatchString(oldName) { // Warn about non-v1 subdirectories. log.Println("Warning: unexpected directory", oldNamePath) } + + destDir := filepath.Join(targetDir, oldName[:2]) + err = migrateV1Subdir(oldNamePath, destDir, kind) + if err != nil { + log.Printf("Warning: failed to read subdir %q: %s", + oldNamePath, err) + continue + } + continue } @@ -166,8 +199,17 @@ func migrateDirectory(dir string) error { continue } - newName := filepath.Join(dir, oldName[:2], oldName) - err = os.Rename(filepath.Join(dir, oldName), newName) + // oldName can now be assumed to be a hash. + hash := oldName + + src := filepath.Join(sourceDir, oldName) + dest := filepath.Join(targetDir, oldName[:2], oldName) + if kind == cache.CAS { + err = migrateCASFile(src, dest, hash) + } else { + // TODO: make this work across filesystems? + err = os.Rename(src, dest) + } if err != nil { return err } @@ -176,6 +218,88 @@ func migrateDirectory(dir string) error { return nil } +func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error { + + listing, err := ioutil.ReadDir(oldDir) + if err != nil { + return err + } + + hashKeyRegex := regexp.MustCompile("^[a-f0-9]{64}$") + + if kind == cache.CAS { + for _, item := range listing { + + oldPath := path.Join(oldDir, item.Name()) + + if !hashKeyRegex.MatchString(item.Name()) { + return fmt.Errorf("Unexpected file: %s", oldPath) + } + + destPath := path.Join(destDir, item.Name()) + + err = migrateCASFile(oldPath, destPath, item.Name()) + if err != nil { + return fmt.Errorf("Failed to migrate CAS blob %s: %w", + oldPath, err) + } + } + + return os.Remove(oldDir) + } + + for _, item := range listing { + oldPath := path.Join(oldDir, item.Name()) + + if !hashKeyRegex.MatchString(item.Name()) { + return fmt.Errorf("Unexpected file: %s", oldPath) + } + + destPath := path.Join(destDir, item.Name()) + + // TODO: support cross-filesystem migration. + err = os.Rename(oldPath, destPath) + if err != nil { + return fmt.Errorf("Failed to migrate blob %s: %w", oldPath, err) + } + } + + return nil +} + +func migrateCASFile(src string, dest string, hash string) error { + + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + fi, err := srcFile.Stat() + if err != nil { + return err + } + + origSize := fi.Size() + + destFile, err := os.Create(dest) + if err != nil { + return err + } + _, err = casblob.WriteAndClose(srcFile, destFile, + casblob.Zstandard, hash, origSize) + if err != nil { + return err + } + + err = os.Remove(src) + if err != nil { + return fmt.Errorf("Failed to remove %s: %w", src, err) + } + + return err +} + // loadExistingFiles lists all files in the cache directory, and adds them to the // LRU index so that they can be served. Files are sorted by access time first, // so that the eviction behavior is preserved across server restarts. @@ -208,7 +332,32 @@ func (c *Cache) loadExistingFiles() error { log.Println("Building LRU index.") for _, f := range files { relPath := f.name[len(c.dir)+1:] - ok := c.lru.Add(relPath, lruItem{size: f.info.Size()}) + var lookupKey string + + sizeOnDisk := f.info.Size() + size := sizeOnDisk + + fields := strings.Split(relPath, "/") + hash := fields[len(fields)-1] + + if strings.HasPrefix(relPath, "cas.v2/") { + size, err = casblob.GetLogicalSize(f.name) + if err != nil { + return err + } + lookupKey = "cas/" + hash + } else if strings.HasPrefix(relPath, "ac/") { + lookupKey = "ac/" + hash + } else if strings.HasPrefix(relPath, "raw/") { + lookupKey = "raw/" + hash + } else { + return fmt.Errorf("Unrecognised file in cache dir: %q", relPath) + } + + ok := c.lru.Add(lookupKey, lruItem{ + size: size, + sizeOnDisk: sizeOnDisk, + }) if !ok { err = os.Remove(filepath.Join(c.dir, relPath)) if err != nil { @@ -218,6 +367,7 @@ func (c *Cache) loadExistingFiles() error { } log.Println("Finished loading disk cache files.") + return nil } @@ -241,7 +391,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) return nil } - key := cache.Key(kind, hash) + key := cache.LookupKey(kind, hash) var tf *os.File // Tempfile. @@ -299,7 +449,8 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) } removeTempfile = true - err = writeAndCloseFile(r, kind, hash, size, tf) + var sizeOnDisk int64 + sizeOnDisk, err = writeAndCloseFile(r, kind, hash, size, tf) if err != nil { return err } @@ -310,17 +461,17 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) log.Println("Failed to proxy Put:", err) } else { // Doesn't block, should be fast. - c.proxy.Put(kind, hash, size, rc) + c.proxy.Put(kind, hash, sizeOnDisk, rc) } } - unreserve, removeTempfile, err = c.commit(key, tf.Name(), filePath, size, size) + unreserve, removeTempfile, err = c.commit(key, tf.Name(), filePath, size, size, sizeOnDisk) // Might be nil. return err } -func writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int64, f *os.File) error { +func writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int64, f *os.File) (int64, error) { closeFile := true defer func() { if closeFile { @@ -329,44 +480,40 @@ func writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int6 }() var err error + var sizeOnDisk int64 - var bytesCopied int64 if kind == cache.CAS { - hasher := sha256.New() - bytesCopied, err = io.Copy(io.MultiWriter(f, hasher), r) + sizeOnDisk, err = casblob.WriteAndClose(r, f, casblob.Zstandard, hash, size) if err != nil { - return err - } - actualHash := hex.EncodeToString(hasher.Sum(nil)) - if actualHash != hash { - return fmt.Errorf( - "checksums don't match. Expected %s, found %s", hash, actualHash) - } - } else { - if bytesCopied, err = io.Copy(f, r); err != nil { - return err + return -1, err } + closeFile = false + return sizeOnDisk, nil } - if isSizeMismatch(bytesCopied, size) { - return fmt.Errorf( - "sizes don't match. Expected %d, found %d", size, bytesCopied) + if sizeOnDisk, err = io.Copy(f, r); err != nil { + return -1, err + } + + if isSizeMismatch(sizeOnDisk, size) { + return -1, fmt.Errorf( + "sizes don't match. Expected %d, found %d", size, sizeOnDisk) } if err = f.Sync(); err != nil { - return err + return -1, err } if err = f.Close(); err != nil { - return err + return -1, err } closeFile = false - return nil + return sizeOnDisk, nil } // This must be called when the lock is not held. -func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSize int64, foundSize int64) (unreserve bool, removeTempfile bool, err error) { +func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSize int64, foundSize int64, sizeOnDisk int64) (unreserve bool, removeTempfile bool, err error) { unreserve = reservedSize > 0 removeTempfile = true @@ -382,8 +529,9 @@ func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSi } unreserve = false - if !c.lru.Add(key, lruItem{size: foundSize}) { - err = fmt.Errorf("INTERNAL ERROR: failed to add: %s, size %d", key, foundSize) + if !c.lru.Add(key, lruItem{size: foundSize, sizeOnDisk: sizeOnDisk}) { + err = fmt.Errorf("INTERNAL ERROR: failed to add: %s, size %d (on disk: %d): %w", + key, foundSize, sizeOnDisk, err) log.Println(err.Error()) return unreserve, removeTempfile, err } @@ -405,7 +553,7 @@ func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSi // Return a non-nil io.ReadCloser and non-negative size if the item is available // locally, and a boolean that indicates if the item is not available locally // but that we can try the proxy backend. -func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string) (rc io.ReadCloser, foundSize int64, tryProxy bool, err error) { +func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string, kind cache.EntryKind) (rc io.ReadCloser, foundSize int64, tryProxy bool, err error) { locked := true c.mu.Lock() @@ -420,6 +568,13 @@ func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string) (rc if err != nil { // Race condition, was the item purged after we released the lock? log.Printf("Warning: expected %s to exist on disk, undersized cache?", blobPath) + } else if kind == cache.CAS { + rc, err = casblob.GetUncompressedReadCloser(f, size) + if err != nil { + log.Printf("Warning: expected item to be on disk, but something happened: %v", err) + } else { + return rc, item.size, false, nil + } } else { var fileInfo os.FileInfo fileInfo, err = f.Stat() @@ -479,7 +634,7 @@ func (c *Cache) Get(kind cache.EntryKind, hash string, size int64) (rc io.ReadCl } var err error - key := cache.Key(kind, hash) + key := cache.LookupKey(kind, hash) var tf *os.File // Tempfile we will write to. @@ -506,7 +661,7 @@ func (c *Cache) Get(kind cache.EntryKind, hash string, size int64) (rc io.ReadCl }() blobPath := cacheFilePath(kind, c.dir, hash) - f, foundSize, tryProxy, err := c.availableOrTryProxy(key, size, blobPath) + f, foundSize, tryProxy, err := c.availableOrTryProxy(key, size, blobPath, kind) if err != nil { return nil, -1, err } @@ -531,7 +686,8 @@ func (c *Cache) Get(kind cache.EntryKind, hash string, size int64) (rc io.ReadCl if err != nil || r == nil { return nil, -1, err } - if isSizeMismatch(size, foundSize) { + + if kind != cache.CAS && isSizeMismatch(size, foundSize) { return nil, -1, nil } @@ -541,24 +697,48 @@ func (c *Cache) Get(kind cache.EntryKind, hash string, size int64) (rc io.ReadCl } removeTempfile = true - err = writeAndCloseFile(r, kind, hash, foundSize, tf) + tfName := tf.Name() + + var sizeOnDisk int64 + sizeOnDisk, err = io.Copy(tf, r) + tf.Close() if err != nil { return nil, -1, err } - rc, err = os.Open(tf.Name()) + logicalSize := foundSize + + if kind == cache.CAS { + logicalSize, err = casblob.GetLogicalSize(tfName) + if err != nil { + return nil, -1, err + } + if isSizeMismatch(size, logicalSize) { + return nil, -1, nil + } + } + + rcf, err := os.Open(tfName) if err != nil { return nil, -1, err } + if kind != cache.CAS { + rc = rcf + } else { + rc, err = casblob.GetUncompressedReadCloser(rcf, size) + if err != nil { + return nil, -1, err + } + } - unreserve, removeTempfile, err = c.commit(key, tf.Name(), blobPath, size, foundSize) + unreserve, removeTempfile, err = c.commit(key, tfName, blobPath, size, foundSize, sizeOnDisk) if err != nil { rc.Close() rc = nil foundSize = -1 } - return rc, foundSize, err + return rc, logicalSize, err } // Contains returns true if the `hash` key exists in the cache, and @@ -581,7 +761,7 @@ func (c *Cache) Contains(kind cache.EntryKind, hash string, size int64) (bool, i } foundSize := int64(-1) - key := cache.Key(kind, hash) + key := cache.LookupKey(kind, hash) c.mu.Lock() item, exists := c.lru.Get(key) @@ -633,7 +813,7 @@ func ensureDirExists(path string) { } func cacheFilePath(kind cache.EntryKind, cacheDir string, hash string) string { - return filepath.Join(cacheDir, cache.Key(kind, hash)) + return filepath.Join(cacheDir, cache.FileLocation(kind, hash)) } // GetValidatedActionResult returns a valid ActionResult and its serialized diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go index a5378dc70..5d2f2828f 100644 --- a/cache/disk/disk_test.go +++ b/cache/disk/disk_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/buchgr/bazel-remote/cache" + "github.com/buchgr/bazel-remote/cache/disk/casblob" "github.com/buchgr/bazel-remote/cache/httpproxy" testutils "github.com/buchgr/bazel-remote/utils" @@ -43,7 +44,9 @@ func TestCacheBasics(t *testing.T) { defer os.RemoveAll(cacheDir) itemSize := int64(256) - cacheSize := itemSize + + // Add some overhead for likely CAS blob storage expansion. + cacheSize := int64(itemSize * 2) testCache, err := New(cacheDir, cacheSize, nil) if err != nil { @@ -203,18 +206,39 @@ func TestCacheGetContainsWrongSizeWithProxy(t *testing.T) { // digest {contentsHash, contentsLength}. type proxyStub struct{} -func (d proxyStub) Put(kind cache.EntryKind, hash string, size int64, rc io.ReadCloser) {} +func (d proxyStub) Put(kind cache.EntryKind, hash string, size int64, rc io.ReadCloser) { + // Not implemented. +} func (d proxyStub) Get(kind cache.EntryKind, hash string) (io.ReadCloser, int64, error) { - if hash != contentsHash { + if hash != contentsHash || kind != cache.CAS { return nil, -1, nil } - return ioutil.NopCloser(strings.NewReader(contents)), contentsLength, nil + tmpfile, err := ioutil.TempFile("", "proxyStubGet") + if err != nil { + return nil, -1, err + } + tfn := tmpfile.Name() + defer os.Remove(tfn) + + _, err = casblob.WriteAndClose(ioutil.NopCloser( + strings.NewReader(contents)), tmpfile, casblob.Zstandard, + hash, contentsLength) + if err != nil { + return nil, -1, err + } + + readme, err := os.Open(tfn) + if err != nil { + return nil, -1, err + } + + return readme, contentsLength, nil } func (d proxyStub) Contains(kind cache.EntryKind, hash string) (bool, int64) { - if hash != contentsHash { + if hash != contentsHash || kind != cache.CAS { return false, -1 } @@ -271,7 +295,8 @@ func hashStr(content string) string { func TestOverwrite(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 10, nil) + + testCache, err := New(cacheDir, 100, nil) if err != nil { t.Fatal(err) } @@ -311,26 +336,31 @@ func TestCacheExistingFiles(t *testing.T) { items := []struct { contents string hash string + key string file string }{ { "hej", "9c478bf63e9500cb5db1e85ece82f18c8eb9e52e2f9135acd7f10972c8d563ba", - "cas/9c/9c478bf63e9500cb5db1e85ece82f18c8eb9e52e2f9135acd7f10972c8d563ba", + "cas/9c478bf63e9500cb5db1e85ece82f18c8eb9e52e2f9135acd7f10972c8d563ba", + "cas.v2/9c/9c478bf63e9500cb5db1e85ece82f18c8eb9e52e2f9135acd7f10972c8d563ba", }, { "världen", "d497feaa39156f4ae61317db9d2adc3a8f2ff1437fd48ccb56f814f0b7ac5fe1", - "cas/d4/d497feaa39156f4ae61317db9d2adc3a8f2ff1437fd48ccb56f814f0b7ac5fe1", + "cas/d497feaa39156f4ae61317db9d2adc3a8f2ff1437fd48ccb56f814f0b7ac5fe1", + "cas.v2/d4/d497feaa39156f4ae61317db9d2adc3a8f2ff1437fd48ccb56f814f0b7ac5fe1", }, { "foo", "733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", + "ac/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", "ac/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", }, { "bar", "733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", + "raw/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", "raw/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", }, } @@ -341,7 +371,17 @@ func TestCacheExistingFiles(t *testing.T) { fp := path.Join(cacheDir, it.file) ensureDirExists(path.Dir(fp)) - err = ioutil.WriteFile(fp, []byte(it.contents), os.ModePerm) + if strings.HasPrefix(it.file, "cas.v2/") { + r := bytes.NewReader([]byte(it.contents)) + var f *os.File + f, err = os.Create(fp) + if err == nil { + _, err = casblob.WriteAndClose(r, f, casblob.Zstandard, + it.hash, int64(len(it.contents))) + } + } else { + err = ioutil.WriteFile(fp, []byte(it.contents), os.ModePerm) + } if err != nil { t.Fatal(err) } @@ -350,7 +390,10 @@ func TestCacheExistingFiles(t *testing.T) { time.Sleep(10 * time.Millisecond) } - testCache, err := New(cacheDir, 1024, nil) + // Add some overhead for likely CAS blob storage expansion. + const cacheSize = 1024 + + testCache, err := New(cacheDir, cacheSize, nil) if err != nil { t.Fatal(err) } @@ -386,9 +429,9 @@ func TestCacheExistingFiles(t *testing.T) { continue } - if evicted[0] != items[0].file { + if evicted[0] != items[0].key { t.Fatalf("Expected first evicted item to be %s, was %s", - items[0].file, evicted[0]) + items[0].key, evicted[0]) } break // First item evicted as expected. @@ -460,23 +503,62 @@ func createRandomFile(dir string, size int64) (string, error) { return hash, ioutil.WriteFile(filepath, data, os.ModePerm) } +func createRandomV1CASFile(dir string, size int64) (string, error) { + data, hash := testutils.RandomDataAndHash(size) + os.MkdirAll(dir, os.ModePerm) + filePath := dir + "/" + hash + + err := ioutil.WriteFile(filePath, data, os.ModePerm) + if err != nil { + return "", err + } + + return hash, nil +} + +func createRandomCASFile(dir string, size int64) (string, error) { + data, hash := testutils.RandomDataAndHash(size) + os.MkdirAll(dir, os.ModePerm) + filePath := dir + "/" + hash + + f, err := os.Create(filePath) + if err != nil { + return "", nil + } + + r := bytes.NewReader(data) + + _, err = casblob.WriteAndClose(r, f, casblob.Zstandard, hash, int64(len(data))) + if err != nil { + return "", nil + } + + return hash, nil +} + func TestMigrateFromOldDirectoryStructure(t *testing.T) { cacheDir := testutils.TempDir(t) defer os.RemoveAll(cacheDir) - acHash, err := createRandomFile(cacheDir+"/ac/", 512) + acHash, err := createRandomFile(cacheDir+"/ac", 512) if err != nil { t.Fatal(err) } - casHash1, err := createRandomFile(cacheDir+"/cas/", 1024) + + casHash1, err := createRandomV1CASFile(cacheDir+"/cas", 1024) if err != nil { t.Fatal(err) } - casHash2, err := createRandomFile(cacheDir+"/cas/", 1024) + + casHash2, err := createRandomV1CASFile(cacheDir+"/cas", 1024) if err != nil { t.Fatal(err) } - testCache, err := New(cacheDir, 2560, nil) + + // Add some overhead for likely CAS blob storage expansion. + const cacheSize = 2560 * 2 + + testCache, err := New(cacheDir, cacheSize, nil) if err != nil { t.Fatal(err) } @@ -511,20 +593,42 @@ func TestLoadExistingEntries(t *testing.T) { numBlobs := int64(3) blobSize := int64(1024) - acHash, err := testutils.CreateCacheFile(cacheDir+"/ac/", blobSize) + var err error + + acData, acHash := testutils.RandomDataAndHash(blobSize) + err = os.MkdirAll(path.Join(cacheDir, "ac"), 0755) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(path.Join(cacheDir, "ac", acHash), acData, 0644) if err != nil { t.Fatal(err) } - casHash, err := testutils.CreateCacheFile(cacheDir+"/cas/", blobSize) + + casData, casHash := testutils.RandomDataAndHash(blobSize) + err = os.MkdirAll(path.Join(cacheDir, "cas"), 0755) if err != nil { t.Fatal(err) } - rawHash, err := testutils.CreateCacheFile(cacheDir+"/raw/", blobSize) + err = ioutil.WriteFile(path.Join(cacheDir, "cas", casHash), casData, 0644) if err != nil { t.Fatal(err) } - testCache, err := New(cacheDir, blobSize*numBlobs, nil) + rawData, rawHash := testutils.RandomDataAndHash(blobSize) + err = os.MkdirAll(path.Join(cacheDir, "raw"), 0755) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(path.Join(cacheDir, "raw", rawHash), rawData, 0644) + if err != nil { + t.Fatal(err) + } + + // Add some overhead for likely CAS blob storage expansion. + cacheSize := int64(blobSize * numBlobs * 2) + + testCache, err := New(cacheDir, cacheSize, nil) if err != nil { t.Fatal(err) } @@ -558,7 +662,9 @@ func TestDistinctKeyspaces(t *testing.T) { defer os.RemoveAll(cacheDir) blobSize := 1024 - cacheSize := int64(blobSize * 3) + + // Add some overhead for likely CAS blob storage expansion. + cacheSize := int64(blobSize*3) * 2 testCache, err := New(cacheDir, cacheSize, nil) if err != nil { @@ -674,7 +780,8 @@ func TestHttpProxyBackend(t *testing.T) { cacheDir := testutils.TempDir(t) defer os.RemoveAll(cacheDir) - cacheSize := int64(1024 * 10) + // Add some overhead for likely CAS blob storage expansion. + cacheSize := int64(1024*10) * 2 testCache, err := New(cacheDir, cacheSize, proxy) if err != nil { diff --git a/cache/disk/lru.go b/cache/disk/lru.go index 67165b5af..be8143401 100644 --- a/cache/disk/lru.go +++ b/cache/disk/lru.go @@ -81,21 +81,21 @@ func NewSizedLRU(maxSize int64, onEvict EvictCallback) SizedLRU { // and does not add the item) if the item size is larger than the maximum size of the cache, // or it cannot be added to the cache because too much space is reserved. func (c *SizedLRU) Add(key Key, value lruItem) (ok bool) { - if value.size > c.maxSize { + if value.sizeOnDisk > c.maxSize { return false } var sizeDelta int64 if ee, ok := c.cache[key]; ok { - sizeDelta = value.size - ee.Value.(*entry).value.size + sizeDelta = value.sizeOnDisk - ee.Value.(*entry).value.sizeOnDisk if c.reservedSize+sizeDelta > c.maxSize { return false } c.ll.MoveToFront(ee) - counterOverwrittenBytes.Add(float64(ee.Value.(*entry).value.size)) + counterOverwrittenBytes.Add(float64(ee.Value.(*entry).value.sizeOnDisk)) ee.Value.(*entry).value = value } else { - sizeDelta = value.size + sizeDelta = value.sizeOnDisk if c.reservedSize+sizeDelta > c.maxSize { return false } @@ -228,8 +228,8 @@ func (c *SizedLRU) removeElement(e *list.Element) { c.ll.Remove(e) kv := e.Value.(*entry) delete(c.cache, kv.key) - c.currentSize -= kv.value.size - counterEvictedBytes.Add(float64(kv.value.size)) + c.currentSize -= kv.value.sizeOnDisk + counterEvictedBytes.Add(float64(kv.value.sizeOnDisk)) if c.onEvict != nil { c.onEvict(kv.key, kv.value) diff --git a/cache/disk/lru_test.go b/cache/disk/lru_test.go index f482fd33a..bb48c9a64 100644 --- a/cache/disk/lru_test.go +++ b/cache/disk/lru_test.go @@ -36,7 +36,7 @@ func TestBasics(t *testing.T) { // Add an item aKey := "akey" - anItem := lruItem{size: 5} + anItem := lruItem{size: 5, sizeOnDisk: 5} ok = lru.Add(aKey, anItem) if !ok { t.Fatalf("Add: failed inserting item") @@ -85,7 +85,7 @@ func TestEviction(t *testing.T) { var expectedEvictions []int for i, thisExpected := range expectedSizesNumItems { - item := lruItem{size: int64(i)} + item := lruItem{size: int64(i), sizeOnDisk: int64(i)} ok := lru.Add(i, item) if !ok { t.Fatalf("Add: failed adding %d", i) @@ -104,7 +104,7 @@ func TestRejectBigItem(t *testing.T) { // Bounded caches should reject big items lru := NewSizedLRU(10, nil) - ok := lru.Add("hello", lruItem{size: 11}) + ok := lru.Add("hello", lruItem{size: 11, sizeOnDisk: 11}) if ok { t.Fatalf("Add succeeded, expected it to fail") } @@ -113,7 +113,7 @@ func TestRejectBigItem(t *testing.T) { } func TestReserveZeroAlwaysPossible(t *testing.T) { - largeItem := lruItem{size: math.MaxInt64} + largeItem := lruItem{size: math.MaxInt64, sizeOnDisk: math.MaxInt64} lru := NewSizedLRU(math.MaxInt64, nil) lru.Add("foo", largeItem) @@ -256,7 +256,7 @@ func TestAddWithSpaceReserved(t *testing.T) { t.Fatalf("Expected to be able to reserve 1") } - ok = lru.Add("hello", lruItem{size: 2}) + ok = lru.Add("hello", lruItem{size: 2, sizeOnDisk: 2}) if ok { t.Fatal("Expected to not be able to add item with size 2") } @@ -266,7 +266,7 @@ func TestAddWithSpaceReserved(t *testing.T) { t.Fatal("Expected to be able to unreserve 1:", err) } - ok = lru.Add("hello", lruItem{size: 2}) + ok = lru.Add("hello", lruItem{size: 2, sizeOnDisk: 2}) if !ok { t.Fatal("Expected to be able to add item with size 2") } diff --git a/cache/httpproxy/BUILD.bazel b/cache/httpproxy/BUILD.bazel index c2527236c..2c9a6be6e 100644 --- a/cache/httpproxy/BUILD.bazel +++ b/cache/httpproxy/BUILD.bazel @@ -19,6 +19,7 @@ go_test( deps = [ "//cache:go_default_library", "//cache/disk:go_default_library", + "//cache/disk/casblob:go_default_library", "//utils:go_default_library", ], ) diff --git a/cache/httpproxy/httpproxy.go b/cache/httpproxy/httpproxy.go index d640a2239..11e96b909 100644 --- a/cache/httpproxy/httpproxy.go +++ b/cache/httpproxy/httpproxy.go @@ -3,6 +3,8 @@ package httpproxy import ( + "bytes" + "encoding/binary" "errors" "fmt" "io" @@ -190,14 +192,55 @@ func (r *remoteHTTPProxyCache) Contains(kind cache.EntryKind, hash string) (bool url := requestURL(r.baseURL, hash, kind) - rsp, err := r.remote.Head(url) - if err == nil && rsp.StatusCode == http.StatusOK { - return true, rsp.ContentLength + if kind != cache.CAS { + rsp, err := r.remote.Head(url) + if err == nil && rsp.StatusCode == http.StatusOK { + return true, rsp.ContentLength + } + + return false, -1 + } + + // The following code is a little ugly. We can't use a HEAD request, + // because we need the uncompressed size from the header of the blob. + // That value is stored in the first 8 bytes of compressed CAS blobs, + // so attempt a GET range-request to get that value and decode it here. + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return false, -1 + } + + req.Header.Add("Range", "bytes=0-7") + rsp, err := r.remote.Do(req) + if err != nil || (rsp.StatusCode != http.StatusOK && rsp.StatusCode != http.StatusPartialContent) { + return false, -1 } - return false, int64(-1) + blobHeader := make([]byte, 8) + n, err := io.ReadFull(rsp.Body, blobHeader) + if err != nil { + return false, -1 + } + defer rsp.Body.Close() + if n != 8 { + return false, -1 + } + + var uncompressedSize int64 + err = binary.Read(bytes.NewReader(blobHeader), binary.LittleEndian, + &uncompressedSize) + if err != nil { + return false, -1 + } + + return true, uncompressedSize } func requestURL(baseURL *url.URL, hash string, kind cache.EntryKind) string { + if kind == cache.CAS { + // We need to distinguish these from uncompressed CAS blobs. + return fmt.Sprintf("%s/cas.v2/%s", baseURL, hash) + } return fmt.Sprintf("%s/%s/%s", baseURL, kind, hash) } diff --git a/cache/httpproxy/httpproxy_test.go b/cache/httpproxy/httpproxy_test.go index 18ae7a709..74f089190 100644 --- a/cache/httpproxy/httpproxy_test.go +++ b/cache/httpproxy/httpproxy_test.go @@ -2,6 +2,7 @@ package httpproxy import ( "bytes" + "fmt" "io" "io/ioutil" "log" @@ -17,6 +18,7 @@ import ( "github.com/buchgr/bazel-remote/cache" "github.com/buchgr/bazel-remote/cache/disk" + "github.com/buchgr/bazel-remote/cache/disk/casblob" testutils "github.com/buchgr/bazel-remote/utils" ) @@ -32,11 +34,16 @@ func (s *testServer) handler(w http.ResponseWriter, r *http.Request) { fields := strings.Split(r.URL.Path, "/") + kind := fields[1] kindMap := s.ac - if fields[1] == "ac" { + if kind == "ac" { kindMap = s.ac - } else if fields[1] == "cas" { + } else if kind == "cas.v2" { kindMap = s.cas + } else { + msg := fmt.Sprintf("unsupported URL: %q", r.URL.Path) + http.Error(w, msg, http.StatusBadRequest) + return } hash := fields[2] @@ -102,7 +109,7 @@ func TestEverything(t *testing.T) { acData := []byte{1, 2, 3, 4} proxyCache := New(url, &http.Client{}, accessLogger, errorLogger, 100, 10000) - diskCacheSize := int64(len(casData) + 1024) + diskCacheSize := int64(len(casData) + 2048) diskCache, err := disk.New(cacheDir, diskCacheSize, proxyCache) if err != nil { t.Fatal(err) @@ -148,11 +155,43 @@ func TestEverything(t *testing.T) { t.Fatal("Proxied AC value does not match") } } + for _, v := range s.cas { - if !bytes.Equal(v, casData) { + + // TODO: tweak the GetUncompressedReadCloser API to accept more than os.File. + + tmpfile, err := ioutil.TempFile("", "bazel-remote-httpproxy-test") + if err != nil { + t.Fatal(err) + } + tfn := tmpfile.Name() + defer os.Remove(tfn) + + _, err = io.Copy(tmpfile, bytes.NewReader(v)) + if err != nil { + t.Fatal(err) + } + tmpfile2, err := os.Open(tfn) + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile2.Name()) + + rc, err := casblob.GetUncompressedReadCloser(tmpfile2, int64(len(casData))) + if err != nil { + t.Fatal(err) + } + defer rc.Close() + vData, err := ioutil.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(vData, casData) { t.Fatal("Proxied CAS value does not match") } } + s.mu.Unlock() // Confirm that we can HEAD both values successfully. diff --git a/cache/s3proxy/s3proxy.go b/cache/s3proxy/s3proxy.go index 32377a5fc..5febea9f2 100644 --- a/cache/s3proxy/s3proxy.go +++ b/cache/s3proxy/s3proxy.go @@ -1,11 +1,16 @@ package s3proxy import ( + "bytes" "context" + "encoding/binary" "errors" "fmt" "io" "log" + "net/http" + "net/url" + "time" "github.com/buchgr/bazel-remote/cache" "github.com/buchgr/bazel-remote/config" @@ -115,10 +120,10 @@ func New(s3Config *config.S3CloudStorageConfig, accessLogger cache.Logger, } func (c *s3Cache) objectKey(hash string, kind cache.EntryKind) string { - baseKey := fmt.Sprintf("%s/%s", kind, hash) - if c.keyVersion == 2 { - baseKey = cache.Key(kind, hash) - } + + // For CAS blobs this includes "cas.v2" to distinguish compressed + // blobs from older uncompressed blobs. + baseKey := cache.FileLocation(kind, hash) if c.prefix == "" { return baseKey @@ -138,11 +143,6 @@ func logResponse(log cache.Logger, method, bucket, key string, err error) { } func (c *s3Cache) uploadFile(item uploadReq) { - uploadDigest := "" - if item.kind == cache.CAS { - uploadDigest = item.hash - } - _, err := c.mcore.PutObject( context.Background(), c.bucket, // bucketName @@ -150,7 +150,7 @@ func (c *s3Cache) uploadFile(item uploadReq) { item.rc, // reader item.size, // objectSize "", // md5base64 - uploadDigest, // sha256 + "", // sha256 minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "application/octet-stream", @@ -209,21 +209,81 @@ func (c *s3Cache) Get(kind cache.EntryKind, hash string) (io.ReadCloser, int64, func (c *s3Cache) Contains(kind cache.EntryKind, hash string) (bool, int64) { size := int64(-1) + exists := false + + if kind != cache.CAS { + s, err := c.mcore.StatObject( + context.Background(), + c.bucket, // bucketName + c.objectKey(hash, kind), // objectName + minio.StatObjectOptions{}, // opts + ) - s, err := c.mcore.StatObject( - context.Background(), - c.bucket, // bucketName - c.objectKey(hash, kind), // objectName - minio.StatObjectOptions{}, // opts - ) + exists = (err == nil) + if err != nil { + err = errNotFound + } else { + size = s.Size + } + + logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err) + + return exists, size + } + + // Handle the more complicated, compressed CAS blob case. + + // https://github.com/minio/minio-go/issues/1106 + + var err error + var uncompressedSize int64 + var n int + var req *http.Request + var rsp *http.Response + var blobHeader []byte + var u *url.URL + + u, err = c.mcore.PresignedGetObject(context.Background(), c.bucket, + c.objectKey(hash, kind), time.Second*20, make(url.Values)) + if err != nil { + goto end + } + + // TODO: The following code is essentially duplicated from the + // httpproxy code. Refactor and reuse it (and drop the goto's)? - exists := (err == nil) + req, err = http.NewRequest("GET", u.String(), nil) if err != nil { - err = errNotFound - } else { - size = s.Size + goto end + } + req.Header.Add("Range", "bytes=0-7") + rsp, err = http.DefaultClient.Do(req) // FIXME: use a single http.Client + if err != nil { + goto end + } + if rsp.StatusCode != http.StatusOK && rsp.StatusCode != http.StatusPartialContent { + goto end + } + + blobHeader = make([]byte, 8) + n, err = io.ReadFull(rsp.Body, blobHeader) + if err != nil { + goto end + } + defer rsp.Body.Close() + if n != 8 { + goto end + } + + err = binary.Read(bytes.NewReader(blobHeader), binary.LittleEndian, + &uncompressedSize) + if err != nil { + goto end } + exists = true + size = uncompressedSize +end: logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err) return exists, size diff --git a/cache/s3proxy/s3proxy_test.go b/cache/s3proxy/s3proxy_test.go index 30a524548..7ed0a514c 100644 --- a/cache/s3proxy/s3proxy_test.go +++ b/cache/s3proxy/s3proxy_test.go @@ -8,9 +8,8 @@ import ( ) func TestObjectKey(t *testing.T) { - result, expected := "", "" logger := &log.Logger{} - tc := &s3Cache{ + s := &s3Cache{ mcore: nil, bucket: "test", keyVersion: 1, @@ -19,30 +18,25 @@ func TestObjectKey(t *testing.T) { errorLogger: logger, } - // Legacy key format tests - tc.keyVersion = 1 - tc.prefix = "" - result, expected = tc.objectKey("1234", cache.CAS), "cas/1234" - checkTestCase(t, result, expected, "legacy format without prefix") - - tc.prefix = "test" - result, expected = tc.objectKey("1234", cache.CAS), "test/cas/1234" - checkTestCase(t, result, expected, "legacy format with prefix") - // New key format tests - tc.keyVersion = 2 - - tc.prefix = "" - result, expected = tc.objectKey("1234", cache.CAS), "cas/12/1234" - checkTestCase(t, result, expected, "new format with prefix") - - tc.prefix = "test" - result, expected = tc.objectKey("1234", cache.CAS), "test/cas/12/1234" - checkTestCase(t, result, expected, "new format with prefix") -} + s.keyVersion = 2 + + testCases := []struct { + prefix string + key string + kind cache.EntryKind + expected string + }{ + {"", "1234", cache.CAS, "cas.v2/12/1234"}, + {"test", "1234", cache.CAS, "test/cas.v2/12/1234"}, + } -func checkTestCase(t *testing.T, result string, expected string, testCase string) { - if result != expected { - t.Errorf("%s objectKey did not match. (result: '%s' expected: '%s'", testCase, result, expected) + for _, tc := range testCases { + s.prefix = tc.prefix + result := s.objectKey(tc.key, tc.kind) + if result != tc.expected { + t.Errorf("objectKey did not match. (result: '%s' expected: '%s'", + result, tc.expected) + } } } diff --git a/go.mod b/go.mod index 3d1a21fbb..1369d8aaf 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/google/go-cmp v0.5.0 github.com/google/uuid v1.1.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/klauspost/compress v1.11.2 github.com/klauspost/cpuid v1.3.1 // indirect github.com/minio/minio-go/v7 v7.0.1 github.com/mostynb/go-grpc-compression v1.1.4 diff --git a/main.go b/main.go index a97b954d0..e31841a6b 100644 --- a/main.go +++ b/main.go @@ -64,6 +64,11 @@ func main() { app.HideHelpCommand = true app.Flags = []cli.Flag{ + &cli.BoolFlag{ + Name: "destructive_compression_prototype", + Usage: "This flag must be specified for this compressed-blobs prototype to run. Beware that this will migrate preexisting cache files to a temporary and incompatible format.", + EnvVars: []string{"BAZEL_REMOTE_DESTRUCTIVE_COMPRESSION_PROTOTYPE"}, + }, &cli.StringFlag{ Name: "config_file", Value: "", @@ -264,6 +269,11 @@ func main() { } app.Action = func(ctx *cli.Context) error { + if !ctx.Bool("destructive_compression_prototype") { + log.Println("WARNING: this is a prototype branch of bazel-remote that will break existing cache directories. Use the --destructive_compression_prototype flag to acknowledge this and run anyway.") + os.Exit(1) + } + configFile := ctx.String("config_file") var c *config.Config var err error diff --git a/server/grpc_test.go b/server/grpc_test.go index 85acb7868..28b29b359 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -69,7 +69,10 @@ func TestMain(m *testing.M) { } defer os.RemoveAll(dir) - diskCache, err = disk.New(dir, int64(10*maxChunkSize), nil) + // Add some overhead for likely CAS blob storage expansion. + cacheSize := int64(10 * maxChunkSize * 2) + + diskCache, err = disk.New(dir, cacheSize, nil) if err != nil { fmt.Println("Test setup failed") os.Exit(1) @@ -482,7 +485,7 @@ func TestGrpcAcRequestInlinedBlobs(t *testing.T) { } if r.Status.GetCode() != int32(codes.OK) { - t.Fatal("missing blob:", r.Digest) + t.Fatal("missing blob:", r.Digest, "message:", r.Status.GetMessage()) } } diff --git a/server/http_test.go b/server/http_test.go index 64db42090..0cc8edd98 100644 --- a/server/http_test.go +++ b/server/http_test.go @@ -30,12 +30,20 @@ func TestDownloadFile(t *testing.T) { blobSize := int64(1024) - hash, err := testutils.CreateCacheFile(filepath.Join(cacheDir, "cas"), blobSize) + data, hash := testutils.RandomDataAndHash(blobSize) + err := os.MkdirAll(filepath.Join(cacheDir, "cas"), 0755) if err != nil { t.Fatal(err) } + err = ioutil.WriteFile(filepath.Join(cacheDir, "cas", hash), data, 0644) + if err != nil { + t.Fatal(err) + } + + // Add some overhead for likely CAS blob storage expansion. + cacheSize := blobSize * 2 - c, err := disk.New(cacheDir, blobSize, nil) + c, err := disk.New(cacheDir, cacheSize, nil) if err != nil { t.Fatal(err) } @@ -50,7 +58,7 @@ func TestDownloadFile(t *testing.T) { handler.ServeHTTP(rr, req) if status := rr.Code; status != http.StatusOK { - t.Fatal("Handler returned wrong status code", + t.Fatal("Handler returned wrong status code for", hash, "expected", http.StatusOK, "got", status, ) @@ -90,10 +98,11 @@ func TestUploadFilesConcurrently(t *testing.T) { defer os.RemoveAll(cacheDir) const NumUploads = 1000 + const blobSize = 1024 var requests [NumUploads]*http.Request for i := 0; i < NumUploads; i++ { - data, hash := testutils.RandomDataAndHash(1024) + data, hash := testutils.RandomDataAndHash(blobSize) r, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data)) if err != nil { t.Fatal(err) @@ -101,7 +110,10 @@ func TestUploadFilesConcurrently(t *testing.T) { requests[i] = r } - c, err := disk.New(cacheDir, 1000*1024, nil) + // Add some overhead for likely CAS blob storage expansion. + cacheSize := int64(NumUploads * blobSize * 2) + + c, err := disk.New(cacheDir, cacheSize, nil) if err != nil { t.Fatal(err) } @@ -162,7 +174,10 @@ func TestUploadSameFileConcurrently(t *testing.T) { numWorkers := 100 - c, err := disk.New(cacheDir, int64(len(data)*numWorkers), nil) + // Add some overhead for likely CAS blob storage expansion. + cacheSize := int64(len(data) * numWorkers * 2) + + c, err := disk.New(cacheDir, cacheSize, nil) if err != nil { t.Fatal(err) } diff --git a/utils/testutils.go b/utils/testutils.go index f0ee56037..2a6513670 100644 --- a/utils/testutils.go +++ b/utils/testutils.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "io/ioutil" "log" - "os" "testing" ) @@ -20,18 +19,6 @@ func TempDir(t *testing.T) string { return dir } -// CreateCacheFile creates a random data file of the given size in the -// provided directory (corresponding to one of bazel-remote's keyspaces) -// and returns its sha256 hash and any error that occurred. -func CreateCacheFile(dir string, size int64) (string, error) { - data, hash := RandomDataAndHash(size) - subdir := dir + "/" + hash[0:2] - os.MkdirAll(subdir, os.ModePerm) - filepath := subdir + "/" + hash - - return hash, ioutil.WriteFile(filepath, data, os.ModePerm) -} - // RandomDataAndHash creates a random blob of the specified size, and // returns that blob along with its sha256 hash. func RandomDataAndHash(size int64) ([]byte, string) { From e9fbf6e88e79845e4f7100dc83f1dc0e25a797f5 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Thu, 31 Dec 2020 12:56:05 +0100 Subject: [PATCH 03/35] [disk] try to preserve atimes when migrating CAS blobs --- cache/disk/disk.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 42e60a46f..5cc6f5f76 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -281,6 +281,7 @@ func migrateCASFile(src string, dest string, hash string) error { } origSize := fi.Size() + accessTime := atime.Get(fi) destFile, err := os.Create(dest) if err != nil { @@ -291,6 +292,7 @@ func migrateCASFile(src string, dest string, hash string) error { if err != nil { return err } + os.Chtimes(dest, accessTime, fi.ModTime()) // Best effort, ignore errors. err = os.Remove(src) if err != nil { From f522e31634f2347a205ee2bcc8cffc8d175c390e Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sun, 20 Dec 2020 13:22:38 +0100 Subject: [PATCH 04/35] [bytestream] add support for compressed-blobs reads and writes --- WORKSPACE | 4 +- cache/disk/disk.go | 69 +++- cache/disk/disk_test.go | 87 ++++- cache/disk/lru.go | 28 +- cache/httpproxy/httpproxy_test.go | 12 +- .../bazel/remote/execution/v2/BUILD.bazel | 1 + go.mod | 5 +- server/BUILD.bazel | 4 + server/grpc.go | 1 + server/grpc_ac.go | 2 +- server/grpc_asset.go | 2 +- server/grpc_bytestream.go | 214 +++++++++--- server/grpc_cas.go | 2 +- server/grpc_test.go | 329 +++++++++++++++++- server/http.go | 36 +- 15 files changed, 678 insertions(+), 118 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index fe4a4d65d..b0e768273 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -976,8 +976,8 @@ go_repository( go_repository( name = "com_github_mostynb_go_grpc_compression", importpath = "github.com/mostynb/go-grpc-compression", - sum = "h1:emrImHjSPW/H4aarNxuiPG1uN1WgIlVdR3M0LH5HH5E=", - version = "v1.1.2", + sum = "h1:wlscKqxoQsZQxUi6uTER+Gh9MbcUFUa4Cw5A69pj/c8=", + version = "v1.1.4", ) go_repository( diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 5cc6f5f76..4908a2768 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -3,6 +3,7 @@ package disk import ( "bytes" "crypto/sha256" + "errors" "fmt" "io" "io/ioutil" @@ -42,9 +43,14 @@ var ( var tfc = tempfile.NewCreator() +var emptyZstdBlob = []byte{40, 181, 47, 253, 32, 0, 1, 0, 0} + // lruItem is the type of the values stored in SizedLRU to keep track of items. type lruItem struct { - size int64 + // Size of the blob in uncompressed form. + size int64 + + // Size of the blob on disk (possibly with header + compression). sizeOnDisk int64 } @@ -555,7 +561,9 @@ func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSi // Return a non-nil io.ReadCloser and non-negative size if the item is available // locally, and a boolean that indicates if the item is not available locally // but that we can try the proxy backend. -func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string, kind cache.EntryKind) (rc io.ReadCloser, foundSize int64, tryProxy bool, err error) { +// +// This function assumes that only CAS blobs are requested in zstd form. +func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string, kind cache.EntryKind, offset int64, zstd bool) (rc io.ReadCloser, foundSize int64, tryProxy bool, err error) { locked := true c.mu.Lock() @@ -571,7 +579,11 @@ func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string, kin // Race condition, was the item purged after we released the lock? log.Printf("Warning: expected %s to exist on disk, undersized cache?", blobPath) } else if kind == cache.CAS { - rc, err = casblob.GetUncompressedReadCloser(f, size) + if zstd { + rc, err = casblob.GetZstdReadCloser(f, size, offset) + } else { + rc, err = casblob.GetUncompressedReadCloser(f, size, offset) + } if err != nil { log.Printf("Warning: expected item to be on disk, but something happened: %v", err) } else { @@ -586,6 +598,7 @@ func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string, kin log.Printf("Warning: expected %s to on disk to have size %d, found %d", blobPath, size, foundSize) } else { + f.Seek(offset, io.SeekStart) return f, foundSize, false, nil } } @@ -616,13 +629,25 @@ func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string, kin return nil, -1, tryProxy, err } +var errOnlyCompressedCAS = errors.New("Only CAS blobs are available in compressed form") + // Get returns an io.ReadCloser with the content of the cache item stored // under `hash` and the number of bytes that can be read from it. If the // item is not found, the io.ReadCloser will be nil. If some error occurred // when processing the request, then it is returned. Callers should provide // the `size` of the item to be retrieved, or -1 if unknown. -func (c *Cache) Get(kind cache.EntryKind, hash string, size int64) (rc io.ReadCloser, s int64, rErr error) { +func (c *Cache) Get(kind cache.EntryKind, hash string, size int64, offset int64) (rc io.ReadCloser, s int64, rErr error) { + return c.get(kind, hash, size, offset, false) +} + +// GetZstd is just like Get, except the data available from rc is zstandard +// compressed. Note that the returned `s` value still refers to the amount +// of data once it has been decompressed. +func (c *Cache) GetZstd(hash string, size int64, offset int64) (rc io.ReadCloser, s int64, rErr error) { + return c.get(cache.CAS, hash, size, offset, true) +} +func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, zstd bool) (rc io.ReadCloser, s int64, rErr error) { // The hash format is checked properly in the http/grpc code. // Just perform a simple/fast check here, to catch bad tests. if len(hash) != sha256HashStrSize { @@ -632,9 +657,25 @@ func (c *Cache) Get(kind cache.EntryKind, hash string, size int64) (rc io.ReadCl if kind == cache.CAS && size <= 0 && hash == emptySha256 { cacheHits.Inc() + + if zstd { + return ioutil.NopCloser(bytes.NewReader(emptyZstdBlob)), 0, nil + } + return ioutil.NopCloser(bytes.NewReader([]byte{})), 0, nil } + if kind != cache.CAS && zstd { + return nil, -1, errOnlyCompressedCAS + } + + if offset < 0 { + return nil, -1, fmt.Errorf("Invalid offset: %d", offset) + } + if size > 0 && offset >= size { + return nil, -1, fmt.Errorf("Invalid offset: %d for size %d", offset, size) + } + var err error key := cache.LookupKey(kind, hash) @@ -663,7 +704,7 @@ func (c *Cache) Get(kind cache.EntryKind, hash string, size int64) (rc io.ReadCl }() blobPath := cacheFilePath(kind, c.dir, hash) - f, foundSize, tryProxy, err := c.availableOrTryProxy(key, size, blobPath, kind) + f, foundSize, tryProxy, err := c.availableOrTryProxy(key, size, blobPath, kind, offset, zstd) if err != nil { return nil, -1, err } @@ -725,9 +766,17 @@ func (c *Cache) Get(kind cache.EntryKind, hash string, size int64) (rc io.ReadCl return nil, -1, err } if kind != cache.CAS { + if offset > 0 { + rcf.Seek(offset, io.SeekStart) + } rc = rcf + } else if zstd { + rc, err = casblob.GetZstdReadCloser(rcf, size, offset) + if err != nil { + return nil, -1, err + } } else { - rc, err = casblob.GetUncompressedReadCloser(rcf, size) + rc, err = casblob.GetUncompressedReadCloser(rcf, size, offset) if err != nil { return nil, -1, err } @@ -794,11 +843,11 @@ func (c *Cache) MaxSize() int64 { // Stats returns the current size of the cache in bytes, and the number of // items stored in the cache. -func (c *Cache) Stats() (totalSize int64, reservedSize int64, numItems int) { +func (c *Cache) Stats() (totalSize int64, reservedSize int64, numItems int, uncompressedSize int64) { c.mu.Lock() defer c.mu.Unlock() - return c.lru.TotalSize(), c.lru.ReservedSize(), c.lru.Len() + return c.lru.TotalSize(), c.lru.ReservedSize(), c.lru.Len(), c.lru.UncompressedSize() } func isSizeMismatch(requestedSize int64, foundSize int64) bool { @@ -824,7 +873,7 @@ func cacheFilePath(kind cache.EntryKind, cacheDir string, hash string) string { // an error. func (c *Cache) GetValidatedActionResult(hash string) (*pb.ActionResult, []byte, error) { - rc, sizeBytes, err := c.Get(cache.AC, hash, -1) + rc, sizeBytes, err := c.Get(cache.AC, hash, -1, 0) if rc != nil { defer rc.Close() } @@ -857,7 +906,7 @@ func (c *Cache) GetValidatedActionResult(hash string) (*pb.ActionResult, []byte, } for _, d := range result.OutputDirectories { - r, size, err := c.Get(cache.CAS, d.TreeDigest.Hash, d.TreeDigest.SizeBytes) + r, size, err := c.Get(cache.CAS, d.TreeDigest.Hash, d.TreeDigest.SizeBytes, 0) if r == nil { return nil, nil, err // aka "not found", or an err if non-nil } diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go index 5d2f2828f..ded345621 100644 --- a/cache/disk/disk_test.go +++ b/cache/disk/disk_test.go @@ -61,7 +61,7 @@ func TestCacheBasics(t *testing.T) { data, hash := testutils.RandomDataAndHash(itemSize) // Non-existing item. - rdr, _, err := testCache.Get(cache.CAS, hash, itemSize) + rdr, _, err := testCache.Get(cache.CAS, hash, itemSize, 0) if err != nil { t.Fatal(err) } @@ -77,7 +77,7 @@ func TestCacheBasics(t *testing.T) { } // Get the item back. - rdr, sizeBytes, err := testCache.Get(cache.CAS, hash, itemSize) + rdr, sizeBytes, err := testCache.Get(cache.CAS, hash, itemSize, 0) if err != nil { t.Fatal(err) } @@ -136,7 +136,7 @@ func TestCacheGetContainsWrongSize(t *testing.T) { t.Error("Expected not found, due to size being different") } - rdr, _, _ = testCache.Get(cache.CAS, contentsHash, contentsLength+1) + rdr, _, _ = testCache.Get(cache.CAS, contentsHash, contentsLength+1, 0) if rdr != nil { t.Error("Expected not found, due to size being different") } @@ -146,7 +146,7 @@ func TestCacheGetContainsWrongSize(t *testing.T) { t.Error("Expected found, when unknown size") } - rdr, _, _ = testCache.Get(cache.CAS, contentsHash, -1) + rdr, _, _ = testCache.Get(cache.CAS, contentsHash, -1, 0) if rdr == nil { t.Error("Expected found, when unknown size") } @@ -176,7 +176,7 @@ func TestCacheGetContainsWrongSizeWithProxy(t *testing.T) { t.Fatal("Expected not found, due to size being different") } - rdr, _, _ = testCache.Get(cache.CAS, contentsHash, contentsLength+1) + rdr, _, _ = testCache.Get(cache.CAS, contentsHash, contentsLength+1, 0) if rdr != nil { t.Fatal("Expected not found, due to size being different") } @@ -191,7 +191,7 @@ func TestCacheGetContainsWrongSizeWithProxy(t *testing.T) { t.Fatal("Expected found, when unknown size") } - rdr, _, _ = testCache.Get(cache.CAS, contentsHash, -1) + rdr, _, _ = testCache.Get(cache.CAS, contentsHash, -1, 0) if rdr == nil { t.Fatal("Expected found, when unknown size") } @@ -278,7 +278,7 @@ func putGetCompareBytes(kind cache.EntryKind, hash string, data []byte, testCach return err } - rdr, sizeBytes, err := testCache.Get(kind, hash, int64(len(data))) + rdr, sizeBytes, err := testCache.Get(kind, hash, int64(len(data)), 0) if err != nil { return err } @@ -563,7 +563,7 @@ func TestMigrateFromOldDirectoryStructure(t *testing.T) { t.Fatal(err) } - _, _, numItems := testCache.Stats() + _, _, numItems, _ := testCache.Stats() if numItems != 3 { t.Fatalf("Expected test cache size 3 but was %d", numItems) } @@ -633,7 +633,7 @@ func TestLoadExistingEntries(t *testing.T) { t.Fatal(err) } - _, _, numItems := testCache.Stats() + _, _, numItems, _ := testCache.Stats() if int64(numItems) != numBlobs { t.Fatalf("Expected test cache size %d but was %d", numBlobs, numItems) @@ -692,7 +692,7 @@ func TestDistinctKeyspaces(t *testing.T) { t.Fatal(err) } - _, _, numItems := testCache.Stats() + _, _, numItems, _ := testCache.Stats() if numItems != 3 { t.Fatalf("Expected test cache size 3 but was %d", numItems) @@ -792,7 +792,7 @@ func TestHttpProxyBackend(t *testing.T) { blob, casHash := testutils.RandomDataAndHash(blobSize) // Non-existing item - r, _, err := testCache.Get(cache.CAS, casHash, blobSize) + r, _, err := testCache.Get(cache.CAS, casHash, blobSize, 0) if err != nil { t.Fatal(err) } @@ -834,7 +834,7 @@ func TestHttpProxyBackend(t *testing.T) { t.Fatalf("Expected the cache not to contain %s", casHash) } - r, _, err = testCache.Get(cache.CAS, casHash, blobSize) + r, _, err = testCache.Get(cache.CAS, casHash, blobSize, 0) if err != nil { t.Fatal(err) } @@ -851,7 +851,7 @@ func TestHttpProxyBackend(t *testing.T) { casHash) } - r, fetchedSize, err := testCache.Get(cache.CAS, casHash, blobSize) + r, fetchedSize, err := testCache.Get(cache.CAS, casHash, blobSize, 0) if err != nil { t.Fatal(err) } @@ -1048,3 +1048,64 @@ func TestGetValidatedActionResult(t *testing.T) { t.Fatal("Returned ActionResult proto does not match") } } + +func TestGetWithOffset(t *testing.T) { + cacheDir := testutils.TempDir(t) + defer os.RemoveAll(cacheDir) + + const blobSize = 2048 + 256 + + testCache, err := New(cacheDir, blobSize*2, nil) + if err != nil { + t.Fatal(err) + } + + data, hash := testutils.RandomDataAndHash(blobSize) + + err = testCache.Put(cache.CAS, hash, blobSize, + ioutil.NopCloser(bytes.NewReader(data))) + if err != nil { + t.Fatal(err) + } + + rc, foundSize, err := testCache.Get(cache.CAS, hash, int64(len(data)), 0) + if err != nil { + t.Fatal(err) + } + if foundSize != int64(len(data)) { + t.Fatalf("Got back a blob with size %d, original blob had size %d", + foundSize, int64(len(data))) + } + + // Read back the full blob, confirm the test state is OK. + foundData, err := ioutil.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + rc.Close() + if !bytes.Equal(data, foundData) { + t.Fatal("Got back different data") + } + + // Now try some partial reads. + for _, offset := range []int64{42, 1023, 1024, 1025, 2048, 2303} { + rc, foundSize, err = testCache.Get(cache.CAS, hash, int64(len(data)), offset) + if err != nil { + t.Fatal(err) + } + if foundSize != int64(len(data)) { + t.Fatalf("Got back a blob with size %d, original blob had size %d", + foundSize, int64(len(data))) + } + + foundData, err = ioutil.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + rc.Close() + if !bytes.Equal(data[offset:], foundData) { + t.Fatalf("Expected data (%d bytes), differs from actual data (%d bytes) for offset %d", + len(data[offset:]), len(foundData), offset) + } + } +} diff --git a/cache/disk/lru.go b/cache/disk/lru.go index be8143401..93c05a7e8 100644 --- a/cache/disk/lru.go +++ b/cache/disk/lru.go @@ -15,6 +15,11 @@ var ( Help: "The current number of bytes in the disk backend", }) + gaugeCacheLogicalBytes = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "bazel_remote_disk_cache_logical_bytes", + Help: "The current number of bytes in the disk backend if they were uncompressed", + }) + counterEvictedBytes = promauto.NewCounter(prometheus.CounterOpts{ Name: "bazel_remote_disk_cache_evicted_bytes_total", Help: "The total number of bytes evicted from disk backend, due to full cache", @@ -50,13 +55,18 @@ type SizedLRU struct { // Map to access the items in O(1) time cache map[interface{}]*list.Element - // Includes reserved size. + // Total cache size including reserved bytes. currentSize int64 + // Total size of all blobs in uncompressed form. + // This does not include reserved space. + uncompressedSize int64 + + // Number of bytes reserved for incoming blobs. reservedSize int64 - // SizedLRU will evict items as needed to maintain the total size of the cache - // below maxSize. + // SizedLRU will evict items as needed to maintain the total size of the + // cache below maxSize. maxSize int64 onEvict EvictCallback @@ -85,12 +95,13 @@ func (c *SizedLRU) Add(key Key, value lruItem) (ok bool) { return false } - var sizeDelta int64 + var sizeDelta, uncompressedSizeDelta int64 if ee, ok := c.cache[key]; ok { sizeDelta = value.sizeOnDisk - ee.Value.(*entry).value.sizeOnDisk if c.reservedSize+sizeDelta > c.maxSize { return false } + uncompressedSizeDelta = value.size - ee.Value.(*entry).value.size c.ll.MoveToFront(ee) counterOverwrittenBytes.Add(float64(ee.Value.(*entry).value.sizeOnDisk)) ee.Value.(*entry).value = value @@ -99,6 +110,7 @@ func (c *SizedLRU) Add(key Key, value lruItem) (ok bool) { if c.reservedSize+sizeDelta > c.maxSize { return false } + uncompressedSizeDelta = value.size ele := c.ll.PushFront(&entry{key, value}) c.cache[key] = ele } @@ -113,8 +125,10 @@ func (c *SizedLRU) Add(key Key, value lruItem) (ok bool) { } c.currentSize += sizeDelta + c.uncompressedSize += uncompressedSizeDelta gaugeCacheSizeBytes.Set(float64(c.currentSize)) + gaugeCacheLogicalBytes.Set(float64(c.uncompressedSize)) return true } @@ -134,6 +148,7 @@ func (c *SizedLRU) Remove(key Key) { if ele, hit := c.cache[key]; hit { c.removeElement(ele) gaugeCacheSizeBytes.Set(float64(c.currentSize)) + gaugeCacheLogicalBytes.Set(float64(c.uncompressedSize)) } } @@ -146,6 +161,10 @@ func (c *SizedLRU) TotalSize() int64 { return c.currentSize } +func (c *SizedLRU) UncompressedSize() int64 { + return c.uncompressedSize +} + func (c *SizedLRU) ReservedSize() int64 { return c.reservedSize } @@ -229,6 +248,7 @@ func (c *SizedLRU) removeElement(e *list.Element) { kv := e.Value.(*entry) delete(c.cache, kv.key) c.currentSize -= kv.value.sizeOnDisk + c.uncompressedSize -= kv.value.size counterEvictedBytes.Add(float64(kv.value.sizeOnDisk)) if c.onEvict != nil { diff --git a/cache/httpproxy/httpproxy_test.go b/cache/httpproxy/httpproxy_test.go index 74f089190..e8342dec7 100644 --- a/cache/httpproxy/httpproxy_test.go +++ b/cache/httpproxy/httpproxy_test.go @@ -177,7 +177,7 @@ func TestEverything(t *testing.T) { } defer os.Remove(tmpfile2.Name()) - rc, err := casblob.GetUncompressedReadCloser(tmpfile2, int64(len(casData))) + rc, err := casblob.GetUncompressedReadCloser(tmpfile2, int64(len(casData)), 0) if err != nil { t.Fatal(err) } @@ -222,7 +222,7 @@ func TestEverything(t *testing.T) { var data []byte var rc io.ReadCloser - rc, size, err = diskCache.Get(cache.AC, hash, int64(len(acData))) + rc, size, err = diskCache.Get(cache.AC, hash, int64(len(acData)), 0) if err != nil { t.Error(err) } @@ -242,7 +242,7 @@ func TestEverything(t *testing.T) { } rc.Close() - rc, size, err = diskCache.Get(cache.CAS, hash, int64(len(casData))) + rc, size, err = diskCache.Get(cache.CAS, hash, int64(len(casData)), 0) if err != nil { t.Error(err) } @@ -273,7 +273,7 @@ func TestEverything(t *testing.T) { t.Fatal(err) } - _, _, numItems := diskCache.Stats() + _, _, numItems, _ := diskCache.Stats() if numItems != 0 { t.Fatalf("Expected an empty disk cache, found %d items", numItems) } @@ -300,7 +300,7 @@ func TestEverything(t *testing.T) { // Confirm that we can GET both values successfully. - rc, size, err = diskCache.Get(cache.AC, hash, int64(len(acData))) + rc, size, err = diskCache.Get(cache.AC, hash, int64(len(acData)), 0) if err != nil { t.Error(err) } @@ -320,7 +320,7 @@ func TestEverything(t *testing.T) { } rc.Close() - rc, size, err = diskCache.Get(cache.CAS, hash, int64(len(casData))) + rc, size, err = diskCache.Get(cache.CAS, hash, int64(len(casData)), 0) if err != nil { t.Error(err) } diff --git a/genproto/build/bazel/remote/execution/v2/BUILD.bazel b/genproto/build/bazel/remote/execution/v2/BUILD.bazel index fd153acbb..5ef2a7a5f 100644 --- a/genproto/build/bazel/remote/execution/v2/BUILD.bazel +++ b/genproto/build/bazel/remote/execution/v2/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "@go_googleapis//google/api:annotations_go_proto", "@go_googleapis//google/longrunning:longrunning_go_proto", "@go_googleapis//google/rpc:status_go_proto", + "@io_bazel_rules_go//proto/wkt:any_go_proto", "@io_bazel_rules_go//proto/wkt:duration_go_proto", "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", "@io_bazel_rules_go//proto/wkt:wrappers_go_proto", diff --git a/go.mod b/go.mod index 1369d8aaf..aa14a25a2 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,13 @@ module github.com/buchgr/bazel-remote require ( cloud.google.com/go v0.61.0 // indirect github.com/abbot/go-http-auth v0.4.1-0.20181019201920-860ed7f246ff - github.com/bazelbuild/remote-apis v0.0.0-20200708200203-1252343900d9 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/djherbis/atime v1.0.0 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.0 github.com/google/uuid v1.1.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/klauspost/compress v1.11.2 + github.com/klauspost/compress v1.11.3 github.com/klauspost/cpuid v1.3.1 // indirect github.com/minio/minio-go/v7 v7.0.1 github.com/mostynb/go-grpc-compression v1.1.4 @@ -26,4 +25,4 @@ require ( gopkg.in/yaml.v2 v2.3.0 ) -go 1.14 +go 1.15 diff --git a/server/BUILD.bazel b/server/BUILD.bazel index f60cf8044..f23ccd746 100644 --- a/server/BUILD.bazel +++ b/server/BUILD.bazel @@ -17,6 +17,7 @@ go_library( deps = [ "//cache:go_default_library", "//cache/disk:go_default_library", + "//cache/disk/casblob:go_default_library", "//genproto/build/bazel/remote/asset/v1:go_default_library", "//genproto/build/bazel/remote/execution/v2:go_default_library", "//genproto/build/bazel/semver:go_default_library", @@ -24,6 +25,7 @@ go_library( "@com_github_abbot_go_http_auth//:go_default_library", "@com_github_golang_protobuf//jsonpb:go_default_library_gen", "@com_github_golang_protobuf//proto:go_default_library", + "@com_github_klauspost_compress//zstd:go_default_library", "@com_github_mostynb_go_grpc_compression//snappy:go_default_library", "@com_github_mostynb_go_grpc_compression//zstd:go_default_library", "@go_googleapis//google/bytestream:bytestream_go_proto", @@ -49,11 +51,13 @@ go_test( deps = [ "//cache:go_default_library", "//cache/disk:go_default_library", + "//cache/disk/casblob:go_default_library", "//genproto/build/bazel/remote/asset/v1:go_default_library", "//genproto/build/bazel/remote/execution/v2:go_default_library", "//utils:go_default_library", "@com_github_golang_protobuf//proto:go_default_library", "@com_github_google_uuid//:go_default_library", + "@com_github_klauspost_compress//zstd:go_default_library", "@go_googleapis//google/bytestream:bytestream_go_proto", "@org_golang_google_grpc//:go_default_library", "@org_golang_google_grpc//codes:go_default_library", diff --git a/server/grpc.go b/server/grpc.go index 6b988e810..035f7bb18 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -103,6 +103,7 @@ func (s *grpcServer) GetCapabilities(ctx context.Context, }, MaxBatchTotalSizeBytes: 0, // "no limit" SymlinkAbsolutePathStrategy: pb.SymlinkAbsolutePathStrategy_ALLOWED, + SupportedCompressor: []pb.Compressor_Value{pb.Compressor_ZSTD}, }, LowApiVersion: &semver.SemVer{Major: int32(2)}, HighApiVersion: &semver.SemVer{Major: int32(2), Minor: int32(1)}, diff --git a/server/grpc_ac.go b/server/grpc_ac.go index 130c644e7..691717293 100644 --- a/server/grpc_ac.go +++ b/server/grpc_ac.go @@ -56,7 +56,7 @@ func (s *grpcServer) GetActionResult(ctx context.Context, if !s.depsCheck { logPrefix = "GRPC AC GET NODEPSCHECK" - rdr, sizeBytes, err := s.cache.Get(cache.AC, req.ActionDigest.Hash, unknownActionResultSize) + rdr, sizeBytes, err := s.cache.Get(cache.AC, req.ActionDigest.Hash, unknownActionResultSize, 0) if err != nil { s.accessLogger.Printf("%s %s %s", logPrefix, req.ActionDigest.Hash, err) return nil, status.Error(codes.Unknown, err.Error()) diff --git a/server/grpc_asset.go b/server/grpc_asset.go index 366eeb033..559d1273a 100644 --- a/server/grpc_asset.go +++ b/server/grpc_asset.go @@ -70,7 +70,7 @@ func (s *grpcServer) FetchBlob(ctx context.Context, req *asset.FetchBlobRequest) if size < 0 { // We don't know the size yet (bad http backend?). - r, size, err := s.cache.Get(cache.CAS, sha256Str, -1) + r, size, err := s.cache.Get(cache.CAS, sha256Str, -1, 0) if r != nil { defer r.Close() } diff --git a/server/grpc_bytestream.go b/server/grpc_bytestream.go index e320d6d57..d34797813 100644 --- a/server/grpc_bytestream.go +++ b/server/grpc_bytestream.go @@ -14,6 +14,9 @@ import ( "google.golang.org/grpc/status" "github.com/buchgr/bazel-remote/cache" + "github.com/buchgr/bazel-remote/cache/disk/casblob" + + "github.com/klauspost/compress/zstd" ) const ( @@ -27,11 +30,26 @@ const ( func (s *grpcServer) Read(req *bytestream.ReadRequest, resp bytestream.ByteStream_ReadServer) error { + errorPrefix := "GRPC BYTESTREAM READ" + + var cmp casblob.CompressionType + hash, size, cmp, err := s.parseReadResource(req.ResourceName, errorPrefix) + if err != nil { + return err + } + if req.ReadOffset < 0 { - s.accessLogger.Printf("GRPC BYTESTREAM READ OFFSET OUT OF RANGE: %s %d", + s.accessLogger.Printf("GRPC BYTESTREAM READ OFFSET INVALID: %s %d", req.ReadOffset) - return status.Error(codes.OutOfRange, - "Negative ReadOffset is out of range") + return status.Error(codes.InvalidArgument, + "Negative ReadOffset is invalid") + } + + if cmp != casblob.Identity && req.ReadLimit != 0 { + s.accessLogger.Printf("GRPC BYTESTREAM READ LIMIT INVALID: %s %d", + req.ReadLimit) + return status.Error(codes.InvalidArgument, + "ReadLimit must be 0 for compressed-blobs") } if req.ReadLimit < 0 { @@ -41,14 +59,8 @@ func (s *grpcServer) Read(req *bytestream.ReadRequest, "Negative ReadLimit is out of range") } - limitedSend := req.ReadLimit != 0 - - errorPrefix := "GRPC BYTESTREAM READ" - - hash, size, err := s.parseReadResource(req.ResourceName, errorPrefix) - if err != nil { - return err - } + limitedSend := (req.ReadLimit != 0) && cmp == casblob.Identity + sendLimitRemaining := req.ReadLimit if req.ReadOffset > size { msg := fmt.Sprintf("ReadOffset %d larger than expected data size %d resource: %s", @@ -57,7 +69,15 @@ func (s *grpcServer) Read(req *bytestream.ReadRequest, return status.Error(codes.OutOfRange, msg) } - rdr, foundSize, err := s.cache.Get(cache.CAS, hash, size) + var rdr io.ReadCloser + var foundSize int64 + + if cmp == casblob.Zstandard { + rdr, foundSize, err = s.cache.GetZstd(hash, size, req.ReadOffset) + } else { + rdr, foundSize, err = s.cache.Get(cache.CAS, hash, size, req.ReadOffset) + } + if err != nil { msg := fmt.Sprintf("GRPC BYTESTREAM READ FAILED: %s %v", hash, err) s.accessLogger.Printf(msg) @@ -85,12 +105,6 @@ func (s *grpcServer) Read(req *bytestream.ReadRequest, buf := make([]byte, bufSize) - if req.ReadOffset > 0 { - seekReaderTo(rdr, req.ReadOffset) - } - - sendLimitRemaining := req.ReadLimit - var chunkResp bytestream.ReadResponse for { n, err := rdr.Read(buf) @@ -128,11 +142,13 @@ func (s *grpcServer) Read(req *bytestream.ReadRequest, } } -// Parse a ReadRequest.ResourceName, return the validated hash, size and an error. -func (s *grpcServer) parseReadResource(name string, errorPrefix string) (string, int64, error) { +// Parse a ReadRequest.ResourceName, return the validated hash, size, compression type and an error. +func (s *grpcServer) parseReadResource(name string, errorPrefix string) (string, int64, casblob.CompressionType, error) { // The resource name should be of the format: // [{instance_name}]/blobs/{hash}/{size} + // Or: + // [{instance_name}]/compressed-blobs/{compressor}/{uncompressed_hash}/{uncompressed_size} // Instance_name is ignored in this bytestream implementation, so don't // bother returning it. It is not allowed to contain "blobs" as a distinct @@ -140,41 +156,91 @@ func (s *grpcServer) parseReadResource(name string, errorPrefix string) (string, fields := strings.Split(name, "/") var rem []string - found := false + foundBlobs := false + foundCompressedBlobs := false for i := range fields { if fields[i] == "blobs" { rem = fields[i+1:] - found = true + foundBlobs = true + break + } + + if fields[i] == "compressed-blobs" { + rem = fields[i+1:] + foundCompressedBlobs = true break } } - if !found || len(rem) != 2 { + if foundBlobs { + if len(rem) != 2 { + msg := fmt.Sprintf("Unable to parse resource name: %s", name) + s.accessLogger.Printf("%s: %s", errorPrefix, msg) + return "", 0, casblob.Identity, + status.Error(codes.InvalidArgument, msg) + } + + hash := rem[0] + + size, err := strconv.ParseInt(rem[1], 10, 64) + if err != nil { + msg := fmt.Sprintf("Invalid size: %s", rem[1]) + s.accessLogger.Printf("%s: %s", errorPrefix, msg) + return "", 0, casblob.Identity, + status.Error(codes.InvalidArgument, msg) + } + if size < 0 { + msg := fmt.Sprintf("Invalid size (must be non-negative): %s", rem[1]) + s.accessLogger.Printf("%s: %s", errorPrefix, msg) + return "", 0, casblob.Identity, + status.Error(codes.InvalidArgument, msg) + } + + err = s.validateHash(hash, size, errorPrefix) + if err != nil { + return "", 0, casblob.Identity, err + } + + return hash, size, casblob.Identity, nil + } + + if !foundCompressedBlobs || len(rem) != 3 { msg := fmt.Sprintf("Unable to parse resource name: %s", name) s.accessLogger.Printf("%s: %s", errorPrefix, msg) - return "", 0, status.Error(codes.InvalidArgument, msg) + return "", 0, casblob.Identity, + status.Error(codes.InvalidArgument, msg) } - hash := rem[0] + if rem[0] != "zstd" { + msg := fmt.Sprintf("Unable to parse compressor in resource name: %s", name) + s.accessLogger.Printf("%s: %s", errorPrefix, msg) + return "", 0, casblob.Identity, + status.Error(codes.InvalidArgument, msg) + } - size, err := strconv.ParseInt(rem[1], 10, 64) + hash := rem[1] + sizeStr := rem[2] + + size, err := strconv.ParseInt(sizeStr, 10, 64) if err != nil { - msg := fmt.Sprintf("Invalid size: %s", rem[1]) + msg := fmt.Sprintf("Invalid size: %s", sizeStr) s.accessLogger.Printf("%s: %s", errorPrefix, msg) - return "", 0, status.Error(codes.InvalidArgument, msg) + return "", 0, casblob.Zstandard, + status.Error(codes.InvalidArgument, msg) } if size < 0 { msg := fmt.Sprintf("Invalid size (must be non-negative): %s", rem[1]) s.accessLogger.Printf("%s: %s", errorPrefix, msg) - return "", 0, status.Error(codes.InvalidArgument, msg) + return "", 0, casblob.Zstandard, + status.Error(codes.InvalidArgument, msg) } err = s.validateHash(hash, size, errorPrefix) if err != nil { - return "", 0, err + return "", 0, casblob.Zstandard, err } - return hash, size, nil + return hash, size, casblob.Zstandard, nil } // Seek to offset in Reader r, from the current position. @@ -187,14 +253,14 @@ func seekReaderTo(r io.Reader, offset int64) { } } -// Parse a WriteRequest.ResourceName, return the validated hash, size and an error. -func (s *grpcServer) parseWriteResource(r string) (string, int64, error) { +// Parse a WriteRequest.ResourceName, return the validated hash, size, +// compression type and an optional error. +func (s *grpcServer) parseWriteResource(r string) (string, int64, casblob.CompressionType, error) { // req.ResourceName is of the form: // [{instance_name}/]uploads/{uuid}/blobs/{hash}/{size}[/{optionalmetadata}] - - // We don't use the uuid so don't bother validating it. - // The path segment must exist but can be empty. + // Or, for compressed blobs: + // [{instance_name}/]uploads/{uuid}/compressed-blobs/{compressor}/{uncompressed_hash}/{uncompressed_size}[{/optional_metadata}] fields := strings.Split(r, "/") var rem []string @@ -205,26 +271,59 @@ func (s *grpcServer) parseWriteResource(r string) (string, int64, error) { } } - if len(rem) < 4 || rem[1] != "blobs" { - return "", 0, status.Errorf(codes.InvalidArgument, "Unable to parse resource name: %s", r) + if len(rem) < 4 { + return "", 0, casblob.Identity, + status.Errorf(codes.InvalidArgument, "Unable to parse resource name: %s", r) + } + + // rem[0] should hold the uuid, which we don't use- ignore it. + + if rem[1] == "blobs" { + hash := rem[2] + size, err := strconv.ParseInt(rem[3], 10, 64) + if err != nil { + return "", 0, casblob.Identity, + status.Errorf(codes.InvalidArgument, "Unable to parse size: %s", rem[3]) + } + + if size < 0 { + return "", 0, casblob.Identity, + status.Errorf(codes.InvalidArgument, "Invalid size (must be non-negative): %d", size) + } + + err = s.validateHash(hash, size, "GRPC BYTESTREAM READ FAILED") + if err != nil { + return "", 0, casblob.Identity, err + } + + return hash, size, casblob.Identity, nil + } + + if rem[1] != "compressed-blobs" || len(rem) < 5 || rem[2] != "zstd" { + return "", 0, casblob.Zstandard, + status.Errorf(codes.InvalidArgument, "Unable to parse resource name: %s", r) } - hash := rem[2] - size, err := strconv.ParseInt(rem[3], 10, 64) + sizeStr := rem[4] + + size, err := strconv.ParseInt(sizeStr, 10, 64) if err != nil { - return "", 0, status.Errorf(codes.InvalidArgument, "Unable to parse size: %s", rem[3]) + return "", 0, casblob.Zstandard, + status.Errorf(codes.InvalidArgument, "Unable to parse size: %s", sizeStr) } if size < 0 { - return "", 0, status.Errorf(codes.InvalidArgument, "Invalid size (must be non-negative): %s", rem[3]) + return "", 0, casblob.Zstandard, + status.Errorf(codes.InvalidArgument, "Invalid size (must be non-negative): %d", size) } + hash := rem[3] err = s.validateHash(hash, size, "GRPC BYTESTREAM READ FAILED") if err != nil { - return "", 0, err + return "", 0, casblob.Zstandard, err } - return hash, size, nil + return hash, size, casblob.Zstandard, nil } var errWriteOffset error = errors.New("bytestream writes from non-zero offsets are unsupported") @@ -238,6 +337,14 @@ func (s *grpcServer) Write(srv bytestream.ByteStream_WriteServer) error { recvResult := make(chan error) resourceNameChan := make(chan string, 1) + cmp := casblob.Identity + var dec *zstd.Decoder + defer func() { + if dec != nil { + dec.Close() + } + }() + go func() { firstIteration := true var resourceName string @@ -273,7 +380,7 @@ func (s *grpcServer) Write(srv bytestream.ByteStream_WriteServer) error { close(resourceNameChan) var hash string - hash, size, err = s.parseWriteResource(resourceName) + hash, size, cmp, err = s.parseWriteResource(resourceName) if err != nil { s.accessLogger.Printf("GRPC BYTESTREAM WRITE FAILED: %s", err) recvResult <- err @@ -296,8 +403,19 @@ func (s *grpcServer) Write(srv bytestream.ByteStream_WriteServer) error { return } + var rc io.ReadCloser = pr + if cmp == casblob.Zstandard { + dec, err = zstd.NewReader(pr) // TODO: use a pool. + if err != nil { + s.accessLogger.Printf("GRPC BYTESTREAM WRITE FAILED: %s", err) + recvResult <- err + return + } + rc = dec.IOReadCloser() + } + go func() { - putResult <- s.cache.Put(cache.CAS, hash, size, pr) + putResult <- s.cache.Put(cache.CAS, hash, size, rc) }() firstIteration = false @@ -317,7 +435,7 @@ func (s *grpcServer) Write(srv bytestream.ByteStream_WriteServer) error { } resp.CommittedSize += int64(n) - if resp.CommittedSize > size { + if cmp == casblob.Identity && resp.CommittedSize > size { msg := fmt.Sprintf("Client sent more than %d data! %d", size, resp.CommittedSize) recvResult <- status.Error(codes.OutOfRange, msg) return @@ -326,7 +444,7 @@ func (s *grpcServer) Write(srv bytestream.ByteStream_WriteServer) error { // Possibly redundant check, since we explicitly check for // EOF at the start of each loop. if req.FinishWrite { - if resp.CommittedSize != size { + if cmp == casblob.Identity && resp.CommittedSize != size { msg := fmt.Sprintf("Unexpected amount of data read: %d expected: %d", resp.CommittedSize, size) recvResult <- status.Error(codes.Unknown, msg) diff --git a/server/grpc_cas.go b/server/grpc_cas.go index b227b827d..8054ce9d9 100644 --- a/server/grpc_cas.go +++ b/server/grpc_cas.go @@ -100,7 +100,7 @@ func (s *grpcServer) getBlobData(hash string, size int64) ([]byte, error) { return []byte{}, nil } - rdr, sizeBytes, err := s.cache.Get(cache.CAS, hash, size) + rdr, sizeBytes, err := s.cache.Get(cache.CAS, hash, size, 0) if err != nil { rdr.Close() return []byte{}, err diff --git a/server/grpc_test.go b/server/grpc_test.go index 28b29b359..5c03ef4bc 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -26,7 +26,10 @@ import ( "github.com/buchgr/bazel-remote/cache" "github.com/buchgr/bazel-remote/cache/disk" + "github.com/buchgr/bazel-remote/cache/disk/casblob" "github.com/buchgr/bazel-remote/utils" + + "github.com/klauspost/compress/zstd" ) type badDigest struct { @@ -582,7 +585,7 @@ func TestGrpcByteStreamDeadline(t *testing.T) { t.Fatal(err) } - _, sz, err := diskCache.Get(cache.CAS, testBlobHash, testBlobSize) + _, sz, err := diskCache.Get(cache.CAS, testBlobHash, testBlobSize, 0) if err != nil { t.Fatalf("get error: %v\n", err) } @@ -740,6 +743,54 @@ func TestGrpcByteStream(t *testing.T) { t.Fatal("Error: bytestream read failed (data doesn't match)") } + // Read again, in zstd form this time. + + bsrReq = bytestream.ReadRequest{ + ResourceName: fmt.Sprintf("%s/compressed-blobs/zstd/%s/%d", + instance, testBlobDigest.Hash, len(testBlob)), + } + + bsrc, err = bsClient.Read(ctx, &bsrReq) + if err != nil { + t.Fatal(err) + } + + var decmpBuf bytes.Buffer + dr, dw := io.Pipe() + dec, err := zstd.NewReader(dr) + + go func() { + for { + bsrResp, err = bsrc.Recv() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if bsrResp == nil { + t.Fatalf("Expected non-nil response") + } + + dw.Write(bsrResp.Data) + + if len(downloadedBlob) > len(testBlob) { + t.Fatalf("Downloaded too much data") + } + } + + dw.Close() + }() + + _, err = io.Copy(&decmpBuf, dec) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(decmpBuf.Bytes(), testBlob) { + t.Fatal("Error: bytestream compressed read failed (data doesn't match)") + } + // Invalid Read's. for _, tc := range badDigestTestCases { @@ -776,6 +827,137 @@ func TestGrpcByteStream(t *testing.T) { } } +func TestGrpcByteStreamZstdWrite(t *testing.T) { + // Must be large enough to test multiple iterations of the + // bytestream Read Recv loop. + testBlobSize := int64(maxChunkSize * 3 / 2) + testBlob, testBlobHash := testutils.RandomDataAndHash(testBlobSize) + testBlobDigest := pb.Digest{ + Hash: testBlobHash, + SizeBytes: int64(len(testBlob)), + } + + enc, err := zstd.NewWriter(nil) + if err != nil { + t.Fatal(err) + } + compressedBlob := enc.EncodeAll(testBlob, nil) + enc.Close() + + bswc, err := bsClient.Write(ctx) + if err != nil { + t.Fatal(err) + } + + instance := "ignoredInstance" + + cutoff := len(compressedBlob) / 2 + blobPart := compressedBlob[:cutoff] + + bswReq := bytestream.WriteRequest{ + ResourceName: fmt.Sprintf("%s/uploads/%s/compressed-blobs/zstd/%s/%d", + instance, uuid.New().String(), testBlobDigest.Hash, len(testBlob)), + FinishWrite: false, + Data: blobPart, + } + + err = bswc.Send(&bswReq) + if err != nil { + t.Fatal(err) + } + + blobPart = compressedBlob[cutoff:] + bswReq.FinishWrite = true + bswReq.Data = blobPart + + err = bswc.Send(&bswReq) + if err != nil { + t.Fatal(err) + } + + bswResp, err := bswc.CloseAndRecv() + if err != nil { + t.Fatal(err) + } + if bswResp.CommittedSize != int64(len(compressedBlob)) { + t.Fatalf("Error: expected to write: %d but committed: %d\n", + len(testBlob), bswResp.CommittedSize) + } + + // Read back. + + bsrReq := bytestream.ReadRequest{ + ResourceName: fmt.Sprintf("%s/blobs/%s/%d", + instance, testBlobDigest.Hash, len(testBlob)), + } + + bsrc, err := bsClient.Read(ctx, &bsrReq) + if err != nil { + t.Fatal(err) + } + + downloadedBlob := make([]byte, 0, len(testBlob)) + + for { + bsrResp, err := bsrc.Recv() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if bsrResp == nil { + t.Fatalf("Expected non-nil response") + } + + downloadedBlob = append(downloadedBlob, bsrResp.Data...) + + if len(downloadedBlob) > len(testBlob) { + t.Fatalf("Downloaded too much data") + } + } + + if !bytes.Equal(downloadedBlob, testBlob) { + t.Fatal("Error: bytestream read failed (data doesn't match)") + } +} + +func TestGrpcByteStreamInvalidReadLimit(t *testing.T) { + testBlobSize := int64(maxChunkSize) + testBlob, testBlobHash := testutils.RandomDataAndHash(testBlobSize) + testBlobDigest := pb.Digest{ + Hash: testBlobHash, + SizeBytes: int64(len(testBlob)), + } + + // Check that non-zero ReadLimit for compressed-blobs returns + // InvalidArgument. + bsrReq := bytestream.ReadRequest{ + ResourceName: fmt.Sprintf("ignoredinstance/compressed-blobs/zstd/%s/%d", + testBlobDigest.Hash, len(testBlob)), + ReadLimit: 1024, + } + + bsrc, err := bsClient.Read(ctx, &bsrReq) + if err != nil { + t.Fatal(err) + } + + _, err = bsrc.Recv() + if err == nil || err == io.EOF { + t.Fatal("Expected error due to non-zero ReadLimit for compressed-blobs read") + } + + statusErr, ok := status.FromError(err) + if !ok { + t.Errorf("Expected a grpc status error, got something else: %v", err) + return + } + if statusErr.Code() != codes.InvalidArgument { + t.Fatal("Expected InvalidArgument response, got", err) + } +} + func TestGrpcByteStreamSkippedWrite(t *testing.T) { // Must be large enough to test multiple iterations of the @@ -1152,16 +1334,26 @@ func TestParseReadResource(t *testing.T) { unusedLogPrefix := "foo" tcs := []struct { - resourceName string - expectedHash string - expectedSize int64 - expectError bool + resourceName string + expectedHash string + expectedSize int64 + expectedCompression casblob.CompressionType + expectError bool }{ { // No instance specified. "blobs/0123456789012345678901234567890123456789012345678901234567890123/42", "0123456789012345678901234567890123456789012345678901234567890123", 42, + casblob.Identity, + false, + }, + { + // No instance specified. + "compressed-blobs/zstd/0123456789012345678901234567890123456789012345678901234567890123/42", + "0123456789012345678901234567890123456789012345678901234567890123", + 42, + casblob.Zstandard, false, }, { @@ -1169,6 +1361,15 @@ func TestParseReadResource(t *testing.T) { "foo/blobs/0123456789012345678901234567890123456789012345678901234567890123/42", "0123456789012345678901234567890123456789012345678901234567890123", 42, + casblob.Identity, + false, + }, + { + // Instance specified. + "foo/compressed-blobs/zstd/0123456789012345678901234567890123456789012345678901234567890123/42", + "0123456789012345678901234567890123456789012345678901234567890123", + 42, + casblob.Zstandard, false, }, { @@ -1176,10 +1377,19 @@ func TestParseReadResource(t *testing.T) { "foo/bar/blobs/0123456789012345678901234567890123456789012345678901234567890123/42", "0123456789012345678901234567890123456789012345678901234567890123", 42, + casblob.Identity, false, }, { - // Missing "/blobs/". + // Instance specified, containing '/'. + "foo/bar/compressed-blobs/zstd/0123456789012345678901234567890123456789012345678901234567890123/42", + "0123456789012345678901234567890123456789012345678901234567890123", + 42, + casblob.Zstandard, + false, + }, + { + // Missing "/blobs/" or "/compressed-blobs/". resourceName: "foo/bar/0123456789012345678901234567890123456789012345678901234567890123/42", expectError: true, }, @@ -1261,10 +1471,36 @@ func TestParseReadResource(t *testing.T) { resourceName: "foo/blobs//42", expectError: true, }, + + // Unsupported/unrecognised compression types. + { + resourceName: "pretenduuid/compressed-blobs/zstandard/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "pretenduuid/compressed-blobs/Zstd/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "pretenduuid/compressed-blobs/ZSTD/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "pretenduuid/compressed-blobs/identity/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "pretenduuid/compressed-blobs/Identity/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "pretenduuid/compressed-blobs/IDENTITY/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, } for _, tc := range tcs { - hash, size, err := s.parseReadResource(tc.resourceName, unusedLogPrefix) + hash, size, cmp, err := s.parseReadResource(tc.resourceName, unusedLogPrefix) if tc.expectError { if err == nil { @@ -1285,11 +1521,16 @@ func TestParseReadResource(t *testing.T) { if size != tc.expectedSize { t.Fatalf("Expected size: %d did not match actual size: %d in %q", tc.expectedSize, size, tc.resourceName) } + + if cmp != tc.expectedCompression { + t.Fatalf("Expected compressor: %d did not match actual compressor: %d in %q", tc.expectedCompression, cmp, tc.resourceName) + } } } func TestParseWriteResource(t *testing.T) { // Format: [{instance_name}/]uploads/{uuid}/blobs/{hash}/{size}[/{optionalmetadata}] + // Or: [{instance_name}/]uploads/{uuid}/compressed-blobs/{compressor}/{uncompressed_hash}/{uncompressed_size}[{/optional_metadata}] // We ignore instance_name and metadata, and we don't verify that the // uuid is valid- it just needs to exist (or be empty) and not contain '/'. @@ -1300,21 +1541,38 @@ func TestParseWriteResource(t *testing.T) { } tcs := []struct { - resourceName string - expectedHash string - expectedSize int64 - expectError bool + resourceName string + expectedHash string + expectedSize int64 + expectedCompression casblob.CompressionType + expectError bool }{ { "foo/uploads/pretenduuid/blobs/0123456789012345678901234567890123456789012345678901234567890123/42", "0123456789012345678901234567890123456789012345678901234567890123", 42, + casblob.Identity, + false, + }, + { + "foo/uploads/pretenduuid/compressed-blobs/zstd/0123456789012345678901234567890123456789012345678901234567890123/42", + "0123456789012345678901234567890123456789012345678901234567890123", + 42, + casblob.Zstandard, false, }, { "uploads/pretenduuid/blobs/0123456789012345678901234567890123456789012345678901234567890123/42", "0123456789012345678901234567890123456789012345678901234567890123", 42, + casblob.Identity, + false, + }, + { + "uploads/pretenduuid/compressed-blobs/zstd/0123456789012345678901234567890123456789012345678901234567890123/42", + "0123456789012345678901234567890123456789012345678901234567890123", + 42, + casblob.Zstandard, false, }, { @@ -1322,12 +1580,29 @@ func TestParseWriteResource(t *testing.T) { "uploads/pretenduuid/blobs/0123456789012345678901234567890123456789012345678901234567890123/9223372036854775807", "0123456789012345678901234567890123456789012345678901234567890123", 9223372036854775807, + casblob.Identity, + false, + }, + { + // max(int64) + "uploads/pretenduuid/compressed-blobs/zstd/0123456789012345678901234567890123456789012345678901234567890123/9223372036854775807", + "0123456789012345678901234567890123456789012345678901234567890123", + 9223372036854775807, + casblob.Zstandard, false, }, { "foo/uploads/pretenduuid/blobs/0123456789012345678901234567890123456789012345678901234567890123/42/some/meta/data", "0123456789012345678901234567890123456789012345678901234567890123", 42, + casblob.Identity, + false, + }, + { + "foo/uploads/pretenduuid/compressed-blobs/zstd/0123456789012345678901234567890123456789012345678901234567890123/42/some/meta/data", + "0123456789012345678901234567890123456789012345678901234567890123", + 42, + casblob.Zstandard, false, }, @@ -1396,10 +1671,36 @@ func TestParseWriteResource(t *testing.T) { resourceName: "foo/blobs/0123456789012345678901234567890123456789012345678901234567890123/9223372036854775808", expectError: true, }, + + // Unsupported/unrecognised compression types. + { + resourceName: "uploads/pretenduuid/compressed-blobs/zstandard/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "uploads/pretenduuid/compressed-blobs/Zstd/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "uploads/pretenduuid/compressed-blobs/ZSTD/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "uploads/pretenduuid/compressed-blobs/identity/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "uploads/pretenduuid/compressed-blobs/Identity/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, + { + resourceName: "uploads/pretenduuid/compressed-blobs/IDENTITY/0123456789012345678901234567890123456789012345678901234567890123/42", + expectError: true, + }, } for _, tc := range tcs { - hash, size, err := s.parseWriteResource(tc.resourceName) + hash, size, cmp, err := s.parseWriteResource(tc.resourceName) if tc.expectError { if err == nil { @@ -1420,5 +1721,9 @@ func TestParseWriteResource(t *testing.T) { if size != tc.expectedSize { t.Fatalf("Expected size: %d did not match actual size: %d in %q", tc.expectedSize, size, tc.resourceName) } + + if cmp != tc.expectedCompression { + t.Fatalf("Expected compression: %d did not match actual compression: %d in %q", tc.expectedCompression, cmp, tc.resourceName) + } } } diff --git a/server/http.go b/server/http.go index 7176ca666..7bfb3d511 100644 --- a/server/http.go +++ b/server/http.go @@ -40,13 +40,14 @@ type httpCache struct { } type statusPageData struct { - CurrSize int64 - ReservedSize int64 - MaxSize int64 - NumFiles int - ServerTime int64 - GitCommit string - NumGoroutines int + CurrSize int64 + UncompressedSize int64 + ReservedSize int64 + MaxSize int64 + NumFiles int + ServerTime int64 + GitCommit string + NumGoroutines int } // NewHTTPCache returns a new instance of the cache. @@ -55,7 +56,7 @@ type statusPageData struct { // be reported. func NewHTTPCache(cache *disk.Cache, accessLogger cache.Logger, errorLogger cache.Logger, validateAC bool, mangleACKeys bool, commit string) HTTPCache { - _, _, numItems := cache.Stats() + _, _, numItems, _ := cache.Stats() errorLogger.Printf("Loaded %d existing disk cache items.", numItems) @@ -205,7 +206,7 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { return } - rdr, sizeBytes, err := h.cache.Get(kind, hash, -1) + rdr, sizeBytes, err := h.cache.Get(kind, hash, -1, 0) if err != nil { if e, ok := err.(*cache.Error); ok { http.Error(w, e.Error(), e.Code) @@ -360,7 +361,7 @@ func addWorkerMetadataHTTP(addr string, ct string, orig []byte) (data []byte, co func (h *httpCache) StatusPageHandler(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() - totalSize, reservedSize, numItems := h.cache.Stats() + totalSize, reservedSize, numItems, uncompressedSize := h.cache.Stats() goroutines := runtime.NumGoroutine() @@ -368,13 +369,14 @@ func (h *httpCache) StatusPageHandler(w http.ResponseWriter, r *http.Request) { enc := json.NewEncoder(w) enc.SetIndent("", " ") enc.Encode(statusPageData{ - MaxSize: h.cache.MaxSize(), - CurrSize: totalSize, - ReservedSize: reservedSize, - NumFiles: numItems, - ServerTime: time.Now().Unix(), - GitCommit: h.gitCommit, - NumGoroutines: goroutines, + MaxSize: h.cache.MaxSize(), + CurrSize: totalSize, + UncompressedSize: uncompressedSize, + ReservedSize: reservedSize, + NumFiles: numItems, + ServerTime: time.Now().Unix(), + GitCommit: h.gitCommit, + NumGoroutines: goroutines, }) } From e1d3c4dd91f33eafaffada1df08abbf1c80d5151 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Tue, 8 Dec 2020 00:37:41 +0100 Subject: [PATCH 05/35] [http] allow zstd-encoded CAS GETs --- server/http.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/server/http.go b/server/http.go index 7bfb3d511..62f788c70 100644 --- a/server/http.go +++ b/server/http.go @@ -206,7 +206,16 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { return } - rdr, sizeBytes, err := h.cache.Get(kind, hash, -1, 0) + var rdr io.ReadCloser + var sizeBytes int64 + + zstdCompressed := false + if kind == cache.CAS && strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") { + rdr, sizeBytes, err = h.cache.GetZstd(hash, -1, 0) + zstdCompressed = true + } else { + rdr, sizeBytes, err = h.cache.Get(kind, hash, -1, 0) + } if err != nil { if e, ok := err.(*cache.Error); ok { http.Error(w, e.Error(), e.Code) @@ -225,7 +234,14 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { defer rdr.Close() w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10)) + if zstdCompressed { + // TODO: calculate Content-Length for compressed blobs too + // (unless compressing on the fly). + w.Header().Set("Content-Encoding", "zstd") + } else { + w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10)) + } + io.Copy(w, rdr) h.logResponse(http.StatusOK, r) From ba069f85623cac4efbdf4f7cc8eb6561731de8a6 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Wed, 9 Dec 2020 00:14:54 +0100 Subject: [PATCH 06/35] [http] accept zstd-encoded PUTs To use this feature, clients must specify the custom X-Digest-SizeBytes header with the size of the blob when uncompressed. --- server/http.go | 70 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/server/http.go b/server/http.go index 62f788c70..cdb717e02 100644 --- a/server/http.go +++ b/server/http.go @@ -18,12 +18,15 @@ import ( "github.com/buchgr/bazel-remote/cache" "github.com/buchgr/bazel-remote/cache/disk" pb "github.com/buchgr/bazel-remote/genproto/build/bazel/remote/execution/v2" + "github.com/klauspost/compress/zstd" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" ) var blobNameSHA256 = regexp.MustCompile("^/?(.*/)?(ac/|cas/)([a-f0-9]{64})$") +var decoder, _ = zstd.NewReader(nil) + // HTTPCache ... type HTTPCache interface { CacheHandler(w http.ResponseWriter, r *http.Request) @@ -249,6 +252,22 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { case http.MethodPut: contentLength := r.ContentLength + // If custom header X-Digest-SizeBytes is set, use that for the + // size of the blob instead of Content-Length (which only works + // for uncompressed PUTs). + sb := r.Header.Get("X-Digest-SizeBytes") + if sb != "" { + cl, err := strconv.Atoi(sb) + if err != nil { + msg := fmt.Sprintf("PUT with unparseable X-Digest-SizeBytes header: %v", sb) + http.Error(w, msg, http.StatusBadRequest) + h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg) + return + } + + contentLength = int64(cl) + } + if contentLength == -1 { // We need the content-length header to make sure we have enough disk space. msg := fmt.Sprintf("PUT without Content-Length (key = %s)", path(kind, hash)) @@ -264,11 +283,24 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { return } - rc := r.Body + zstdCompressed := false + + // Content-Encoding must be one of "identity", "zstd" or not present. + ce := r.Header.Get("Content-Encoding") + if ce == "zstd" { + zstdCompressed = true + } else if ce != "" && ce != "identity" { + msg := fmt.Sprintf("Unsupported content-encoding: %q", ce) + http.Error(w, msg, http.StatusBadRequest) + h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg) + return + } + + var rdr io.Reader = r.Body if h.validateAC && kind == cache.AC { // verify that this is a valid ActionResult - data, err := ioutil.ReadAll(rc) + data, err := ioutil.ReadAll(rdr) if err != nil { msg := "failed to read request body" http.Error(w, msg, http.StatusInternalServerError) @@ -276,6 +308,19 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { return } + if zstdCompressed { + uncompressed, err := decoder.DecodeAll(data, nil) + if err != nil { + msg := fmt.Sprintf("failed to uncompress zstd-encoded request body: %v", err) + http.Error(w, msg, http.StatusBadRequest) + h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg) + return + } + + data = uncompressed + zstdCompressed = false + } + if int64(len(data)) != contentLength { msg := fmt.Sprintf("sizes don't match. Expected %d, found %d", contentLength, len(data)) @@ -296,10 +341,27 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { // Note: we do not currently verify that the blobs exist // in the CAS. - rc = ioutil.NopCloser(bytes.NewReader(data)) + rdr = bytes.NewReader(data) + } + + if zstdCompressed { + z, err := zstd.NewReader(rdr) // TODO: use a pool. + if err != nil { + if z != nil { + defer z.Close() + } + msg := fmt.Sprintf("Failed to create zstd reader: %v", err) + http.Error(w, msg, http.StatusInternalServerError) + h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg) + return + } + + rc := z.IOReadCloser() + defer rc.Close() + rdr = rc } - err := h.cache.Put(kind, hash, contentLength, rc) + err := h.cache.Put(kind, hash, contentLength, rdr) if err != nil { if cerr, ok := err.(*cache.Error); ok { http.Error(w, err.Error(), cerr.Code) From 5eac267c0a85034a04a3a8c38fca84fbe713d161 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Thu, 31 Dec 2020 12:10:39 +0100 Subject: [PATCH 07/35] [http] document how to upload/download zstd-compressed data --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 17605c093..dee7673a5 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ commodity hardware and AWS servers. Outgoing bandwidth can exceed 15 Gbit/s on t ## HTTP/1.1 REST API Cache entries are set and retrieved by key, and there are two types of keys that can be used: -1. Content addressed storage (CAS), where the key is the lowercase SHA256 hash of the stored value. +1. Content addressed storage (CAS), where the key is the lowercase SHA256 hash of the entry. The REST API for these entries is: `/cas/` or with an optional but ignored instance name: `//cas/`. 2. Action cache, where the key is an arbitrary 64 character lowercase hexadecimal string. @@ -24,6 +24,14 @@ Cache entries are set and retrieved by key, and there are two types of keys that Values are stored via HTTP PUT requests, and retrieved via GET requests. HEAD requests can be used to confirm whether a key exists or not. +If GET requests specify `zstd` in the `Accept-Encoding` header, then +zstandard-encoded data may be returned. + +To upload zstandard compressed data, PUT requests must set +`Content-Encoding: zstd` and include a custom `X-Digest-SizeBytes` header +with the size of the uncompressed entry. The key must also refer to +the uncompressed entry. + If the `--enable_ac_key_instance_mangling` flag is specified and the instance name is not empty, then action cache keys are hashed along with the instance name to produce the action cache lookup key. Since the URL path is processed From 997695aaba4d8c252a5494ccd446d65f81ddbbe3 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Mon, 21 Dec 2020 00:32:01 +0100 Subject: [PATCH 08/35] [zstd] limit concurrency to 1 for single-use {en,de}coders --- cache/disk/casblob/casblob.go | 16 +++++++++------- server/grpc_bytestream.go | 2 +- server/grpc_test.go | 4 ++-- server/http.go | 6 ++++-- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index c301a00b7..f94fc0d31 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -22,11 +22,13 @@ const ( Zstandard CompressionType = 1 ) -var zstdLevelOpt = zstd.WithEncoderLevel(zstd.SpeedFastest) -var encoder, _ = zstd.NewWriter(nil, zstdLevelOpt) -var decoder, _ = zstd.NewReader(nil) +var zstdFastestLevel = zstd.WithEncoderLevel(zstd.SpeedFastest) -const defaultChunkSize = 1024 * 1024 * 1 // 1M +var encoder, _ = zstd.NewWriter(nil, zstdFastestLevel) // TODO: raise WithEncoderConcurrency ? +var decoder, _ = zstd.NewReader(nil) // TODO: raise WithDecoderConcurrency ? + +var singleDecoder = zstd.WithDecoderConcurrency(1) +var singleEncoder = zstd.WithEncoderConcurrency(1) const defaultChunkSize = 1024 * 1024 * 1 // 1M @@ -185,7 +187,7 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io f.Seek(h.chunkOffsets[chunkNum], io.SeekStart) } if remainder == 0 { - z, err := zstd.NewReader(f) // TODO: use a pool, or a chunk decoder. + z, err := zstd.NewReader(f, singleDecoder) // TODO: use a pool, or a chunk decoder. if err != nil { f.Close() return nil, err @@ -219,7 +221,7 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io return ioutil.NopCloser(r), nil } - z, err := zstd.NewReader(f) // TODO: use a pool. + z, err := zstd.NewReader(f, singleDecoder) // TODO: use a pool. if err != nil { f.Close() return nil, err @@ -265,7 +267,7 @@ func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadClo pr, pw := io.Pipe() // TODO: use a pool - enc, err := zstd.NewWriter(pw, zstd.WithEncoderLevel(zstd.SpeedFastest)) + enc, err := zstd.NewWriter(pw, zstdFastestLevel, singleEncoder) if err != nil { f.Close() return nil, err diff --git a/server/grpc_bytestream.go b/server/grpc_bytestream.go index d34797813..e0a3bef22 100644 --- a/server/grpc_bytestream.go +++ b/server/grpc_bytestream.go @@ -405,7 +405,7 @@ func (s *grpcServer) Write(srv bytestream.ByteStream_WriteServer) error { var rc io.ReadCloser = pr if cmp == casblob.Zstandard { - dec, err = zstd.NewReader(pr) // TODO: use a pool. + dec, err = zstd.NewReader(pr, singleDecoder) // TODO: use a pool. if err != nil { s.accessLogger.Printf("GRPC BYTESTREAM WRITE FAILED: %s", err) recvResult <- err diff --git a/server/grpc_test.go b/server/grpc_test.go index 5c03ef4bc..5ecfc198a 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -757,7 +757,7 @@ func TestGrpcByteStream(t *testing.T) { var decmpBuf bytes.Buffer dr, dw := io.Pipe() - dec, err := zstd.NewReader(dr) + dec, err := zstd.NewReader(dr, zstd.WithDecoderConcurrency(1)) go func() { for { @@ -837,7 +837,7 @@ func TestGrpcByteStreamZstdWrite(t *testing.T) { SizeBytes: int64(len(testBlob)), } - enc, err := zstd.NewWriter(nil) + enc, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1)) if err != nil { t.Fatal(err) } diff --git a/server/http.go b/server/http.go index cdb717e02..24c3b0006 100644 --- a/server/http.go +++ b/server/http.go @@ -25,7 +25,9 @@ import ( var blobNameSHA256 = regexp.MustCompile("^/?(.*/)?(ac/|cas/)([a-f0-9]{64})$") -var decoder, _ = zstd.NewReader(nil) +var decoder, _ = zstd.NewReader(nil) // TODO: raise WithDecoderConcurrency ? + +var singleDecoder = zstd.WithDecoderConcurrency(1) // HTTPCache ... type HTTPCache interface { @@ -345,7 +347,7 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { } if zstdCompressed { - z, err := zstd.NewReader(rdr) // TODO: use a pool. + z, err := zstd.NewReader(rdr, singleDecoder) // TODO: use a pool. if err != nil { if z != nil { defer z.Close() From fe70a245331d5db055b36c7773bbaf7b67c38327 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Tue, 22 Dec 2020 15:10:08 +0100 Subject: [PATCH 09/35] [zstd] use decoder and encoder pools --- WORKSPACE | 11 ++++++++-- cache/disk/casblob/BUILD.bazel | 6 ++++- cache/disk/casblob/casblob.go | 40 +++++++++++++++++++++++++--------- go.mod | 3 ++- go.sum | 4 ++++ server/BUILD.bazel | 2 ++ server/grpc_bytestream.go | 14 +++++++++++- server/http.go | 11 ++++++++-- utils/zstdpool/BUILD.bazel | 12 ++++++++++ utils/zstdpool/zstdpool.go | 33 ++++++++++++++++++++++++++++ 10 files changed, 119 insertions(+), 17 deletions(-) create mode 100644 utils/zstdpool/BUILD.bazel create mode 100644 utils/zstdpool/zstdpool.go diff --git a/WORKSPACE b/WORKSPACE index b0e768273..00c79ccea 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -969,8 +969,8 @@ go_repository( go_repository( name = "com_github_klauspost_compress", importpath = "github.com/klauspost/compress", - sum = "h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=", - version = "v1.11.3", + sum = "h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU=", + version = "v1.11.4", ) go_repository( @@ -1070,3 +1070,10 @@ go_repository( sum = "h1:y7Vn4YH/rfUHOCwNhvkAcA0gMQvFdKzSE8Ri3qtcFlc=", version = "v1.10.1", ) + +go_repository( + name = "com_github_mostynb_zstdpool_syncpool", + importpath = "github.com/mostynb/zstdpool-syncpool", + sum = "h1:tSmtZSiSAfffEgy9ziKvOZOOlccUwF2Ar78+CeOH+8A=", + version = "v0.0.2", +) diff --git a/cache/disk/casblob/BUILD.bazel b/cache/disk/casblob/BUILD.bazel index 567123310..d181a70b9 100644 --- a/cache/disk/casblob/BUILD.bazel +++ b/cache/disk/casblob/BUILD.bazel @@ -5,5 +5,9 @@ go_library( srcs = ["casblob.go"], importpath = "github.com/buchgr/bazel-remote/cache/disk/casblob", visibility = ["//visibility:public"], - deps = ["@com_github_klauspost_compress//zstd:go_default_library"], + deps = [ + "//utils/zstdpool:go_default_library", + "@com_github_klauspost_compress//zstd:go_default_library", + "@com_github_mostynb_zstdpool_syncpool//:go_default_library", + ], ) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index f94fc0d31..2c1421d99 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -12,7 +12,9 @@ import ( "log" "os" + "github.com/buchgr/bazel-remote/utils/zstdpool" "github.com/klauspost/compress/zstd" + syncpool "github.com/mostynb/zstdpool-syncpool" ) type CompressionType uint8 @@ -25,10 +27,13 @@ const ( var zstdFastestLevel = zstd.WithEncoderLevel(zstd.SpeedFastest) var encoder, _ = zstd.NewWriter(nil, zstdFastestLevel) // TODO: raise WithEncoderConcurrency ? -var decoder, _ = zstd.NewReader(nil) // TODO: raise WithDecoderConcurrency ? +var decoder, _ = zstd.NewReader(nil) // TODO: raise WithDecoderConcurrency ? -var singleDecoder = zstd.WithDecoderConcurrency(1) -var singleEncoder = zstd.WithEncoderConcurrency(1) +var encoderPool = zstdpool.GetEncoderPool() +var decoderPool = zstdpool.GetDecoderPool() + +var errDecoderPoolFail error = errors.New("failed to get DecoderWrapper from pool") +var errEncoderPoolFail error = errors.New("failed to get EncoderWrapper from pool") const defaultChunkSize = 1024 * 1024 * 1 // 1M @@ -43,7 +48,7 @@ type header struct { // Offsets in the file on disk, of each chunk, with an additional // entry for the end of the file. // - // Stored as an int64 number of chunks, followed by their int64 offsets, + // Stored as an int64 number of chunks, followed by their int64 offsets, // and a final value for the size of the file (header + data). // 8 bytes + (n+1)*8 bytes. chunkOffsets []int64 @@ -62,7 +67,7 @@ type readCloserWrapper struct { rdr io.Reader // Read from this, not from decoder or file. - decoder *zstd.Decoder // Might be nil. + decoder *syncpool.DecoderWrapper // Might be nil. file *os.File } @@ -187,7 +192,12 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io f.Seek(h.chunkOffsets[chunkNum], io.SeekStart) } if remainder == 0 { - z, err := zstd.NewReader(f, singleDecoder) // TODO: use a pool, or a chunk decoder. + z, ok := decoderPool.Get().(*syncpool.DecoderWrapper) + if !ok { + f.Close() + return nil, err + } + err := z.Reset(f) if err != nil { f.Close() return nil, err @@ -221,10 +231,17 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io return ioutil.NopCloser(r), nil } - z, err := zstd.NewReader(f, singleDecoder) // TODO: use a pool. + z, ok := decoderPool.Get().(*syncpool.DecoderWrapper) + if !ok { + f.Close() + return nil, errDecoderPoolFail + } + + err = z.Reset(f) if err != nil { f.Close() - return nil, err + decoderPool.Put(z) + return nil, fmt.Errorf("Failed to setup zstd decoder: %w", err) } br := bytes.NewReader(uncompressedFirstChunk[remainder:]) @@ -266,12 +283,15 @@ func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadClo pr, pw := io.Pipe() - // TODO: use a pool - enc, err := zstd.NewWriter(pw, zstdFastestLevel, singleEncoder) + enc, ok := encoderPool.Get().(*syncpool.EncoderWrapper) + if !ok { + err = errEncoderPoolFail + } if err != nil { f.Close() return nil, err } + enc.Reset(pw) go func() { // Read from the file, write to enc. diff --git a/go.mod b/go.mod index aa14a25a2..f24a6c457 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,11 @@ require ( github.com/google/go-cmp v0.5.0 github.com/google/uuid v1.1.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/klauspost/compress v1.11.3 + github.com/klauspost/compress v1.11.4 github.com/klauspost/cpuid v1.3.1 // indirect github.com/minio/minio-go/v7 v7.0.1 github.com/mostynb/go-grpc-compression v1.1.4 + github.com/mostynb/zstdpool-syncpool v0.0.2 github.com/prometheus/client_golang v1.7.1 github.com/slok/go-http-metrics v0.8.0 github.com/smartystreets/goconvey v1.6.4 // indirect diff --git a/go.sum b/go.sum index 8c19f8d04..2ec89ed88 100644 --- a/go.sum +++ b/go.sum @@ -155,6 +155,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -193,6 +195,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mostynb/go-grpc-compression v1.1.4 h1:wlscKqxoQsZQxUi6uTER+Gh9MbcUFUa4Cw5A69pj/c8= github.com/mostynb/go-grpc-compression v1.1.4/go.mod h1:rUYnd4NcUyZqHgjjISqjBMa6qTxMFjFu5+fHLI3k3Ho= +github.com/mostynb/zstdpool-syncpool v0.0.2 h1:tSmtZSiSAfffEgy9ziKvOZOOlccUwF2Ar78+CeOH+8A= +github.com/mostynb/zstdpool-syncpool v0.0.2/go.mod h1:j/I43b31x+jxmpN8j/o+jc9SdLDoTiDei95IvdCLWgA= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/server/BUILD.bazel b/server/BUILD.bazel index f23ccd746..b27555f1c 100644 --- a/server/BUILD.bazel +++ b/server/BUILD.bazel @@ -22,12 +22,14 @@ go_library( "//genproto/build/bazel/remote/execution/v2:go_default_library", "//genproto/build/bazel/semver:go_default_library", "//utils/idle:go_default_library", + "//utils/zstdpool:go_default_library", "@com_github_abbot_go_http_auth//:go_default_library", "@com_github_golang_protobuf//jsonpb:go_default_library_gen", "@com_github_golang_protobuf//proto:go_default_library", "@com_github_klauspost_compress//zstd:go_default_library", "@com_github_mostynb_go_grpc_compression//snappy:go_default_library", "@com_github_mostynb_go_grpc_compression//zstd:go_default_library", + "@com_github_mostynb_zstdpool_syncpool//:go_default_library", "@go_googleapis//google/bytestream:bytestream_go_proto", "@go_googleapis//google/rpc:code_go_proto", "@go_googleapis//google/rpc:status_go_proto", diff --git a/server/grpc_bytestream.go b/server/grpc_bytestream.go index e0a3bef22..6039ca2ba 100644 --- a/server/grpc_bytestream.go +++ b/server/grpc_bytestream.go @@ -17,6 +17,9 @@ import ( "github.com/buchgr/bazel-remote/cache/disk/casblob" "github.com/klauspost/compress/zstd" + + "github.com/buchgr/bazel-remote/utils/zstdpool" + syncpool "github.com/mostynb/zstdpool-syncpool" ) const ( @@ -25,6 +28,8 @@ const ( maxChunkSize = 2 * 1024 * 1024 // 2M ) +var decoderPool = zstdpool.GetDecoderPool() + // ByteStreamServer interface: func (s *grpcServer) Read(req *bytestream.ReadRequest, @@ -327,6 +332,7 @@ func (s *grpcServer) parseWriteResource(r string) (string, int64, casblob.Compre } var errWriteOffset error = errors.New("bytestream writes from non-zero offsets are unsupported") +var errDecoderPoolFail error = errors.New("failed to get DecoderWrapper from pool") func (s *grpcServer) Write(srv bytestream.ByteStream_WriteServer) error { @@ -405,7 +411,13 @@ func (s *grpcServer) Write(srv bytestream.ByteStream_WriteServer) error { var rc io.ReadCloser = pr if cmp == casblob.Zstandard { - dec, err = zstd.NewReader(pr, singleDecoder) // TODO: use a pool. + dec, ok := decoderPool.Get().(*syncpool.DecoderWrapper) + if !ok { + s.accessLogger.Printf("GRPC BYTESTREAM WRITE FAILED: %s", errDecoderPoolFail) + recvResult <- errDecoderPoolFail + return + } + err = dec.Reset(pr) if err != nil { s.accessLogger.Printf("GRPC BYTESTREAM WRITE FAILED: %s", err) recvResult <- err diff --git a/server/http.go b/server/http.go index 24c3b0006..fdf248a16 100644 --- a/server/http.go +++ b/server/http.go @@ -18,9 +18,11 @@ import ( "github.com/buchgr/bazel-remote/cache" "github.com/buchgr/bazel-remote/cache/disk" pb "github.com/buchgr/bazel-remote/genproto/build/bazel/remote/execution/v2" - "github.com/klauspost/compress/zstd" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" + "github.com/klauspost/compress/zstd" + + syncpool "github.com/mostynb/zstdpool-syncpool" ) var blobNameSHA256 = regexp.MustCompile("^/?(.*/)?(ac/|cas/)([a-f0-9]{64})$") @@ -347,7 +349,12 @@ func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) { } if zstdCompressed { - z, err := zstd.NewReader(rdr, singleDecoder) // TODO: use a pool. + z, ok := decoderPool.Get().(*syncpool.DecoderWrapper) + if !ok { + err = errDecoderPoolFail + } else { + err = z.Reset(rdr) + } if err != nil { if z != nil { defer z.Close() diff --git a/utils/zstdpool/BUILD.bazel b/utils/zstdpool/BUILD.bazel new file mode 100644 index 000000000..ee576a2f8 --- /dev/null +++ b/utils/zstdpool/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["zstdpool.go"], + importpath = "github.com/buchgr/bazel-remote/utils/zstdpool", + visibility = ["//visibility:public"], + deps = [ + "@com_github_klauspost_compress//zstd:go_default_library", + "@com_github_mostynb_zstdpool_syncpool//:go_default_library", + ], +) diff --git a/utils/zstdpool/zstdpool.go b/utils/zstdpool/zstdpool.go new file mode 100644 index 000000000..15cdea68f --- /dev/null +++ b/utils/zstdpool/zstdpool.go @@ -0,0 +1,33 @@ +package zstdpool + +import ( + "sync" + + "github.com/klauspost/compress/zstd" + syncpool "github.com/mostynb/zstdpool-syncpool" +) + +var onceEncPool sync.Once +var encoderPool *sync.Pool + +var onceDecPool sync.Once +var decoderPool *sync.Pool + +func GetEncoderPool() *sync.Pool { + onceEncPool.Do(func() { + encoderPool = syncpool.NewEncoderPool( + zstd.WithEncoderConcurrency(1), + zstd.WithEncoderLevel(zstd.SpeedFastest)) + }) + + return encoderPool +} + +func GetDecoderPool() *sync.Pool { + onceDecPool.Do(func() { + decoderPool = syncpool.NewDecoderPool( + zstd.WithDecoderConcurrency(1)) + }) + + return decoderPool +} From 0accc93a3ecbaf8f7778e0df000232c0afce7ca1 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Fri, 25 Dec 2020 21:48:24 +0100 Subject: [PATCH 10/35] [bytestream] be more careful about calling Close on the result of Get calls --- server/grpc_bytestream.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/grpc_bytestream.go b/server/grpc_bytestream.go index 6039ca2ba..5e58f3d7d 100644 --- a/server/grpc_bytestream.go +++ b/server/grpc_bytestream.go @@ -74,13 +74,17 @@ func (s *grpcServer) Read(req *bytestream.ReadRequest, return status.Error(codes.OutOfRange, msg) } - var rdr io.ReadCloser + var rc io.ReadCloser var foundSize int64 if cmp == casblob.Zstandard { - rdr, foundSize, err = s.cache.GetZstd(hash, size, req.ReadOffset) + rc, foundSize, err = s.cache.GetZstd(hash, size, req.ReadOffset) } else { - rdr, foundSize, err = s.cache.Get(cache.CAS, hash, size, req.ReadOffset) + rc, foundSize, err = s.cache.Get(cache.CAS, hash, size, req.ReadOffset) + } + + if rc != nil { + defer rc.Close() } if err != nil { @@ -88,12 +92,11 @@ func (s *grpcServer) Read(req *bytestream.ReadRequest, s.accessLogger.Printf(msg) return status.Error(codes.Unknown, msg) } - if rdr == nil { + if rc == nil { msg := fmt.Sprintf("GRPC BYTESTREAM READ BLOB NOT FOUND: %s", hash) s.accessLogger.Printf(msg) return status.Error(codes.NotFound, msg) } - defer rdr.Close() if foundSize != size { // This should have been caught above. @@ -112,7 +115,7 @@ func (s *grpcServer) Read(req *bytestream.ReadRequest, var chunkResp bytestream.ReadResponse for { - n, err := rdr.Read(buf) + n, err := rc.Read(buf) if n > 0 { if limitedSend { From 6cc58ac206e894bf1b7acb4d049677357814ad58 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Thu, 31 Dec 2020 12:08:10 +0100 Subject: [PATCH 11/35] [disk] make the CAS storage mode (zstd/uncompressed) configurable The default is to use zstd, but some installs might choose to use uncompressed data (for example if clients mostly upload blobs which are already compressed). --- README.md | 4 ++ cache/disk/disk.go | 25 +++++--- cache/disk/disk_test.go | 30 +++++----- cache/httpproxy/httpproxy_test.go | 4 +- config/config.go | 10 +++- config/config_test.go | 96 ++++++++++++++++++++++++++----- main.go | 9 ++- server/grpc_test.go | 2 +- server/http_test.go | 16 +++--- 9 files changed, 145 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index dee7673a5..1e310805c 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ GLOBAL OPTIONS: --config_file value Path to a YAML configuration file. If this flag is specified then all other flags are ignored. [$BAZEL_REMOTE_CONFIG_FILE] --dir value Directory path where to store the cache contents. This flag is required. [$BAZEL_REMOTE_DIR] --max_size value The maximum size of the remote cache in GiB. This flag is required. (default: -1) [$BAZEL_REMOTE_MAX_SIZE] + --storage_mode value Which format to store CAS blobs in. Must be one of "zstd" or "uncompressed". (default: "zstd") [$BAZEL_REMOTE_STORAGE_MODE] --host value Address to listen on. Listens on all network interfaces by default. [$BAZEL_REMOTE_HOST] --port value The port the HTTP server listens on. (default: 8080) [$BAZEL_REMOTE_PORT] --grpc_port value The port the gRPC server listens on. Set to 0 to disable. (default: 9092) [$BAZEL_REMOTE_GRPC_PORT] @@ -165,6 +166,9 @@ GLOBAL OPTIONS: dir: path/to/cache-dir max_size: 100 +# The form to store CAS blobs in ("zstd" or "uncompressed"): +#storage_mode: zstd + host: localhost # The port to use for HTTP/HTTPS: #port: 8080 diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 4908a2768..f06893efd 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -56,8 +56,9 @@ type lruItem struct { // Cache is a filesystem-based LRU cache, with an optional backend proxy. type Cache struct { - dir string - proxy cache.Proxy + dir string + proxy cache.Proxy + storageMode casblob.CompressionType mu sync.Mutex lru SizedLRU @@ -74,7 +75,7 @@ const emptySha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b785 // New returns a new instance of a filesystem-based cache rooted at `dir`, // with a maximum size of `maxSizeBytes` bytes and an optional backend `proxy`. // Cache is safe for concurrent use. -func New(dir string, maxSizeBytes int64, proxy cache.Proxy) (*Cache, error) { +func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) (*Cache, error) { // Create the directory structure. hexLetters := []byte("0123456789abcdef") for _, c1 := range hexLetters { @@ -118,10 +119,16 @@ func New(dir string, maxSizeBytes int64, proxy cache.Proxy) (*Cache, error) { } } + compressionType := casblob.Zstandard + if storageMode == "uncompressed" { + compressionType = casblob.Identity + } + c := &Cache{ - dir: filepath.Clean(dir), - proxy: proxy, - lru: NewSizedLRU(maxSizeBytes, onEvict), + dir: filepath.Clean(dir), + storageMode: compressionType, + proxy: proxy, + lru: NewSizedLRU(maxSizeBytes, onEvict), } err := c.migrateDirectories() @@ -458,7 +465,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) removeTempfile = true var sizeOnDisk int64 - sizeOnDisk, err = writeAndCloseFile(r, kind, hash, size, tf) + sizeOnDisk, err = c.writeAndCloseFile(r, kind, hash, size, tf) if err != nil { return err } @@ -479,7 +486,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) return err } -func writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int64, f *os.File) (int64, error) { +func (c *Cache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int64, f *os.File) (int64, error) { closeFile := true defer func() { if closeFile { @@ -491,7 +498,7 @@ func writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int6 var sizeOnDisk int64 if kind == cache.CAS { - sizeOnDisk, err = casblob.WriteAndClose(r, f, casblob.Zstandard, hash, size) + sizeOnDisk, err = casblob.WriteAndClose(r, f, c.storageMode, hash, size) if err != nil { return -1, err } diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go index ded345621..00545cb41 100644 --- a/cache/disk/disk_test.go +++ b/cache/disk/disk_test.go @@ -48,7 +48,7 @@ func TestCacheBasics(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(itemSize * 2) - testCache, err := New(cacheDir, cacheSize, nil) + testCache, err := New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -91,7 +91,7 @@ func TestCacheBasics(t *testing.T) { func TestCachePutWrongSize(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 100, nil) + testCache, err := New(cacheDir, 100, "zstd", nil) if err != nil { t.Fatal(err) } @@ -118,7 +118,7 @@ func TestCacheGetContainsWrongSize(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 100, nil) + testCache, err := New(cacheDir, 100, "zstd", nil) if err != nil { t.Fatal(err) } @@ -156,7 +156,7 @@ func TestCacheGetContainsWrongSizeWithProxy(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 100, new(proxyStub)) + testCache, err := New(cacheDir, 100, "zstd", new(proxyStub)) if err != nil { t.Fatal(err) } @@ -296,7 +296,7 @@ func TestOverwrite(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 100, nil) + testCache, err := New(cacheDir, 100, "zstd", nil) if err != nil { t.Fatal(err) } @@ -393,7 +393,7 @@ func TestCacheExistingFiles(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. const cacheSize = 1024 - testCache, err := New(cacheDir, cacheSize, nil) + testCache, err := New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -448,7 +448,7 @@ func TestCacheExistingFiles(t *testing.T) { func TestCacheBlobTooLarge(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 100, nil) + testCache, err := New(cacheDir, 100, "zstd", nil) if err != nil { t.Fatal(err) } @@ -474,7 +474,7 @@ func TestCacheBlobTooLarge(t *testing.T) { func TestCacheCorruptedCASBlob(t *testing.T) { cacheDir := tempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 1000, nil) + testCache, err := New(cacheDir, 1000, "zstd", nil) if err != nil { t.Fatal(err) } @@ -558,7 +558,7 @@ func TestMigrateFromOldDirectoryStructure(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. const cacheSize = 2560 * 2 - testCache, err := New(cacheDir, cacheSize, nil) + testCache, err := New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -628,7 +628,7 @@ func TestLoadExistingEntries(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(blobSize * numBlobs * 2) - testCache, err := New(cacheDir, cacheSize, nil) + testCache, err := New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -666,7 +666,7 @@ func TestDistinctKeyspaces(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(blobSize*3) * 2 - testCache, err := New(cacheDir, cacheSize, nil) + testCache, err := New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -783,7 +783,7 @@ func TestHttpProxyBackend(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(1024*10) * 2 - testCache, err := New(cacheDir, cacheSize, proxy) + testCache, err := New(cacheDir, cacheSize, "zstd", proxy) if err != nil { t.Fatal(err) } @@ -821,7 +821,7 @@ func TestHttpProxyBackend(t *testing.T) { // Create a new (empty) testCache, without a proxy backend. cacheDir = testutils.TempDir(t) defer os.RemoveAll(cacheDir) - testCache, err = New(cacheDir, cacheSize, nil) + testCache, err = New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -883,7 +883,7 @@ func TestGetValidatedActionResult(t *testing.T) { cacheDir := testutils.TempDir(t) defer os.RemoveAll(cacheDir) - testCache, err := New(cacheDir, 1024*32, nil) + testCache, err := New(cacheDir, 1024*32, "zstd", nil) if err != nil { t.Fatal(err) } @@ -1055,7 +1055,7 @@ func TestGetWithOffset(t *testing.T) { const blobSize = 2048 + 256 - testCache, err := New(cacheDir, blobSize*2, nil) + testCache, err := New(cacheDir, blobSize*2, "zstd", nil) if err != nil { t.Fatal(err) } diff --git a/cache/httpproxy/httpproxy_test.go b/cache/httpproxy/httpproxy_test.go index e8342dec7..963e92597 100644 --- a/cache/httpproxy/httpproxy_test.go +++ b/cache/httpproxy/httpproxy_test.go @@ -110,7 +110,7 @@ func TestEverything(t *testing.T) { proxyCache := New(url, &http.Client{}, accessLogger, errorLogger, 100, 10000) diskCacheSize := int64(len(casData) + 2048) - diskCache, err := disk.New(cacheDir, diskCacheSize, proxyCache) + diskCache, err := disk.New(cacheDir, diskCacheSize, "zstd", proxyCache) if err != nil { t.Fatal(err) } @@ -268,7 +268,7 @@ func TestEverything(t *testing.T) { cacheDir2 := testutils.TempDir(t) defer os.RemoveAll(cacheDir2) - diskCache, err = disk.New(cacheDir2, diskCacheSize, proxyCache) + diskCache, err = disk.New(cacheDir2, diskCacheSize, "zstd", proxyCache) if err != nil { t.Fatal(err) } diff --git a/config/config.go b/config/config.go index 2abd9b613..fb59926aa 100644 --- a/config/config.go +++ b/config/config.go @@ -45,6 +45,7 @@ type Config struct { ProfilePort int `yaml:"profile_port"` Dir string `yaml:"dir"` MaxSize int `yaml:"max_size"` + StorageMode string `yaml:"storage_mode"` HtpasswdFile string `yaml:"htpasswd_file"` TLSCaFile string `yaml:"tls_ca_file"` TLSCertFile string `yaml:"tls_cert_file"` @@ -69,7 +70,8 @@ var defaultDurationBuckets = []float64{.5, 1, 2.5, 5, 10, 20, 40, 80, 160, 320} // New returns a validated Config with the specified values, and an error // if there were any problems with the validation. -func New(dir string, maxSize int, host string, port int, grpcPort int, +func New(dir string, maxSize int, storageMode string, + host string, port int, grpcPort int, profileHost string, profilePort int, htpasswdFile string, maxQueuedUploads int, @@ -94,6 +96,7 @@ func New(dir string, maxSize int, host string, port int, grpcPort int, ProfilePort: profilePort, Dir: dir, MaxSize: maxSize, + StorageMode: storageMode, HtpasswdFile: htpasswdFile, MaxQueuedUploads: maxQueuedUploads, NumUploaders: numUploaders, @@ -141,6 +144,7 @@ func NewFromYamlFile(path string) (*Config, error) { func newFromYaml(data []byte) (*Config, error) { c := Config{ + StorageMode: "zstd", NumUploaders: 100, MaxQueuedUploads: 1000000, MetricsDurationBuckets: defaultDurationBuckets, @@ -172,6 +176,10 @@ func validateConfig(c *Config) error { return errors.New("The 'max_size' flag/key must be set to a value > 0") } + if c.StorageMode != "zstd" && c.StorageMode != "uncompressed" { + return errors.New("storage_mode must be set to either \"zstd\" or \"uncompressed\"") + } + if c.Port == 0 { return errors.New("A valid 'port' flag/key must be specified") } diff --git a/config/config_test.go b/config/config_test.go index 25ef6315b..000d29b63 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -37,6 +37,7 @@ http_write_timeout: 10s GRPCPort: 9092, Dir: "/opt/cache-dir", MaxSize: 100, + StorageMode: "zstd", HtpasswdFile: "/opt/.htpasswd", TLSCertFile: "/opt/tls.cert", TLSKeyFile: "/opt/tls.key", @@ -73,11 +74,12 @@ gcs_proxy: } expectedConfig := &Config{ - Host: "localhost", - Port: 8080, - GRPCPort: 9092, - Dir: "/opt/cache-dir", - MaxSize: 100, + Host: "localhost", + Port: 8080, + GRPCPort: 9092, + Dir: "/opt/cache-dir", + MaxSize: 100, + StorageMode: "zstd", GoogleCloudStorage: &GoogleCloudStorageConfig{ Bucket: "gcs-bucket", UseDefaultCredentials: false, @@ -108,11 +110,12 @@ http_proxy: } expectedConfig := &Config{ - Host: "localhost", - Port: 8080, - GRPCPort: 9092, - Dir: "/opt/cache-dir", - MaxSize: 100, + Host: "localhost", + Port: 8080, + GRPCPort: 9092, + Dir: "/opt/cache-dir", + MaxSize: 100, + StorageMode: "zstd", HTTPBackend: &HTTPBackendConfig{ BaseURL: "https://remote-cache.com:8080/cache", }, @@ -177,10 +180,11 @@ s3_proxy: } expectedConfig := &Config{ - Host: "localhost", - Port: 8080, - Dir: "/opt/cache-dir", - MaxSize: 100, + Host: "localhost", + Port: 8080, + Dir: "/opt/cache-dir", + MaxSize: 100, + StorageMode: "zstd", S3CloudStorage: &S3CloudStorageConfig{ Endpoint: "minio.example.com:9000", Bucket: "test-bucket", @@ -216,6 +220,7 @@ profile_port: 7070 Port: 1234, Dir: "/opt/cache-dir", MaxSize: 42, + StorageMode: "zstd", ProfilePort: 7070, ProfileHost: "", NumUploaders: 100, @@ -247,6 +252,7 @@ func TestValidMetricsDurationBuckets(t *testing.T) { port: 1234 dir: /opt/cache-dir max_size: 42 +storage_mode: zstd endpoint_metrics_duration_buckets: [.005, .1, 5] ` config, err := newFromYaml([]byte(yaml)) @@ -259,6 +265,7 @@ endpoint_metrics_duration_buckets: [.005, .1, 5] Port: 1234, Dir: "/opt/cache-dir", MaxSize: 42, + StorageMode: "zstd", NumUploaders: 100, MaxQueuedUploads: 1000000, MetricsDurationBuckets: []float64{0.005, 0.1, 5}, @@ -275,6 +282,7 @@ func TestMetricsDurationBucketsNoDupliates(t *testing.T) { Port: 8080, MaxSize: 42, Dir: "/opt/cache-dir", + StorageMode: "uncompressed", MetricsDurationBuckets: []float64{1, 2, 3, 3}, } err := validateConfig(testConfig) @@ -285,3 +293,63 @@ func TestMetricsDurationBucketsNoDupliates(t *testing.T) { t.Fatal("Expected the error message to mention the invalid 'endpoint_metrics_duration_buckets' key") } } + +func TestStorageModes(t *testing.T) { + tests := []struct { + yaml string + expected string + invalid bool + }{{ + yaml: `host: localhost +port: 1234 +dir: /foo/bar +max_size: 20 +`, + expected: "zstd", + }, + { + yaml: `host: localhost +port: 1234 +dir: /foo/bar +max_size: 20 +storage_mode: zstd +`, + expected: "zstd", + }, + { + yaml: `host: localhost +port: 1234 +dir: /foo/bar +max_size: 20 +storage_mode: uncompressed +`, + expected: "uncompressed", + }, + { + yaml: `host: localhost +port: 1234 +dir: /foo/bar +max_size: 20 +storage_mode: gzip +`, + invalid: true, + }} + + for _, tc := range tests { + cfg, err := newFromYaml([]byte(tc.yaml)) + if !tc.invalid && err != nil { + t.Error("Expected to succeed, got", err) + } + + if tc.invalid { + if err == nil { + t.Error("Expected an error, got nil") + } + continue + } + + if cfg.StorageMode != tc.expected { + t.Errorf("Expected %q, got %q", tc.expected, cfg.StorageMode) + } + } +} diff --git a/main.go b/main.go index e31841a6b..7d8f5bac8 100644 --- a/main.go +++ b/main.go @@ -88,6 +88,12 @@ func main() { Usage: "The maximum size of the remote cache in GiB. This flag is required.", EnvVars: []string{"BAZEL_REMOTE_MAX_SIZE"}, }, + &cli.StringFlag{ + Name: "storage_mode", + Value: "zstd", + Usage: "Which format to store CAS blobs in. Must be one of \"zstd\" or \"uncompressed\".", + EnvVars: []string{"BAZEL_REMOTE_STORAGE_MODE"}, + }, &cli.StringFlag{ Name: "host", Value: "", @@ -297,6 +303,7 @@ func main() { c, err = config.New( ctx.String("dir"), ctx.Int("max_size"), + ctx.String("storage_mode"), ctx.String("host"), ctx.Int("port"), ctx.Int("grpc_port"), @@ -364,7 +371,7 @@ func main() { proxyCache = s3proxy.New(c.S3CloudStorage, accessLogger, errorLogger, c.NumUploaders, c.MaxQueuedUploads) } - diskCache, err := disk.New(c.Dir, int64(c.MaxSize)*1024*1024*1024, proxyCache) + diskCache, err := disk.New(c.Dir, int64(c.MaxSize)*1024*1024*1024, c.StorageMode, proxyCache) if err != nil { log.Fatal(err) } diff --git a/server/grpc_test.go b/server/grpc_test.go index 5ecfc198a..8825c4c2c 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -75,7 +75,7 @@ func TestMain(m *testing.M) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(10 * maxChunkSize * 2) - diskCache, err = disk.New(dir, cacheSize, nil) + diskCache, err = disk.New(dir, cacheSize, "zstd", nil) if err != nil { fmt.Println("Test setup failed") os.Exit(1) diff --git a/server/http_test.go b/server/http_test.go index 0cc8edd98..eec5064db 100644 --- a/server/http_test.go +++ b/server/http_test.go @@ -43,7 +43,7 @@ func TestDownloadFile(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := blobSize * 2 - c, err := disk.New(cacheDir, cacheSize, nil) + c, err := disk.New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -113,7 +113,7 @@ func TestUploadFilesConcurrently(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(NumUploads * blobSize * 2) - c, err := disk.New(cacheDir, cacheSize, nil) + c, err := disk.New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -177,7 +177,7 @@ func TestUploadSameFileConcurrently(t *testing.T) { // Add some overhead for likely CAS blob storage expansion. cacheSize := int64(len(data) * numWorkers * 2) - c, err := disk.New(cacheDir, cacheSize, nil) + c, err := disk.New(cacheDir, cacheSize, "zstd", nil) if err != nil { t.Fatal(err) } @@ -226,7 +226,7 @@ func TestUploadCorruptedFile(t *testing.T) { t.Fatal(err) } - c, err := disk.New(cacheDir, 2048, nil) + c, err := disk.New(cacheDir, 2048, "zstd", nil) if err != nil { t.Fatal(err) } @@ -269,7 +269,7 @@ func TestUploadEmptyActionResult(t *testing.T) { t.Fatal(err) } - c, err := disk.New(cacheDir, 2048, nil) + c, err := disk.New(cacheDir, 2048, "zstd", nil) if err != nil { t.Fatal(err) } @@ -329,7 +329,7 @@ func testEmptyBlobAvailable(t *testing.T, method string) { t.Fatal(err) } - c, err := disk.New(cacheDir, 2048, nil) + c, err := disk.New(cacheDir, 2048, "zstd", nil) if err != nil { t.Fatal(err) } @@ -357,7 +357,7 @@ func TestStatusPage(t *testing.T) { t.Fatal(err) } - c, err := disk.New(cacheDir, 2048, nil) + c, err := disk.New(cacheDir, 2048, "zstd", nil) if err != nil { t.Fatal(err) } @@ -500,7 +500,7 @@ func TestRemoteReturnsNotFound(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(cacheDir) - emptyCache, err := disk.New(cacheDir, 1024, nil) + emptyCache, err := disk.New(cacheDir, 1024, "zstd", nil) if err != nil { t.Fatal(err) } From 2176041c23f474bddfb2a6ed14bf29a37ab4a0a2 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Thu, 31 Dec 2020 15:10:43 +0100 Subject: [PATCH 12/35] [disk] fan out when migrating old data files --- cache/disk/disk.go | 106 +++++++++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 38 deletions(-) diff --git a/cache/disk/disk.go b/cache/disk/disk.go index f06893efd..0366ef707 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -13,6 +13,7 @@ import ( "path" "path/filepath" "regexp" + "runtime" "sort" "strings" "sync" @@ -181,54 +182,83 @@ func migrateDirectory(baseDir string, kind cache.EntryKind) error { targetDir := path.Join(baseDir, kind.DirName()) - for _, item := range listing { - oldName := item.Name() - oldNamePath := filepath.Join(sourceDir, oldName) + itemChan := make(chan os.FileInfo) + errChan := make(chan error) - if item.IsDir() { - if !v1DirRegex.MatchString(oldName) { - // Warn about non-v1 subdirectories. - log.Println("Warning: unexpected directory", oldNamePath) - } + var wg sync.WaitGroup + for i := 0; i < runtime.NumCPU(); i++ { + wg.Add(1) + go func() { + defer wg.Done() - destDir := filepath.Join(targetDir, oldName[:2]) - err = migrateV1Subdir(oldNamePath, destDir, kind) - if err != nil { - log.Printf("Warning: failed to read subdir %q: %s", - oldNamePath, err) - continue - } + for item := range itemChan { - continue - } + oldName := item.Name() + oldNamePath := filepath.Join(sourceDir, oldName) - if !item.Mode().IsRegular() { - log.Println("Warning: skipping non-regular file:", oldNamePath) - continue - } + if item.IsDir() { + if !v1DirRegex.MatchString(oldName) { + // Warn about non-v1 subdirectories. + log.Println("Warning: unexpected directory", oldNamePath) + } - if !hashKeyRegex.MatchString(oldName) { - log.Println("Warning: skipping unexpected file:", oldNamePath) - continue - } + destDir := filepath.Join(targetDir, oldName[:2]) + err = migrateV1Subdir(oldNamePath, destDir, kind) + if err != nil { + log.Printf("Warning: failed to read subdir %q: %s", + oldNamePath, err) + continue + } - // oldName can now be assumed to be a hash. - hash := oldName + continue + } - src := filepath.Join(sourceDir, oldName) - dest := filepath.Join(targetDir, oldName[:2], oldName) - if kind == cache.CAS { - err = migrateCASFile(src, dest, hash) - } else { - // TODO: make this work across filesystems? - err = os.Rename(src, dest) - } - if err != nil { - return err + if !item.Mode().IsRegular() { + log.Println("Warning: skipping non-regular file:", oldNamePath) + continue + } + + if !hashKeyRegex.MatchString(oldName) { + log.Println("Warning: skipping unexpected file:", oldNamePath) + continue + } + + // oldName can now be assumed to be a hash. + hash := oldName + + src := filepath.Join(sourceDir, oldName) + dest := filepath.Join(targetDir, oldName[:2], oldName) + if kind == cache.CAS { + err = migrateCASFile(src, dest, hash) + } else { + // TODO: make this work across filesystems? + err = os.Rename(src, dest) + } + if err != nil { + errChan <- err + return + } + } + }() + } + + err = nil + numItems := len(listing) + i := 1 + for _, item := range listing { + select { + case itemChan <- item: + log.Printf("Migrating %s item(s) %d/%d, %s\n", sourceDir, i, numItems, item.Name()) + i++ + case err = <-errChan: + log.Println("Encountered error while migrating files:", err) + close(itemChan) } } + close(itemChan) + wg.Wait() - return nil + return err } func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error { From 038e067b9d4f7b7f25d7785bdce0ebde201472c7 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 2 Jan 2021 09:34:57 +0100 Subject: [PATCH 13/35] [disk] use raw uncompressed CAS blob files Migrating old data files to the new format is too slow, we will need to add back support for raw uncompressed files. --- cache/disk/casblob/casblob.go | 57 ++++++++++++++++--------------- cache/disk/disk.go | 64 ++++++++++++++++++++++++----------- 2 files changed, 75 insertions(+), 46 deletions(-) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index 2c1421d99..6bc7c514c 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -281,33 +281,7 @@ func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadClo } } - pr, pw := io.Pipe() - - enc, ok := encoderPool.Get().(*syncpool.EncoderWrapper) - if !ok { - err = errEncoderPoolFail - } - if err != nil { - f.Close() - return nil, err - } - enc.Reset(pw) - - go func() { - // Read from the file, write to enc. - - // TODO: consider implementing something with a timeout? - _, err := enc.ReadFrom(f) - if err != nil { - // We can't do anything here except log an error. - log.Println("Error while compressing file:", err) - } - - enc.Close() - f.Close() - }() - - return pr, nil + return GetLegacyZstdReadCloser(f) } if h.compression != Zstandard { @@ -358,6 +332,35 @@ func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadClo }, nil } +// GetLegacyZstdReadCloser returns an io.ReadCloser that provides +// zstandard-compressed data from an uncompressed file. +func GetLegacyZstdReadCloser(f *os.File) (io.ReadCloser, error) { + enc, ok := encoderPool.Get().(*syncpool.EncoderWrapper) + if !ok { + f.Close() + return nil, errEncoderPoolFail + } + + pr, pw := io.Pipe() + enc.Reset(pw) + + go func() { + // Read from the file, write to enc. + + // TODO: consider implementing something with a timeout? + _, err := enc.ReadFrom(f) + if err != nil { + // We can't do anything here except log an error. + log.Println("Error while compressing file:", err) + } + + encoderPool.Put(enc) + f.Close() + }() + + return pr, nil +} + func (h *header) write(f *os.File) error { var err error diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 0366ef707..11dcc3b70 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -53,6 +53,10 @@ type lruItem struct { // Size of the blob on disk (possibly with header + compression). sizeOnDisk int64 + + // If true, the blob is a raw CAS file (no header, uncompressed) + // with a ".v1" filename suffix. + legacy bool } // Cache is a filesystem-based LRU cache, with an optional backend proxy. @@ -203,7 +207,7 @@ func migrateDirectory(baseDir string, kind cache.EntryKind) error { } destDir := filepath.Join(targetDir, oldName[:2]) - err = migrateV1Subdir(oldNamePath, destDir, kind) + err := migrateV1Subdir(oldNamePath, destDir, kind) if err != nil { log.Printf("Warning: failed to read subdir %q: %s", oldNamePath, err) @@ -223,17 +227,14 @@ func migrateDirectory(baseDir string, kind cache.EntryKind) error { continue } - // oldName can now be assumed to be a hash. - hash := oldName - src := filepath.Join(sourceDir, oldName) dest := filepath.Join(targetDir, oldName[:2], oldName) if kind == cache.CAS { - err = migrateCASFile(src, dest, hash) - } else { - // TODO: make this work across filesystems? - err = os.Rename(src, dest) + dest += ".v1" } + + // TODO: make this work across filesystems? + err := os.Rename(src, dest) if err != nil { errChan <- err return @@ -279,9 +280,8 @@ func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error return fmt.Errorf("Unexpected file: %s", oldPath) } - destPath := path.Join(destDir, item.Name()) - - err = migrateCASFile(oldPath, destPath, item.Name()) + destPath := path.Join(destDir, item.Name()) + ".v1" + err = os.Rename(oldPath, destPath) if err != nil { return fmt.Errorf("Failed to migrate CAS blob %s: %w", oldPath, err) @@ -310,6 +310,7 @@ func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error return nil } +/* TODO: remove this dead code? func migrateCASFile(src string, dest string, hash string) error { srcFile, err := os.Open(src) @@ -344,6 +345,7 @@ func migrateCASFile(src string, dest string, hash string) error { return err } +*/ // loadExistingFiles lists all files in the cache directory, and adds them to the // LRU index so that they can be served. Files are sorted by access time first, @@ -377,18 +379,27 @@ func (c *Cache) loadExistingFiles() error { log.Println("Building LRU index.") for _, f := range files { relPath := f.name[len(c.dir)+1:] - var lookupKey string - sizeOnDisk := f.info.Size() - size := sizeOnDisk + legacy := strings.HasSuffix(f.name, ".v1") fields := strings.Split(relPath, "/") + hash := fields[len(fields)-1] + if legacy { + hash = strings.TrimSuffix(hash, ".v1") + } + + sizeOnDisk := f.info.Size() + size := sizeOnDisk + + var lookupKey string if strings.HasPrefix(relPath, "cas.v2/") { - size, err = casblob.GetLogicalSize(f.name) - if err != nil { - return err + if !legacy { + size, err = casblob.GetLogicalSize(f.name) + if err != nil { + return err + } } lookupKey = "cas/" + hash } else if strings.HasPrefix(relPath, "ac/") { @@ -402,6 +413,7 @@ func (c *Cache) loadExistingFiles() error { ok := c.lru.Add(lookupKey, lruItem{ size: size, sizeOnDisk: sizeOnDisk, + legacy: legacy, }) if !ok { err = os.Remove(filepath.Join(c.dir, relPath)) @@ -527,7 +539,7 @@ func (c *Cache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string var err error var sizeOnDisk int64 - if kind == cache.CAS { + if kind == cache.CAS && c.storageMode != casblob.Identity { sizeOnDisk, err = casblob.WriteAndClose(r, f, c.storageMode, hash, size) if err != nil { return -1, err @@ -610,19 +622,33 @@ func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string, kin locked = false if !isSizeMismatch(size, item.size) { + if item.legacy { + blobPath += ".v1" + } + var f *os.File f, err = os.Open(blobPath) if err != nil { // Race condition, was the item purged after we released the lock? log.Printf("Warning: expected %s to exist on disk, undersized cache?", blobPath) } else if kind == cache.CAS { - if zstd { + if item.legacy { + _, err = f.Seek(offset, io.SeekStart) + if !zstd && err == nil { + rc = f + } else if err == nil { + rc, err = casblob.GetLegacyZstdReadCloser(f) + } + } else if zstd { rc, err = casblob.GetZstdReadCloser(f, size, offset) } else { rc, err = casblob.GetUncompressedReadCloser(f, size, offset) } + if err != nil { log.Printf("Warning: expected item to be on disk, but something happened: %v", err) + f.Close() + rc = nil } else { return rc, item.size, false, nil } From f8eb86cad73e11e73ac67d481cd61523ab6051d2 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 2 Jan 2021 10:13:04 +0100 Subject: [PATCH 14/35] [disk] validate cache file hash values when loading --- cache/disk/disk.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 11dcc3b70..d915ab48c 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -46,6 +46,8 @@ var tfc = tempfile.NewCreator() var emptyZstdBlob = []byte{40, 181, 47, 253, 32, 0, 1, 0, 0} +var hashKeyRegex = regexp.MustCompile("^[a-f0-9]{64}$") + // lruItem is the type of the values stored in SizedLRU to keep track of items. type lruItem struct { // Size of the blob in uncompressed form. @@ -178,7 +180,6 @@ func migrateDirectory(baseDir string, kind cache.EntryKind) error { // The v0 directory structure was lowercase sha256 hash filenames // stored directly in the ac/ and cas/ directories. - hashKeyRegex := regexp.MustCompile("^[a-f0-9]{64}$") // The v1 directory structure has subdirs for each two lowercase // hex character pairs. @@ -269,8 +270,6 @@ func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error return err } - hashKeyRegex := regexp.MustCompile("^[a-f0-9]{64}$") - if kind == cache.CAS { for _, item := range listing { @@ -389,6 +388,10 @@ func (c *Cache) loadExistingFiles() error { hash = strings.TrimSuffix(hash, ".v1") } + if !hashKeyRegex.MatchString(hash) { + return fmt.Errorf("Invalid hash: %q", hash) + } + sizeOnDisk := f.info.Size() size := sizeOnDisk From 512ad74a991ea82d4f4cffa5c08813b01d646ba3 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 2 Jan 2021 10:15:09 +0100 Subject: [PATCH 15/35] remove --destructive_compression_prototype flag The on-disk format seems to be settled now, and migration should be as fast as is reasonably possible. I no longer consider it a risky move to migrate old caches. --- main.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/main.go b/main.go index 7d8f5bac8..4bf40edaa 100644 --- a/main.go +++ b/main.go @@ -64,11 +64,6 @@ func main() { app.HideHelpCommand = true app.Flags = []cli.Flag{ - &cli.BoolFlag{ - Name: "destructive_compression_prototype", - Usage: "This flag must be specified for this compressed-blobs prototype to run. Beware that this will migrate preexisting cache files to a temporary and incompatible format.", - EnvVars: []string{"BAZEL_REMOTE_DESTRUCTIVE_COMPRESSION_PROTOTYPE"}, - }, &cli.StringFlag{ Name: "config_file", Value: "", @@ -275,11 +270,6 @@ func main() { } app.Action = func(ctx *cli.Context) error { - if !ctx.Bool("destructive_compression_prototype") { - log.Println("WARNING: this is a prototype branch of bazel-remote that will break existing cache directories. Use the --destructive_compression_prototype flag to acknowledge this and run anyway.") - os.Exit(1) - } - configFile := ctx.String("config_file") var c *config.Config var err error From d5ad6e79e9c140119b0deccbbc2626bfed5fdf63 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sun, 3 Jan 2021 16:35:09 +0100 Subject: [PATCH 16/35] doc: mention compressed-blobs in the gRPC section of the readme file --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 1e310805c..e3d0bb903 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,19 @@ which can be enabled with the `--experimental_remote_asset_api` flag. To use this with Bazel, specify [--experimental_remote_downloader=grpc://replace-with-your.host:port](https://docs.bazel.build/versions/master/command-line-reference.html#flag--experimental_remote_downloader). +### Byte Stream compressed-blobs + +This version of bazel-remote supports the +[Byte Stream compressed-blobs proposal](https://github.com/bazelbuild/remote-apis/pull/168), +which provides a way for clients to upload and download CAS blobs compressed +with zstandard, in order to improve network efficiency. + +Uploaded CAS blobs are stored in a zstandard compressed format by default, +which can increase the effective cache size and reduce load on the server +if clients also download blobs in zstandard compressed form. If you would +rather store CAS blobs in uncompressed form, add `--storage_mode uncompressed` +to your configuration. + ## Usage If a YAML configuration file is specified by the `--config_file` command line From e8736120061ccdace8b11bea53eda229d00996de Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sun, 10 Jan 2021 15:13:48 +0100 Subject: [PATCH 17/35] [disk] fix space accounting --- cache/disk/disk.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cache/disk/disk.go b/cache/disk/disk.go index d915ab48c..432c861ad 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -573,7 +573,7 @@ func (c *Cache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string } // This must be called when the lock is not held. -func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSize int64, foundSize int64, sizeOnDisk int64) (unreserve bool, removeTempfile bool, err error) { +func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSize int64, logicalSize int64, sizeOnDisk int64) (unreserve bool, removeTempfile bool, err error) { unreserve = reservedSize > 0 removeTempfile = true @@ -589,9 +589,9 @@ func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSi } unreserve = false - if !c.lru.Add(key, lruItem{size: foundSize, sizeOnDisk: sizeOnDisk}) { + if !c.lru.Add(key, lruItem{size: logicalSize, sizeOnDisk: sizeOnDisk}) { err = fmt.Errorf("INTERNAL ERROR: failed to add: %s, size %d (on disk: %d): %w", - key, foundSize, sizeOnDisk, err) + key, logicalSize, sizeOnDisk, err) log.Println(err.Error()) return unreserve, removeTempfile, err } @@ -848,7 +848,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, } } - unreserve, removeTempfile, err = c.commit(key, tfName, blobPath, size, foundSize, sizeOnDisk) + unreserve, removeTempfile, err = c.commit(key, tfName, blobPath, size, logicalSize, sizeOnDisk) if err != nil { rc.Close() rc = nil From eaffa8276248158644ea26fa63c12f1a88fcc65b Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sun, 10 Jan 2021 16:31:36 +0100 Subject: [PATCH 18/35] [httpproxy] don't read remote file headers for CAS.v2 Contains size value This could put significant load on the backend server. Instead, just return "unknown size" and rely on the SHA256 key by itself to avoid collisions. --- cache/httpproxy/httpproxy.go | 48 ++++++------------------------- cache/httpproxy/httpproxy_test.go | 2 +- 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/cache/httpproxy/httpproxy.go b/cache/httpproxy/httpproxy.go index 11e96b909..bbac734fa 100644 --- a/cache/httpproxy/httpproxy.go +++ b/cache/httpproxy/httpproxy.go @@ -3,8 +3,6 @@ package httpproxy import ( - "bytes" - "encoding/binary" "errors" "fmt" "io" @@ -192,49 +190,19 @@ func (r *remoteHTTPProxyCache) Contains(kind cache.EntryKind, hash string) (bool url := requestURL(r.baseURL, hash, kind) - if kind != cache.CAS { - rsp, err := r.remote.Head(url) - if err == nil && rsp.StatusCode == http.StatusOK { + rsp, err := r.remote.Head(url) + if err == nil && rsp.StatusCode == http.StatusOK { + if kind != cache.CAS { return true, rsp.ContentLength } - return false, -1 - } - - // The following code is a little ugly. We can't use a HEAD request, - // because we need the uncompressed size from the header of the blob. - // That value is stored in the first 8 bytes of compressed CAS blobs, - // so attempt a GET range-request to get that value and decode it here. - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return false, -1 - } - - req.Header.Add("Range", "bytes=0-7") - rsp, err := r.remote.Do(req) - if err != nil || (rsp.StatusCode != http.StatusOK && rsp.StatusCode != http.StatusPartialContent) { - return false, -1 - } - - blobHeader := make([]byte, 8) - n, err := io.ReadFull(rsp.Body, blobHeader) - if err != nil { - return false, -1 - } - defer rsp.Body.Close() - if n != 8 { - return false, -1 - } - - var uncompressedSize int64 - err = binary.Read(bytes.NewReader(blobHeader), binary.LittleEndian, - &uncompressedSize) - if err != nil { - return false, -1 + // We don't know the content size without reading the file header + // and that could be very costly for the backend server. So return + // "unknown size". + return true, -1 } - return true, uncompressedSize + return false, -1 } func requestURL(baseURL *url.URL, hash string, kind cache.EntryKind) string { diff --git a/cache/httpproxy/httpproxy_test.go b/cache/httpproxy/httpproxy_test.go index 963e92597..afdc7a900 100644 --- a/cache/httpproxy/httpproxy_test.go +++ b/cache/httpproxy/httpproxy_test.go @@ -293,7 +293,7 @@ func TestEverything(t *testing.T) { if !found { t.Fatalf("Expected to find CAS item %s", hash) } - if size != int64(len(casData)) { + if size != -1 && size != int64(len(casData)) { t.Fatalf("Expected to find CAS item with size %d, got %d", len(casData), size) } From 67a2ad097f4a8b22f55e86e37324172f6426b9d9 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Thu, 14 Jan 2021 17:51:06 +0100 Subject: [PATCH 19/35] [disk] make compressed cas blob files valid .zst By using a zstd skippable frame for our header, the file becomes a valid .zst file on disk. This makes it possible to inspect cache files with standard tools. The cost is only 8 bytes per (compressed) file. Compressed CAS blobs are now limited to ~511TB (with 1M chunks), which should not be an issue in practical use. --- cache/disk/casblob/casblob.go | 57 +++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index 6bc7c514c..cb84d41d1 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -37,10 +37,24 @@ var errEncoderPoolFail error = errors.New("failed to get EncoderWrapper from poo const defaultChunkSize = 1024 * 1024 * 1 // 1M -// CAS blobs are stored with a header followed by chunks of data in either -// compressed or uncompressed format. +// 4 bytes, to be written to disk in little-endian format. +// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#skippable-frames +const skippableFrameMagicNumber = 0x184D2A50 + +// Compressed CAS blobs are stored in .zst format, with a header which +// describes the locations of independently compressed chunks. type header struct { // The following data is stored in little-endian format on disk. + + // We start with two zstd skippable frame fields: + + //magicNumber uint32 // 4 bytes: skippableFrameMagicNumber + + // With 1M chunks this gives a limit of ~511TB which is plenty. + //frameSize uint32 // 4 bytes: the size of the rest of the header. + + // Then we put our metadata: + uncompressedSize int64 // 8 bytes compression CompressionType // uint8, 1 byte chunkSize uint32 // 4 bytes @@ -54,13 +68,17 @@ type header struct { chunkOffsets []int64 } -const chunkTableOffset = 8 + 1 + 4 + 8 +const chunkTableOffset = 4 + 4 + 8 + 1 + 4 + 8 // Returns the size of the header itself. func (h *header) size() int64 { return chunkTableOffset + (int64(len(h.chunkOffsets)) * 8) } +func (h *header) frameSize() uint32 { + return chunkTableOffset + (uint32(len(h.chunkOffsets)) * 8) - 4 - 4 +} + // Provides an io.ReadCloser that returns uncompressed data from a cas blob. type readCloserWrapper struct { *header @@ -71,6 +89,8 @@ type readCloserWrapper struct { file *os.File } +var errWrongMagicNum = errors.New("expected magic number not found") + // Read the header and leave f at the start of the data. func readHeader(f *os.File) (*header, error) { var err error @@ -82,6 +102,21 @@ func readHeader(f *os.File) (*header, error) { } foundFileSize := fileInfo.Size() + var magicNumber uint32 + err = binary.Read(f, binary.LittleEndian, &magicNumber) + if err != nil { + return nil, fmt.Errorf("unable to read magic number: %w", err) + } + if magicNumber != skippableFrameMagicNumber { + return nil, errWrongMagicNum + } + + var frameSize uint32 + err = binary.Read(f, binary.LittleEndian, &frameSize) + if err != nil { + return nil, fmt.Errorf("unable to read frameSize: %w", err) + } + err = binary.Read(f, binary.LittleEndian, &h.uncompressedSize) if err != nil { return nil, err @@ -108,6 +143,12 @@ func readHeader(f *os.File) (*header, error) { return nil, fmt.Errorf("internal error: need at least one chunk, found %d", numOffsets-1) } + metadataSize := numOffsets*8 + 8 + 1 + 4 + 8 + if int64(frameSize) != metadataSize { + return nil, fmt.Errorf("metadata frame size %d, but metadata size %d", + frameSize, metadataSize) + } + prevOffset := int64(-1) h.chunkOffsets = make([]int64, numOffsets, numOffsets) for i := 0; int64(i) < numOffsets; i++ { @@ -364,6 +405,16 @@ func GetLegacyZstdReadCloser(f *os.File) (io.ReadCloser, error) { func (h *header) write(f *os.File) error { var err error + err = binary.Write(f, binary.LittleEndian, uint32(skippableFrameMagicNumber)) + if err != nil { + return err + } + + err = binary.Write(f, binary.LittleEndian, h.frameSize()) + if err != nil { + return err + } + err = binary.Write(f, binary.LittleEndian, h.uncompressedSize) if err != nil { return err From 9154319e952af1da16dd2e570c493a87ce14d319 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Thu, 14 Jan 2021 23:55:01 +0100 Subject: [PATCH 20/35] [disk] encode logical CAS blob sizes in the filename By doing this we can build the LRU index on startup using only the directory listings (ie without reading the header of compressed blobs). This should keep startup times low. --- cache/cache.go | 10 ++++----- cache/disk/disk.go | 48 ++++++++++++++++++++-------------------- cache/s3proxy/s3proxy.go | 14 +++++++----- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 9360e5c62..0489cdd99 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -3,6 +3,7 @@ package cache import ( "crypto/sha256" "encoding/hex" + "fmt" "io" "path" ) @@ -98,15 +99,12 @@ func TransformActionCacheKey(key, instance string, logger Logger) string { } // Key returns the proper cache key for an entry kind and hash. -func FileLocation(kind EntryKind, hash string) string { - var kindWithVersion string +func FileLocation(kind EntryKind, hash string, size int64) string { if kind == CAS { - kindWithVersion = "cas.v2" - } else { - kindWithVersion = kind.String() + return fmt.Sprintf("cas.v2/%s/%s-%d", hash[:2], hash, size) } - return path.Join(kindWithVersion, hash[:2], hash) + return path.Join(kind.String(), hash[:2], hash) } func LookupKey(kind EntryKind, hash string) string { diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 432c861ad..8d38ce1e2 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -15,6 +15,7 @@ import ( "regexp" "runtime" "sort" + "strconv" "strings" "sync" "syscall" @@ -47,6 +48,7 @@ var tfc = tempfile.NewCreator() var emptyZstdBlob = []byte{40, 181, 47, 253, 32, 0, 1, 0, 0} var hashKeyRegex = regexp.MustCompile("^[a-f0-9]{64}$") +var hashKeyWithSizeRegex = regexp.MustCompile("^([a-f0-9]{64})(?:-([1-9][0-9]*))?$") // lruItem is the type of the values stored in SizedLRU to keep track of items. type lruItem struct { @@ -118,7 +120,7 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) kind = cache.RAW } - f := filepath.Join(dir, cache.FileLocation(kind, hash)) + f := filepath.Join(dir, cache.FileLocation(kind, hash, value.size)) err := os.Remove(f) if err != nil { @@ -264,7 +266,6 @@ func migrateDirectory(baseDir string, kind cache.EntryKind) error { } func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error { - listing, err := ioutil.ReadDir(oldDir) if err != nil { return err @@ -383,27 +384,26 @@ func (c *Cache) loadExistingFiles() error { fields := strings.Split(relPath, "/") - hash := fields[len(fields)-1] + file := fields[len(fields)-1] if legacy { - hash = strings.TrimSuffix(hash, ".v1") + file = strings.TrimSuffix(file, ".v1") } - if !hashKeyRegex.MatchString(hash) { - return fmt.Errorf("Invalid hash: %q", hash) + sm := hashKeyWithSizeRegex.FindStringSubmatch(file) + if len(sm) == 0 || len(sm[1]) == 0 { + return fmt.Errorf("Unrecognized file: %q", file) } + hash := sm[1] sizeOnDisk := f.info.Size() size := sizeOnDisk + if len(sm[2]) > 0 { + size, err = strconv.ParseInt(sm[2], 10, 64) + } var lookupKey string if strings.HasPrefix(relPath, "cas.v2/") { - if !legacy { - size, err = casblob.GetLogicalSize(f.name) - if err != nil { - return err - } - } lookupKey = "cas/" + hash } else if strings.HasPrefix(relPath, "ac/") { lookupKey = "ac/" + hash @@ -500,7 +500,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) } // Final destination, if all goes well. - filePath := cacheFilePath(kind, c.dir, hash) + filePath := path.Join(c.dir, cache.FileLocation(kind, hash, size)) // We will download to this temporary file. tf, err := tfc.Create(filePath) @@ -615,20 +615,24 @@ func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSi // but that we can try the proxy backend. // // This function assumes that only CAS blobs are requested in zstd form. -func (c *Cache) availableOrTryProxy(key string, size int64, blobPath string, kind cache.EntryKind, offset int64, zstd bool) (rc io.ReadCloser, foundSize int64, tryProxy bool, err error) { +func (c *Cache) availableOrTryProxy(kind cache.EntryKind, hash string, size int64, offset int64, zstd bool) (rc io.ReadCloser, foundSize int64, tryProxy bool, err error) { locked := true c.mu.Lock() + key := cache.LookupKey(kind, hash) item, available := c.lru.Get(key) if available { c.mu.Unlock() // We expect a cache hit below. locked = false - if !isSizeMismatch(size, item.size) { - if item.legacy { - blobPath += ".v1" - } + var blobPath string + if item.legacy { + blobPath = path.Join(c.dir, "cas.v2", hash[:2], hash+".v1") + } else { + blobPath = path.Join(c.dir, cache.FileLocation(kind, hash, item.size)) + } + if !isSizeMismatch(size, item.size) { var f *os.File f, err = os.Open(blobPath) if err != nil { @@ -769,8 +773,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, } }() - blobPath := cacheFilePath(kind, c.dir, hash) - f, foundSize, tryProxy, err := c.availableOrTryProxy(key, size, blobPath, kind, offset, zstd) + f, foundSize, tryProxy, err := c.availableOrTryProxy(kind, hash, size, offset, zstd) if err != nil { return nil, -1, err } @@ -800,6 +803,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, return nil, -1, nil } + blobPath := path.Join(c.dir, cache.FileLocation(kind, hash, foundSize)) tf, err = tfc.Create(blobPath) if err != nil { return nil, -1, err @@ -929,10 +933,6 @@ func ensureDirExists(path string) { } } -func cacheFilePath(kind cache.EntryKind, cacheDir string, hash string) string { - return filepath.Join(cacheDir, cache.FileLocation(kind, hash)) -} - // GetValidatedActionResult returns a valid ActionResult and its serialized // value from the CAS if it and all its dependencies are also available. If // not, nil values are returned. If something unexpected went wrong, return diff --git a/cache/s3proxy/s3proxy.go b/cache/s3proxy/s3proxy.go index 5febea9f2..f53e7174d 100644 --- a/cache/s3proxy/s3proxy.go +++ b/cache/s3proxy/s3proxy.go @@ -10,6 +10,7 @@ import ( "log" "net/http" "net/url" + "path" "time" "github.com/buchgr/bazel-remote/cache" @@ -120,16 +121,19 @@ func New(s3Config *config.S3CloudStorageConfig, accessLogger cache.Logger, } func (c *s3Cache) objectKey(hash string, kind cache.EntryKind) string { - - // For CAS blobs this includes "cas.v2" to distinguish compressed - // blobs from older uncompressed blobs. - baseKey := cache.FileLocation(kind, hash) + var baseKey string + if kind == cache.CAS { + // Use "cas.v2" to distinguish new from old format blobs. + baseKey = path.Join("cas.v2", hash[:2], hash) + } else { + baseKey = path.Join(kind.String(), hash[:2], hash) + } if c.prefix == "" { return baseKey } - return fmt.Sprintf("%s/%s", c.prefix, baseKey) + return path.Join(c.prefix, baseKey) } // Helper function for logging responses From 3832c9cc8614fcbfcd0a3852c486b56ba3d64bb7 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Fri, 15 Jan 2021 09:11:44 +0100 Subject: [PATCH 21/35] [disk] fix --storage_mode uncompressed uploads While preexisting legacy/.v1 CAS blobs were being imported and used correctly, newly uploaded blobs weren't being named correctly with --storage_mode uncompressed. --- cache/cache.go | 11 ----------- cache/disk/disk.go | 43 ++++++++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 0489cdd99..37533230c 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -3,9 +3,7 @@ package cache import ( "crypto/sha256" "encoding/hex" - "fmt" "io" - "path" ) // EntryKind describes the kind of cache entry @@ -98,15 +96,6 @@ func TransformActionCacheKey(key, instance string, logger Logger) string { return newKey } -// Key returns the proper cache key for an entry kind and hash. -func FileLocation(kind EntryKind, hash string, size int64) string { - if kind == CAS { - return fmt.Sprintf("cas.v2/%s/%s-%d", hash[:2], hash, size) - } - - return path.Join(kind.String(), hash[:2], hash) -} - func LookupKey(kind EntryKind, hash string) string { return kind.String() + "/" + hash } diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 8d38ce1e2..bc9d559c5 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -105,6 +105,17 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) } } + compressionType := casblob.Zstandard + if storageMode == "uncompressed" { + compressionType = casblob.Identity + } + + c := &Cache{ + dir: filepath.Clean(dir), + storageMode: compressionType, + proxy: proxy, + } + // The eviction callback deletes the file from disk. // This function is only called while the lock is held // by the current goroutine. @@ -120,7 +131,7 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) kind = cache.RAW } - f := filepath.Join(dir, cache.FileLocation(kind, hash, value.size)) + f := filepath.Join(dir, c.FileLocation(kind, hash, value.size)) err := os.Remove(f) if err != nil { @@ -128,17 +139,7 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) } } - compressionType := casblob.Zstandard - if storageMode == "uncompressed" { - compressionType = casblob.Identity - } - - c := &Cache{ - dir: filepath.Clean(dir), - storageMode: compressionType, - proxy: proxy, - lru: NewSizedLRU(maxSizeBytes, onEvict), - } + c.lru = NewSizedLRU(maxSizeBytes, onEvict) err := c.migrateDirectories() if err != nil { @@ -152,6 +153,18 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) return c, nil } +func (c *Cache) FileLocation(kind cache.EntryKind, hash string, size int64) string { + if kind != cache.CAS { + return path.Join(kind.String(), hash[:2], hash) + } + + if c.storageMode == casblob.Identity { + return fmt.Sprintf("cas.v2/%s/%s.v1", hash[:2], hash) + } + + return fmt.Sprintf("cas.v2/%s/%s-%d", hash[:2], hash, size) +} + func (c *Cache) migrateDirectories() error { err := migrateDirectory(c.dir, cache.AC) if err != nil { @@ -500,7 +513,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) } // Final destination, if all goes well. - filePath := path.Join(c.dir, cache.FileLocation(kind, hash, size)) + filePath := path.Join(c.dir, c.FileLocation(kind, hash, size)) // We will download to this temporary file. tf, err := tfc.Create(filePath) @@ -629,7 +642,7 @@ func (c *Cache) availableOrTryProxy(kind cache.EntryKind, hash string, size int6 if item.legacy { blobPath = path.Join(c.dir, "cas.v2", hash[:2], hash+".v1") } else { - blobPath = path.Join(c.dir, cache.FileLocation(kind, hash, item.size)) + blobPath = path.Join(c.dir, c.FileLocation(kind, hash, item.size)) } if !isSizeMismatch(size, item.size) { @@ -803,7 +816,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, return nil, -1, nil } - blobPath := path.Join(c.dir, cache.FileLocation(kind, hash, foundSize)) + blobPath := path.Join(c.dir, c.FileLocation(kind, hash, foundSize)) tf, err = tfc.Create(blobPath) if err != nil { return nil, -1, err From f90788900d450af41428ca26b48b7fc69827cdbb Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 16 Jan 2021 18:47:30 +0100 Subject: [PATCH 22/35] [proxy] add cmdline flags for http and gcs backend proxies Unlike the other settings, these were previously only settable from a config file. --- README.md | 72 +++++++++++++++++++++++++----------------------- config/config.go | 22 +++++++++++++-- main.go | 43 +++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index e3d0bb903..9b2e674e2 100644 --- a/README.md +++ b/README.md @@ -136,40 +136,44 @@ DESCRIPTION: A remote build cache for Bazel. GLOBAL OPTIONS: - --config_file value Path to a YAML configuration file. If this flag is specified then all other flags are ignored. [$BAZEL_REMOTE_CONFIG_FILE] - --dir value Directory path where to store the cache contents. This flag is required. [$BAZEL_REMOTE_DIR] - --max_size value The maximum size of the remote cache in GiB. This flag is required. (default: -1) [$BAZEL_REMOTE_MAX_SIZE] - --storage_mode value Which format to store CAS blobs in. Must be one of "zstd" or "uncompressed". (default: "zstd") [$BAZEL_REMOTE_STORAGE_MODE] - --host value Address to listen on. Listens on all network interfaces by default. [$BAZEL_REMOTE_HOST] - --port value The port the HTTP server listens on. (default: 8080) [$BAZEL_REMOTE_PORT] - --grpc_port value The port the gRPC server listens on. Set to 0 to disable. (default: 9092) [$BAZEL_REMOTE_GRPC_PORT] - --profile_host value A host address to listen on for profiling, if enabled by a valid --profile_port setting. (default: "127.0.0.1") [$BAZEL_REMOTE_PROFILE_HOST] - --profile_port value If a positive integer, serve /debug/pprof/* URLs from http://profile_host:profile_port. (default: 0, ie profiling disabled) [$BAZEL_REMOTE_PROFILE_PORT] - --http_read_timeout value The HTTP read timeout for a client request in seconds (does not apply to the proxy backends or the profiling endpoint) (default: 0s, ie disabled) [$BAZEL_REMOTE_HTTP_READ_TIMEOUT] - --http_write_timeout value The HTTP write timeout for a server response in seconds (does not apply to the proxy backends or the profiling endpoint) (default: 0s, ie disabled) [$BAZEL_REMOTE_HTTP_WRITE_TIMEOUT] - --htpasswd_file value Path to a .htpasswd file. This flag is optional. Please read https://httpd.apache.org/docs/2.4/programs/htpasswd.html. [$BAZEL_REMOTE_HTPASSWD_FILE] - --tls_enabled This flag has been deprecated. Specify tls_cert_file and tls_key_file instead. (default: false) [$BAZEL_REMOTE_TLS_ENABLED] - --tls_ca_file value Optional. Enables mTLS (authenticating client certificates), should be the certificate authority that signed the client certificates. [$BAZEL_REMOTE_TLS_CA_FILE] - --tls_cert_file value Path to a pem encoded certificate file. [$BAZEL_REMOTE_TLS_CERT_FILE] - --tls_key_file value Path to a pem encoded key file. [$BAZEL_REMOTE_TLS_KEY_FILE] - --idle_timeout value The maximum period of having received no request after which the server will shut itself down. (default: 0s, ie disabled) [$BAZEL_REMOTE_IDLE_TIMEOUT] - --max_queued_uploads value When using proxy backends, sets the maximum number of objects in queue for upload. If the queue is full, uploads will be skipped until the queue has space again. (default: 1000000) [$BAZEL_REMOTE_MAX_QUEUED_UPLOADS] - --num_uploaders value When using proxy backends, sets the number of Goroutines to process parallel uploads to backend. (default: 100) [$BAZEL_REMOTE_NUM_UPLOADERS] - --s3.endpoint value The S3/minio endpoint to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_ENDPOINT] - --s3.bucket value The S3/minio bucket to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_BUCKET] - --s3.prefix value The S3/minio object prefix to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_PREFIX] - --s3.access_key_id value The S3/minio access key to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_ACCESS_KEY_ID] - --s3.secret_access_key value The S3/minio secret access key to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_SECRET_ACCESS_KEY] - --s3.disable_ssl Whether to disable TLS/SSL when using the S3 proxy backend. (default: false, ie enable TLS/SSL) [$BAZEL_REMOTE_S3_DISABLE_SSL] - --s3.iam_role_endpoint value Endpoint for using IAM security credentials. By default it will look for credentials in the standard locations for the AWS platform. [$BAZEL_REMOTE_S3_IAM_ROLE_ENDPOINT] - --s3.region value The AWS region. Required when not specifying S3/minio access keys. [$BAZEL_REMOTE_S3_REGION] - --s3.key_version value Set to 1 for the legacy flat key format, or 2 for the newer format that reduces the impact of S3 rate limits. (default: 1) [$BAZEL_REMOTE_S3_KEY_VERSION] - --disable_http_ac_validation Whether to disable ActionResult validation for HTTP requests. (default: false, ie enable validation) [$BAZEL_REMOTE_DISABLE_HTTP_AC_VALIDATION] - --disable_grpc_ac_deps_check Whether to disable ActionResult dependency checks for gRPC GetActionResult requests. (default: false, ie enable ActionCache dependency checks) [$BAZEL_REMOTE_DISABLE_GRPS_AC_DEPS_CHECK] - --enable_ac_key_instance_mangling Whether to enable mangling ActionCache keys with non-empty instance names. (default: false, ie disable mangling) [$BAZEL_REMOTE_ENABLE_AC_KEY_INSTANCE_MANGLING] - --enable_endpoint_metrics Whether to enable metrics for each HTTP/gRPC endpoint. (default: false, ie disable metrics) [$BAZEL_REMOTE_ENABLE_ENDPOINT_METRICS] - --experimental_remote_asset_api Whether to enable the experimental remote asset API implementation. (default: false, ie disable remote asset API) [$BAZEL_REMOTE_EXPERIMENTAL_REMOTE_ASSET_API] - --help, -h show help (default: false) + --config_file value Path to a YAML configuration file. If this flag is specified then all other flags are ignored. [$BAZEL_REMOTE_CONFIG_FILE] + --dir value Directory path where to store the cache contents. This flag is required. [$BAZEL_REMOTE_DIR] + --max_size value The maximum size of the remote cache in GiB. This flag is required. (default: -1) [$BAZEL_REMOTE_MAX_SIZE] + --storage_mode value Which format to store CAS blobs in. Must be one of "zstd" or "uncompressed". (default: "zstd") [$BAZEL_REMOTE_STORAGE_MODE] + --host value Address to listen on. Listens on all network interfaces by default. [$BAZEL_REMOTE_HOST] + --port value The port the HTTP server listens on. (default: 8080) [$BAZEL_REMOTE_PORT] + --grpc_port value The port the gRPC server listens on. Set to 0 to disable. (default: 9092) [$BAZEL_REMOTE_GRPC_PORT] + --profile_host value A host address to listen on for profiling, if enabled by a valid --profile_port setting. (default: "127.0.0.1") [$BAZEL_REMOTE_PROFILE_HOST] + --profile_port value If a positive integer, serve /debug/pprof/* URLs from http://profile_host:profile_port. (default: 0, ie profiling disabled) [$BAZEL_REMOTE_PROFILE_PORT] + --http_read_timeout value The HTTP read timeout for a client request in seconds (does not apply to the proxy backends or the profiling endpoint) (default: 0s, ie disabled) [$BAZEL_REMOTE_HTTP_READ_TIMEOUT] + --http_write_timeout value The HTTP write timeout for a server response in seconds (does not apply to the proxy backends or the profiling endpoint) (default: 0s, ie disabled) [$BAZEL_REMOTE_HTTP_WRITE_TIMEOUT] + --htpasswd_file value Path to a .htpasswd file. This flag is optional. Please read https://httpd.apache.org/docs/2.4/programs/htpasswd.html. [$BAZEL_REMOTE_HTPASSWD_FILE] + --tls_enabled This flag has been deprecated. Specify tls_cert_file and tls_key_file instead. (default: false) [$BAZEL_REMOTE_TLS_ENABLED] + --tls_ca_file value Optional. Enables mTLS (authenticating client certificates), should be the certificate authority that signed the client certificates. [$BAZEL_REMOTE_TLS_CA_FILE] + --tls_cert_file value Path to a pem encoded certificate file. [$BAZEL_REMOTE_TLS_CERT_FILE] + --tls_key_file value Path to a pem encoded key file. [$BAZEL_REMOTE_TLS_KEY_FILE] + --idle_timeout value The maximum period of having received no request after which the server will shut itself down. (default: 0s, ie disabled) [$BAZEL_REMOTE_IDLE_TIMEOUT] + --max_queued_uploads value When using proxy backends, sets the maximum number of objects in queue for upload. If the queue is full, uploads will be skipped until the queue has space again. (default: 1000000) [$BAZEL_REMOTE_MAX_QUEUED_UPLOADS] + --num_uploaders value When using proxy backends, sets the number of Goroutines to process parallel uploads to backend. (default: 100) [$BAZEL_REMOTE_NUM_UPLOADERS] + --http_proxy.url value The base URL to use for a http proxy backend. [$BAZEL_REMOTE_HTTP_PROXY_URL] + --gcs_proxy.bucket value The bucket to use for the Google Cloud Storage proxy backend. [$BAZEL_REMOTE_GCS_BUCKET] + --gcs_proxy.use_default_credentials Whether or not to use authentication for the Google Cloud Storage proxy backend. (default: false) [$BAZEL_REMOTE_GCS_USE_DEFAULT_CREDENTIALS] + --gcs_proxy.json_credentials_file value Path to a JSON file that contains Google credentials for the Google Cloud Storage proxy backend. [$BAZEL_REMOTE_GCS_JSON_CREDENTIALS_FILE] + --s3.endpoint value The S3/minio endpoint to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_ENDPOINT] + --s3.bucket value The S3/minio bucket to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_BUCKET] + --s3.prefix value The S3/minio object prefix to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_PREFIX] + --s3.access_key_id value The S3/minio access key to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_ACCESS_KEY_ID] + --s3.secret_access_key value The S3/minio secret access key to use when using S3 proxy backend. [$BAZEL_REMOTE_S3_SECRET_ACCESS_KEY] + --s3.disable_ssl Whether to disable TLS/SSL when using the S3 proxy backend. (default: false, ie enable TLS/SSL) [$BAZEL_REMOTE_S3_DISABLE_SSL] + --s3.iam_role_endpoint value Endpoint for using IAM security credentials. By default it will look for credentials in the standard locations for the AWS platform. [$BAZEL_REMOTE_S3_IAM_ROLE_ENDPOINT] + --s3.region value The AWS region. Required when not specifying S3/minio access keys. [$BAZEL_REMOTE_S3_REGION] + --s3.key_version value Set to 1 for the legacy flat key format, or 2 for the newer format that reduces the impact of S3 rate limits. (default: 1) [$BAZEL_REMOTE_S3_KEY_VERSION] + --disable_http_ac_validation Whether to disable ActionResult validation for HTTP requests. (default: false, ie enable validation) [$BAZEL_REMOTE_DISABLE_HTTP_AC_VALIDATION] + --disable_grpc_ac_deps_check Whether to disable ActionResult dependency checks for gRPC GetActionResult requests. (default: false, ie enable ActionCache dependency checks) [$BAZEL_REMOTE_DISABLE_GRPS_AC_DEPS_CHECK] + --enable_ac_key_instance_mangling Whether to enable mangling ActionCache keys with non-empty instance names. (default: false, ie disable mangling) [$BAZEL_REMOTE_ENABLE_AC_KEY_INSTANCE_MANGLING] + --enable_endpoint_metrics Whether to enable metrics for each HTTP/gRPC endpoint. (default: false, ie disable metrics) [$BAZEL_REMOTE_ENABLE_ENDPOINT_METRICS] + --experimental_remote_asset_api Whether to enable the experimental remote asset API implementation. (default: false, ie disable remote asset API) [$BAZEL_REMOTE_EXPERIMENTAL_REMOTE_ASSET_API] + --help, -h show help (default: false) ``` ### Example configuration file diff --git a/config/config.go b/config/config.go index fb59926aa..66051fc4c 100644 --- a/config/config.go +++ b/config/config.go @@ -80,6 +80,8 @@ func New(dir string, maxSize int, storageMode string, tlsCertFile string, tlsKeyFile string, idleTimeout time.Duration, + hc *HTTPBackendConfig, + gcs *GoogleCloudStorageConfig, s3 *S3CloudStorageConfig, disableHTTPACValidation bool, disableGRPCACDepsCheck bool, @@ -88,6 +90,7 @@ func New(dir string, maxSize int, storageMode string, experimentalRemoteAssetAPI bool, httpReadTimeout time.Duration, httpWriteTimeout time.Duration) (*Config, error) { + c := Config{ Host: host, Port: port, @@ -104,8 +107,8 @@ func New(dir string, maxSize int, storageMode string, TLSCertFile: tlsCertFile, TLSKeyFile: tlsKeyFile, S3CloudStorage: s3, - GoogleCloudStorage: nil, - HTTPBackend: nil, + GoogleCloudStorage: gcs, + HTTPBackend: hc, IdleTimeout: idleTimeout, DisableHTTPACValidation: disableHTTPACValidation, DisableGRPCACDepsCheck: disableGRPCACDepsCheck, @@ -180,6 +183,21 @@ func validateConfig(c *Config) error { return errors.New("storage_mode must be set to either \"zstd\" or \"uncompressed\"") } + proxyCount := 0 + if c.S3CloudStorage != nil { + proxyCount++ + } + if c.HTTPBackend != nil { + proxyCount++ + } + if c.GoogleCloudStorage != nil { + proxyCount++ + } + + if proxyCount > 1 { + return errors.New("At most one of the S3/GCS/HTTP proxy backends is allowed") + } + if c.Port == 0 { return errors.New("A valid 'port' flag/key must be specified") } diff --git a/main.go b/main.go index 4bf40edaa..7d1a6214a 100644 --- a/main.go +++ b/main.go @@ -182,6 +182,30 @@ func main() { Usage: "When using proxy backends, sets the number of Goroutines to process parallel uploads to backend.", EnvVars: []string{"BAZEL_REMOTE_NUM_UPLOADERS"}, }, + &cli.StringFlag{ + Name: "http_proxy.url", + Value: "", + Usage: "The base URL to use for a http proxy backend.", + EnvVars: []string{"BAZEL_REMOTE_HTTP_PROXY_URL"}, + }, + &cli.StringFlag{ + Name: "gcs_proxy.bucket", + Value: "", + Usage: "The bucket to use for the Google Cloud Storage proxy backend.", + EnvVars: []string{"BAZEL_REMOTE_GCS_BUCKET"}, + }, + &cli.BoolFlag{ + Name: "gcs_proxy.use_default_credentials", + Value: false, + Usage: "Whether or not to use authentication for the Google Cloud Storage proxy backend.", + EnvVars: []string{"BAZEL_REMOTE_GCS_USE_DEFAULT_CREDENTIALS"}, + }, + &cli.StringFlag{ + Name: "gcs_proxy.json_credentials_file", + Value: "", + Usage: "Path to a JSON file that contains Google credentials for the Google Cloud Storage proxy backend.", + EnvVars: []string{"BAZEL_REMOTE_GCS_JSON_CREDENTIALS_FILE"}, + }, &cli.StringFlag{ Name: "s3.endpoint", Value: "", @@ -290,6 +314,23 @@ func main() { KeyVersion: ctx.Int("s3.key_version"), } } + + var hc *config.HTTPBackendConfig + if ctx.String("http_proxy.url") != "" { + hc = &config.HTTPBackendConfig{ + BaseURL: ctx.String("http_proxy.url"), + } + } + + var gcs *config.GoogleCloudStorageConfig + if ctx.String("gcs_proxy.bucket") != "" { + gcs = &config.GoogleCloudStorageConfig{ + Bucket: ctx.String("gcs_proxy.bucket"), + UseDefaultCredentials: ctx.Bool("gcs_proxy.use_default_credentials"), + JSONCredentialsFile: ctx.String("gcs_proxy.json_credentials_file"), + } + } + c, err = config.New( ctx.String("dir"), ctx.Int("max_size"), @@ -306,6 +347,8 @@ func main() { ctx.String("tls_cert_file"), ctx.String("tls_key_file"), ctx.Duration("idle_timeout"), + hc, + gcs, s3, ctx.Bool("disable_http_ac_validation"), ctx.Bool("disable_grpc_ac_deps_check"), From 0cdfdeb6ecb0dca0b1abdb97b69c210061164a0d Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sun, 31 Jan 2021 18:54:15 +0100 Subject: [PATCH 23/35] [proxy] switch between v1 and v2 CAS blobs depending on the storage mode Since we have two different on-disk CAS blob formats, and we don't want to perform transcoding in the proxy backends, we need to use a different path for v2 CAS blobs. This means that for now, it's not possible to use storage_mode: zstd and use the httpproxy backend to proxy blobs to another bazel-remote instance. Also, when switching storage modes, stored blobs for the other storage mode on the proxy backends will not be used. --- cache/cache.go | 22 ++++++---- cache/disk/disk.go | 73 +++++++++++++++++-------------- cache/disk/disk_test.go | 5 ++- cache/gcsproxy/gcsproxy.go | 4 +- cache/httpproxy/httpproxy.go | 72 ++++++++++++++++++------------ cache/httpproxy/httpproxy_test.go | 6 ++- cache/s3proxy/s3proxy.go | 34 ++++++++++++-- cache/s3proxy/s3proxy_test.go | 46 +++++++++---------- config/config_test.go | 1 + main.go | 9 ++-- 10 files changed, 167 insertions(+), 105 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 37533230c..5fd3da7fb 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -62,14 +62,20 @@ func (e *Error) Error() string { // Proxy is the interface that (optional) proxy backends must implement. // Implementations are expected to be safe for concurrent use. type Proxy interface { - // Put should make a reasonable effort to proxy this data to the backend. - // This is allowed to fail silently (eg when under heavy load). - Put(kind EntryKind, hash string, size int64, rdr io.ReadCloser) - - // Get should return the cache item identified by `hash`, or an error - // if something went wrong. If the item was not found, the io.ReadCloser - // will be nil. - Get(kind EntryKind, hash string) (io.ReadCloser, int64, error) + + // Put makes a reasonable effort to upload the cache item identified by + // `hash` with logical size `size`, whose data is readable from `rc` to + // the proxy backend. The data available in `rc` is in the same format + // as used by the disk.Cache instance. + // + // This is allowed to fail silently (for example when under heavy load). + Put(kind EntryKind, hash string, size int64, rc io.ReadCloser) + + // Get returns an io.ReadCloser from which the cache item identified by + // `hash` can be read, its logical size, and an error if something went + // wrong. The data available from `rc` is in the same format as used by + // the disk.Cache instance. + Get(kind EntryKind, hash string) (rc io.ReadCloser, size int64, err error) // Contains returns whether or not the cache item exists on the // remote end, and the size if it exists (and -1 if the size is diff --git a/cache/disk/disk.go b/cache/disk/disk.go index bc9d559c5..9c4669874 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -131,7 +131,7 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) kind = cache.RAW } - f := filepath.Join(dir, c.FileLocation(kind, hash, value.size)) + f := filepath.Join(dir, c.FileLocation(kind, value.legacy, hash, value.size)) err := os.Remove(f) if err != nil { @@ -153,12 +153,12 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) return c, nil } -func (c *Cache) FileLocation(kind cache.EntryKind, hash string, size int64) string { +func (c *Cache) FileLocation(kind cache.EntryKind, legacy bool, hash string, size int64) string { if kind != cache.CAS { return path.Join(kind.String(), hash[:2], hash) } - if c.storageMode == casblob.Identity { + if legacy || c.storageMode == casblob.Identity { return fmt.Sprintf("cas.v2/%s/%s.v1", hash[:2], hash) } @@ -512,8 +512,10 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) unreserve = true } + legacy := kind == cache.CAS && c.storageMode == casblob.Identity + // Final destination, if all goes well. - filePath := path.Join(c.dir, c.FileLocation(kind, hash, size)) + filePath := path.Join(c.dir, c.FileLocation(kind, legacy, hash, size)) // We will download to this temporary file. tf, err := tfc.Create(filePath) @@ -538,7 +540,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) } } - unreserve, removeTempfile, err = c.commit(key, tf.Name(), filePath, size, size, sizeOnDisk) + unreserve, removeTempfile, err = c.commit(key, legacy, tf.Name(), filePath, size, size, sizeOnDisk) // Might be nil. return err @@ -586,7 +588,7 @@ func (c *Cache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string } // This must be called when the lock is not held. -func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSize int64, logicalSize int64, sizeOnDisk int64) (unreserve bool, removeTempfile bool, err error) { +func (c *Cache) commit(key string, legacy bool, tempfile string, finalPath string, reservedSize int64, logicalSize int64, sizeOnDisk int64) (unreserve bool, removeTempfile bool, err error) { unreserve = reservedSize > 0 removeTempfile = true @@ -602,7 +604,7 @@ func (c *Cache) commit(key string, tempfile string, finalPath string, reservedSi } unreserve = false - if !c.lru.Add(key, lruItem{size: logicalSize, sizeOnDisk: sizeOnDisk}) { + if !c.lru.Add(key, lruItem{size: logicalSize, sizeOnDisk: sizeOnDisk, legacy: legacy}) { err = fmt.Errorf("INTERNAL ERROR: failed to add: %s, size %d (on disk: %d): %w", key, logicalSize, sizeOnDisk, err) log.Println(err.Error()) @@ -638,12 +640,7 @@ func (c *Cache) availableOrTryProxy(kind cache.EntryKind, hash string, size int6 c.mu.Unlock() // We expect a cache hit below. locked = false - var blobPath string - if item.legacy { - blobPath = path.Join(c.dir, "cas.v2", hash[:2], hash+".v1") - } else { - blobPath = path.Join(c.dir, c.FileLocation(kind, hash, item.size)) - } + blobPath := path.Join(c.dir, c.FileLocation(kind, item.legacy, hash, item.size)) if !isSizeMismatch(size, item.size) { var f *os.File @@ -653,16 +650,20 @@ func (c *Cache) availableOrTryProxy(kind cache.EntryKind, hash string, size int6 log.Printf("Warning: expected %s to exist on disk, undersized cache?", blobPath) } else if kind == cache.CAS { if item.legacy { + // The file is uncompressed, without a casblob header. _, err = f.Seek(offset, io.SeekStart) - if !zstd && err == nil { - rc = f - } else if err == nil { + if zstd && err == nil { rc, err = casblob.GetLegacyZstdReadCloser(f) + } else if err == nil { + rc = f } - } else if zstd { - rc, err = casblob.GetZstdReadCloser(f, size, offset) } else { - rc, err = casblob.GetUncompressedReadCloser(f, size, offset) + // The file is compressed. + if zstd { + rc, err = casblob.GetZstdReadCloser(f, size, offset) + } else { + rc, err = casblob.GetUncompressedReadCloser(f, size, offset) + } } if err != nil { @@ -816,7 +817,9 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, return nil, -1, nil } - blobPath := path.Join(c.dir, c.FileLocation(kind, hash, foundSize)) + legacy := kind == cache.CAS && c.storageMode == casblob.Identity + + blobPath := path.Join(c.dir, c.FileLocation(kind, legacy, hash, foundSize)) tf, err = tfc.Create(blobPath) if err != nil { return nil, -1, err @@ -834,7 +837,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, logicalSize := foundSize - if kind == cache.CAS { + if kind == cache.CAS && c.storageMode != casblob.Identity { logicalSize, err = casblob.GetLogicalSize(tfName) if err != nil { return nil, -1, err @@ -848,24 +851,30 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, if err != nil { return nil, -1, err } - if kind != cache.CAS { + + uncompressedOnDisk := (kind != cache.CAS) || (c.storageMode == casblob.Identity) + if uncompressedOnDisk { if offset > 0 { rcf.Seek(offset, io.SeekStart) } - rc = rcf - } else if zstd { - rc, err = casblob.GetZstdReadCloser(rcf, size, offset) - if err != nil { - return nil, -1, err + + if zstd { + rc, err = casblob.GetLegacyZstdReadCloser(rcf) + } else { + rc = rcf } - } else { - rc, err = casblob.GetUncompressedReadCloser(rcf, size, offset) - if err != nil { - return nil, -1, err + } else { // Compressed CAS blob. + if zstd { + rc, err = casblob.GetZstdReadCloser(rcf, logicalSize, offset) + } else { + rc, err = casblob.GetUncompressedReadCloser(rcf, logicalSize, offset) } } + if err != nil { + return nil, -1, err + } - unreserve, removeTempfile, err = c.commit(key, tfName, blobPath, size, logicalSize, sizeOnDisk) + unreserve, removeTempfile, err = c.commit(key, legacy, tfName, blobPath, size, logicalSize, sizeOnDisk) if err != nil { rc.Close() rc = nil diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go index 00545cb41..9bee14d41 100644 --- a/cache/disk/disk_test.go +++ b/cache/disk/disk_test.go @@ -775,7 +775,10 @@ func TestHttpProxyBackend(t *testing.T) { accessLogger := testutils.NewSilentLogger() errorLogger := testutils.NewSilentLogger() - proxy := httpproxy.New(url, &http.Client{}, accessLogger, errorLogger, 100, 1000000) + proxy, err := httpproxy.New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 1000000) + if err != nil { + t.Fatal(err) + } cacheDir := testutils.TempDir(t) defer os.RemoveAll(cacheDir) diff --git a/cache/gcsproxy/gcsproxy.go b/cache/gcsproxy/gcsproxy.go index 74444e3c7..f5ed4ef87 100644 --- a/cache/gcsproxy/gcsproxy.go +++ b/cache/gcsproxy/gcsproxy.go @@ -16,7 +16,7 @@ import ( ) // New creates a cache that proxies requests to Google Cloud Storage. -func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string, +func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string, storageMode string, accessLogger cache.Logger, errorLogger cache.Logger, numUploaders, maxQueuedUploads int) (cache.Proxy, error) { var remoteClient *http.Client var err error @@ -54,5 +54,5 @@ func New(bucket string, useDefaultCredentials bool, jsonCredentialsFile string, Path: bucket, } - return httpproxy.New(&baseURL, remoteClient, accessLogger, errorLogger, numUploaders, maxQueuedUploads), nil + return httpproxy.New(&baseURL, storageMode, remoteClient, accessLogger, errorLogger, numUploaders, maxQueuedUploads) } diff --git a/cache/httpproxy/httpproxy.go b/cache/httpproxy/httpproxy.go index bbac734fa..ff9c775a7 100644 --- a/cache/httpproxy/httpproxy.go +++ b/cache/httpproxy/httpproxy.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "github.com/buchgr/bazel-remote/cache" @@ -26,10 +27,11 @@ type uploadReq struct { type remoteHTTPProxyCache struct { remote *http.Client - baseURL *url.URL + baseURL string uploadQueue chan<- uploadReq accessLogger cache.Logger errorLogger cache.Logger + requestURL func(hash string, kind cache.EntryKind) string } var ( @@ -43,8 +45,7 @@ var ( }) ) -func uploadFile(remote *http.Client, baseURL *url.URL, accessLogger cache.Logger, - errorLogger cache.Logger, item uploadReq) { +func (r *remoteHTTPProxyCache) uploadFile(item uploadReq) { if item.size == 0 { item.rc.Close() @@ -52,11 +53,11 @@ func uploadFile(remote *http.Client, baseURL *url.URL, accessLogger cache.Logger item.rc = http.NoBody } - url := requestURL(baseURL, item.hash, item.kind) + url := r.requestURL(item.hash, item.kind) - rsp, err := remote.Head(url) + rsp, err := r.remote.Head(url) if err == nil && rsp.StatusCode == http.StatusOK { - accessLogger.Printf("SKIP UPLOAD %s", item.hash) + r.accessLogger.Printf("SKIP UPLOAD %s", item.hash) return } @@ -71,44 +72,63 @@ func uploadFile(remote *http.Client, baseURL *url.URL, accessLogger cache.Logger req.Header.Set("Content-Type", "application/octet-stream") req.ContentLength = item.size - rsp, err = remote.Do(req) + rsp, err = r.remote.Do(req) if err != nil { return } io.Copy(ioutil.Discard, rsp.Body) rsp.Body.Close() - logResponse(accessLogger, "UPLOAD", rsp.StatusCode, url) + logResponse(r.accessLogger, "UPLOAD", rsp.StatusCode, url) return } // New creates a cache that proxies requests to a HTTP remote cache. -func New(baseURL *url.URL, remote *http.Client, accessLogger cache.Logger, - errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy { +// `storageMode` must be one of "uncompressed" (which expects legacy +// CAS blobs) or "zstd" (which expects cas.v2 blobs). +func New(baseURL *url.URL, storageMode string, remote *http.Client, + accessLogger cache.Logger, errorLogger cache.Logger, + numUploaders, maxQueuedUploads int) (cache.Proxy, error) { proxy := &remoteHTTPProxyCache{ remote: remote, - baseURL: baseURL, + baseURL: strings.TrimRight(baseURL.String(), "/"), accessLogger: accessLogger, errorLogger: errorLogger, } + if storageMode == "zstd" { + proxy.requestURL = func(hash string, kind cache.EntryKind) string { + if kind == cache.CAS { + return fmt.Sprintf("%s/cas.v2/%s", proxy.baseURL, hash) + } + + return fmt.Sprintf("%s/%s/%s", proxy.baseURL, kind, hash) + } + } else if storageMode == "uncompressed" { + proxy.requestURL = func(hash string, kind cache.EntryKind) string { + return fmt.Sprintf("%s/%s/%s", proxy.baseURL, kind, hash) + } + } else { + return nil, fmt.Errorf("Invalid http_proxy.mode specified: %q", + storageMode) + } + if maxQueuedUploads > 0 && numUploaders > 0 { uploadQueue := make(chan uploadReq, maxQueuedUploads) for i := 0; i < numUploaders; i++ { - go func(remote *http.Client, baseURL *url.URL, accessLogger cache.Logger, - errorLogger cache.Logger) { + go func() { for item := range uploadQueue { - uploadFile(remote, baseURL, accessLogger, errorLogger, item) + proxy.uploadFile(item) } - }(remote, baseURL, accessLogger, errorLogger) + }() } proxy.uploadQueue = uploadQueue } - return proxy + return proxy, nil } // Helper function for logging responses @@ -122,13 +142,15 @@ func (r *remoteHTTPProxyCache) Put(kind cache.EntryKind, hash string, size int64 return } - select { - case r.uploadQueue <- uploadReq{ + item := uploadReq{ hash: hash, size: size, kind: kind, rc: rc, - }: + } + + select { + case r.uploadQueue <- item: default: r.errorLogger.Printf("too many uploads queued") rc.Close() @@ -136,7 +158,7 @@ func (r *remoteHTTPProxyCache) Put(kind cache.EntryKind, hash string, size int64 } func (r *remoteHTTPProxyCache) Get(kind cache.EntryKind, hash string) (io.ReadCloser, int64, error) { - url := requestURL(r.baseURL, hash, kind) + url := r.requestURL(hash, kind) rsp, err := r.remote.Get(url) if err != nil { cacheMisses.Inc() @@ -188,7 +210,7 @@ func (r *remoteHTTPProxyCache) Get(kind cache.EntryKind, hash string) (io.ReadCl func (r *remoteHTTPProxyCache) Contains(kind cache.EntryKind, hash string) (bool, int64) { - url := requestURL(r.baseURL, hash, kind) + url := r.requestURL(hash, kind) rsp, err := r.remote.Head(url) if err == nil && rsp.StatusCode == http.StatusOK { @@ -204,11 +226,3 @@ func (r *remoteHTTPProxyCache) Contains(kind cache.EntryKind, hash string) (bool return false, -1 } - -func requestURL(baseURL *url.URL, hash string, kind cache.EntryKind) string { - if kind == cache.CAS { - // We need to distinguish these from uncompressed CAS blobs. - return fmt.Sprintf("%s/cas.v2/%s", baseURL, hash) - } - return fmt.Sprintf("%s/%s/%s", baseURL, kind, hash) -} diff --git a/cache/httpproxy/httpproxy_test.go b/cache/httpproxy/httpproxy_test.go index afdc7a900..49ec6a8cb 100644 --- a/cache/httpproxy/httpproxy_test.go +++ b/cache/httpproxy/httpproxy_test.go @@ -108,7 +108,11 @@ func TestEverything(t *testing.T) { t.Log("cas HASH:", hash) acData := []byte{1, 2, 3, 4} - proxyCache := New(url, &http.Client{}, accessLogger, errorLogger, 100, 10000) + proxyCache, err := New(url, "zstd", &http.Client{}, accessLogger, errorLogger, 100, 10000) + if err != nil { + t.Fatal(err) + } + diskCacheSize := int64(len(casData) + 2048) diskCache, err := disk.New(cacheDir, diskCacheSize, "zstd", proxyCache) if err != nil { diff --git a/cache/s3proxy/s3proxy.go b/cache/s3proxy/s3proxy.go index f53e7174d..e2dab93ae 100644 --- a/cache/s3proxy/s3proxy.go +++ b/cache/s3proxy/s3proxy.go @@ -36,6 +36,8 @@ type s3Cache struct { uploadQueue chan<- uploadReq accessLogger cache.Logger errorLogger cache.Logger + v2mode bool + objectKey func(hash string, kind cache.EntryKind) string } var ( @@ -53,7 +55,7 @@ var ( var errNotFound = errors.New("NOT FOUND") // New returns a new instance of the S3-API based cache -func New(s3Config *config.S3CloudStorageConfig, accessLogger cache.Logger, +func New(s3Config *config.S3CloudStorageConfig, storageMode string, accessLogger cache.Logger, errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy { fmt.Println("Using S3 backend.") @@ -95,6 +97,11 @@ func New(s3Config *config.S3CloudStorageConfig, accessLogger cache.Logger, } } + if storageMode != "zstd" && storageMode != "uncompressed" { + log.Fatalf("Unsupported storage mode for the s3proxy backend: %q, must be one of \"zstd\" or \"uncompressed\"", + storageMode) + } + c := &s3Cache{ mcore: minioCore, prefix: s3Config.Prefix, @@ -102,6 +109,17 @@ func New(s3Config *config.S3CloudStorageConfig, accessLogger cache.Logger, keyVersion: s3Config.KeyVersion, accessLogger: accessLogger, errorLogger: errorLogger, + v2mode: storageMode == "zstd", + } + + if c.v2mode { + c.objectKey = func(hash string, kind cache.EntryKind) string { + return objectKeyV2(c.prefix, hash, kind) + } + } else { + c.objectKey = func(hash string, kind cache.EntryKind) string { + return objectKeyV1(c.prefix, hash, kind) + } } if maxQueuedUploads > 0 && numUploaders > 0 { @@ -120,7 +138,7 @@ func New(s3Config *config.S3CloudStorageConfig, accessLogger cache.Logger, return c } -func (c *s3Cache) objectKey(hash string, kind cache.EntryKind) string { +func objectKeyV2(prefix string, hash string, kind cache.EntryKind) string { var baseKey string if kind == cache.CAS { // Use "cas.v2" to distinguish new from old format blobs. @@ -129,11 +147,19 @@ func (c *s3Cache) objectKey(hash string, kind cache.EntryKind) string { baseKey = path.Join(kind.String(), hash[:2], hash) } - if c.prefix == "" { + if prefix == "" { return baseKey } - return path.Join(c.prefix, baseKey) + return path.Join(prefix, baseKey) +} + +func objectKeyV1(prefix string, hash string, kind cache.EntryKind) string { + if prefix == "" { + return path.Join(kind.String(), hash[:2], hash) + } + + return path.Join(prefix, kind.String(), hash[:2], hash) } // Helper function for logging responses diff --git a/cache/s3proxy/s3proxy_test.go b/cache/s3proxy/s3proxy_test.go index 7ed0a514c..b87f134fc 100644 --- a/cache/s3proxy/s3proxy_test.go +++ b/cache/s3proxy/s3proxy_test.go @@ -1,42 +1,38 @@ package s3proxy import ( - "log" "testing" "github.com/buchgr/bazel-remote/cache" ) func TestObjectKey(t *testing.T) { - logger := &log.Logger{} - s := &s3Cache{ - mcore: nil, - bucket: "test", - keyVersion: 1, - uploadQueue: make(chan uploadReq, 1), - accessLogger: logger, - errorLogger: logger, - } - - // New key format tests - s.keyVersion = 2 - testCases := []struct { - prefix string - key string - kind cache.EntryKind - expected string + prefix string + key string + kind cache.EntryKind + expectedV1 string + expectedV2 string }{ - {"", "1234", cache.CAS, "cas.v2/12/1234"}, - {"test", "1234", cache.CAS, "test/cas.v2/12/1234"}, + {"", "1234", cache.CAS, "cas/12/1234", "cas.v2/12/1234"}, + {"test", "1234", cache.CAS, "test/cas/12/1234", "test/cas.v2/12/1234"}, + {"foo/bar/grok", "1234", cache.CAS, "foo/bar/grok/cas/12/1234", "foo/bar/grok/cas.v2/12/1234"}, + {"", "1234", cache.AC, "ac/12/1234", "ac/12/1234"}, + {"", "1234", cache.RAW, "raw/12/1234", "raw/12/1234"}, + {"foo/bar", "1234", cache.AC, "foo/bar/ac/12/1234", "foo/bar/ac/12/1234"}, } for _, tc := range testCases { - s.prefix = tc.prefix - result := s.objectKey(tc.key, tc.kind) - if result != tc.expected { - t.Errorf("objectKey did not match. (result: '%s' expected: '%s'", - result, tc.expected) + result := objectKeyV2(tc.prefix, tc.key, tc.kind) + if result != tc.expectedV2 { + t.Errorf("objectKeyV2 did not match. (result: '%s' expected: '%s'", + result, tc.expectedV2) + } + + result = objectKeyV1(tc.prefix, tc.key, tc.kind) + if result != tc.expectedV1 { + t.Errorf("objectKeyV1 did not match. (result: '%s' expected: '%s'", + result, tc.expectedV1) } } } diff --git a/config/config_test.go b/config/config_test.go index 000d29b63..2644681cb 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -103,6 +103,7 @@ dir: /opt/cache-dir max_size: 100 http_proxy: url: https://remote-cache.com:8080/cache + mode: zstd ` config, err := newFromYaml([]byte(yaml)) if err != nil { diff --git a/main.go b/main.go index 7d1a6214a..05d7d22f0 100644 --- a/main.go +++ b/main.go @@ -387,7 +387,7 @@ func main() { if c.GoogleCloudStorage != nil { proxyCache, err = gcsproxy.New(c.GoogleCloudStorage.Bucket, c.GoogleCloudStorage.UseDefaultCredentials, c.GoogleCloudStorage.JSONCredentialsFile, - accessLogger, errorLogger, c.NumUploaders, c.MaxQueuedUploads) + c.StorageMode, accessLogger, errorLogger, c.NumUploaders, c.MaxQueuedUploads) if err != nil { log.Fatal(err) } @@ -398,10 +398,13 @@ func main() { if err != nil { log.Fatal(err) } - proxyCache = httpproxy.New(baseURL, + proxyCache, err = httpproxy.New(baseURL, c.StorageMode, httpClient, accessLogger, errorLogger, c.NumUploaders, c.MaxQueuedUploads) + if err != nil { + log.Fatal(err) + } } else if c.S3CloudStorage != nil { - proxyCache = s3proxy.New(c.S3CloudStorage, accessLogger, errorLogger, c.NumUploaders, c.MaxQueuedUploads) + proxyCache = s3proxy.New(c.S3CloudStorage, c.StorageMode, accessLogger, errorLogger, c.NumUploaders, c.MaxQueuedUploads) } diskCache, err := disk.New(c.Dir, int64(c.MaxSize)*1024*1024*1024, c.StorageMode, proxyCache) From 163eb402240745512b797494fec814f5880b6e76 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Mon, 8 Feb 2021 00:08:11 +0100 Subject: [PATCH 24/35] [s3proxy] don't read remote file headers for CAS.v2 Contains size value Rely on the sha256 hash alone, like we do in httpproxy.Contains. This avoids the need to update this code since we added the zstandard skippable framing, which I don't have time to debug right now. --- cache/s3proxy/s3proxy.go | 84 +++++----------------------------------- 1 file changed, 10 insertions(+), 74 deletions(-) diff --git a/cache/s3proxy/s3proxy.go b/cache/s3proxy/s3proxy.go index e2dab93ae..66b44c6db 100644 --- a/cache/s3proxy/s3proxy.go +++ b/cache/s3proxy/s3proxy.go @@ -1,17 +1,12 @@ package s3proxy import ( - "bytes" "context" - "encoding/binary" "errors" "fmt" "io" "log" - "net/http" - "net/url" "path" - "time" "github.com/buchgr/bazel-remote/cache" "github.com/buchgr/bazel-remote/config" @@ -241,79 +236,20 @@ func (c *s3Cache) Contains(kind cache.EntryKind, hash string) (bool, int64) { size := int64(-1) exists := false - if kind != cache.CAS { - s, err := c.mcore.StatObject( - context.Background(), - c.bucket, // bucketName - c.objectKey(hash, kind), // objectName - minio.StatObjectOptions{}, // opts - ) - - exists = (err == nil) - if err != nil { - err = errNotFound - } else { - size = s.Size - } - - logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err) - - return exists, size - } - - // Handle the more complicated, compressed CAS blob case. - - // https://github.com/minio/minio-go/issues/1106 - - var err error - var uncompressedSize int64 - var n int - var req *http.Request - var rsp *http.Response - var blobHeader []byte - var u *url.URL - - u, err = c.mcore.PresignedGetObject(context.Background(), c.bucket, - c.objectKey(hash, kind), time.Second*20, make(url.Values)) - if err != nil { - goto end - } - - // TODO: The following code is essentially duplicated from the - // httpproxy code. Refactor and reuse it (and drop the goto's)? - - req, err = http.NewRequest("GET", u.String(), nil) - if err != nil { - goto end - } - req.Header.Add("Range", "bytes=0-7") - rsp, err = http.DefaultClient.Do(req) // FIXME: use a single http.Client - if err != nil { - goto end - } - if rsp.StatusCode != http.StatusOK && rsp.StatusCode != http.StatusPartialContent { - goto end - } - - blobHeader = make([]byte, 8) - n, err = io.ReadFull(rsp.Body, blobHeader) - if err != nil { - goto end - } - defer rsp.Body.Close() - if n != 8 { - goto end - } + s, err := c.mcore.StatObject( + context.Background(), + c.bucket, // bucketName + c.objectKey(hash, kind), // objectName + minio.StatObjectOptions{}, // opts + ) - err = binary.Read(bytes.NewReader(blobHeader), binary.LittleEndian, - &uncompressedSize) + exists = (err == nil) if err != nil { - goto end + err = errNotFound + } else if kind != cache.CAS { + size = s.Size } - exists = true - size = uncompressedSize -end: logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err) return exists, size From 9af561e4fcdcda1820fb5dbde9f05c6fa8295dac Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Tue, 9 Feb 2021 20:12:41 +0100 Subject: [PATCH 25/35] [http] don't interact with files in tests, use disk.Cache public API --- server/http_test.go | 68 +++++++++++++-------------------------------- 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/server/http_test.go b/server/http_test.go index eec5064db..85a47b38e 100644 --- a/server/http_test.go +++ b/server/http_test.go @@ -12,7 +12,6 @@ import ( "net/http/httptest" "net/url" "os" - "path/filepath" "sync" "testing" @@ -31,14 +30,6 @@ func TestDownloadFile(t *testing.T) { blobSize := int64(1024) data, hash := testutils.RandomDataAndHash(blobSize) - err := os.MkdirAll(filepath.Join(cacheDir, "cas"), 0755) - if err != nil { - t.Fatal(err) - } - err = ioutil.WriteFile(filepath.Join(cacheDir, "cas", hash), data, 0644) - if err != nil { - t.Fatal(err) - } // Add some overhead for likely CAS blob storage expansion. cacheSize := blobSize * 2 @@ -49,13 +40,14 @@ func TestDownloadFile(t *testing.T) { } h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, false, "") - req, err := http.NewRequest("GET", "/cas/"+hash, bytes.NewReader([]byte{})) - if err != nil { - t.Fatal(err) - } rr := httptest.NewRecorder() handler := http.HandlerFunc(h.CacheHandler) - handler.ServeHTTP(rr, req) + + pr := httptest.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data)) + handler.ServeHTTP(rr, pr) + + gr := httptest.NewRequest("GET", "/cas/"+hash, nil) + handler.ServeHTTP(rr, gr) if status := rr.Code; status != http.StatusOK { t.Fatal("Handler returned wrong status code for", hash, @@ -80,11 +72,8 @@ func TestDownloadFile(t *testing.T) { ) } - req, err = http.NewRequest("HEAD", "/cas/"+hash, bytes.NewReader([]byte{})) - if err != nil { - t.Fatal(err) - } - handler.ServeHTTP(rr, req) + hr := httptest.NewRequest("HEAD", "/cas/"+hash, nil) + handler.ServeHTTP(rr, hr) rsp = rr.Result() if contentLen := rsp.ContentLength; contentLen != blobSize { t.Error("HEAD request returned wrong content length", @@ -103,10 +92,7 @@ func TestUploadFilesConcurrently(t *testing.T) { var requests [NumUploads]*http.Request for i := 0; i < NumUploads; i++ { data, hash := testutils.RandomDataAndHash(blobSize) - r, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data)) - if err != nil { - t.Fatal(err) - } + r := httptest.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data)) requests[i] = r } @@ -190,14 +176,9 @@ func TestUploadSameFileConcurrently(t *testing.T) { go func() { defer wg.Done() - rr := httptest.NewRecorder() - - request, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data)) - if err != nil { - t.Error(err) - return - } + request := httptest.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data)) + rr := httptest.NewRecorder() handler.ServeHTTP(rr, request) if status := rr.Code; status != http.StatusOK { @@ -221,10 +202,7 @@ func TestUploadCorruptedFile(t *testing.T) { data, hash := testutils.RandomDataAndHash(1024) corruptedData := data[:999] - r, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(corruptedData)) - if err != nil { - t.Fatal(err) - } + r := httptest.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(corruptedData)) c, err := disk.New(cacheDir, 2048, "zstd", nil) if err != nil { @@ -264,10 +242,7 @@ func TestUploadEmptyActionResult(t *testing.T) { data, hash := testutils.RandomDataAndHash(0) - r, err := http.NewRequest("PUT", "/ac/"+hash, bytes.NewReader(data)) - if err != nil { - t.Fatal(err) - } + r := httptest.NewRequest("PUT", "/ac/"+hash, bytes.NewReader(data)) c, err := disk.New(cacheDir, 2048, "zstd", nil) if err != nil { @@ -286,8 +261,11 @@ func TestUploadEmptyActionResult(t *testing.T) { "got", status) } - cacheFile := filepath.Join(cacheDir, "ac", hash[:2], hash) - cachedData, err := ioutil.ReadFile(cacheFile) + getReq := httptest.NewRequest("GET", "/ac/"+hash, nil) + rr2 := httptest.NewRecorder() + handler.ServeHTTP(rr2, getReq) + + cachedData, err := ioutil.ReadAll(rr2.Result().Body) if err != nil { t.Fatal(err) } @@ -324,10 +302,7 @@ func testEmptyBlobAvailable(t *testing.T, method string) { defer os.RemoveAll(cacheDir) data, hash := testutils.RandomDataAndHash(0) - r, err := http.NewRequest(method, "/cas/"+hash, bytes.NewReader(data)) - if err != nil { - t.Fatal(err) - } + r := httptest.NewRequest(method, "/cas/"+hash, bytes.NewReader(data)) c, err := disk.New(cacheDir, 2048, "zstd", nil) if err != nil { @@ -352,10 +327,7 @@ func TestStatusPage(t *testing.T) { cacheDir := testutils.TempDir(t) defer os.RemoveAll(cacheDir) - r, err := http.NewRequest("GET", "/status", bytes.NewReader([]byte{})) - if err != nil { - t.Fatal(err) - } + r := httptest.NewRequest("GET", "/status", nil) c, err := disk.New(cacheDir, 2048, "zstd", nil) if err != nil { From 0bdc1b400cda1d8c0e446217c46a733d67fddf86 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Thu, 11 Feb 2021 21:23:12 +0100 Subject: [PATCH 26/35] [disk] rework the disk storage to avoid fs operations while holding the LRU lock By adding a psuedo-random string to the filename for blobs and storing that string in the index, we no longer need to rename files while holding the LRU index lock. To avoid incomplete files from being reused after restarting bazel-remote, we use a bit of a hack: create the blob file with the setgid bit set, and un-set it once the blob is complete. On startup, remove any files with the setgid bit set. --- cache/cache.go | 4 +- cache/disk/disk.go | 171 ++++++++++++++++++++++---------- cache/disk/disk_test.go | 8 +- cache/disk/lru.go | 6 ++ utils/tempfile/tempfile.go | 37 ++++--- utils/tempfile/tempfile_test.go | 16 ++- 6 files changed, 171 insertions(+), 71 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 5fd3da7fb..7c5281e66 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -34,12 +34,12 @@ func (e EntryKind) String() string { func (e EntryKind) DirName() string { if e == AC { - return "ac" + return "ac.v2" } if e == CAS { return "cas.v2" } - return "raw" + return "raw.v2" } // Logger is designed to be satisfied by log.Logger. diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 9c4669874..a6a6fff38 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -18,7 +18,6 @@ import ( "strconv" "strings" "sync" - "syscall" "github.com/buchgr/bazel-remote/cache" "github.com/buchgr/bazel-remote/cache/disk/casblob" @@ -48,7 +47,6 @@ var tfc = tempfile.NewCreator() var emptyZstdBlob = []byte{40, 181, 47, 253, 32, 0, 1, 0, 0} var hashKeyRegex = regexp.MustCompile("^[a-f0-9]{64}$") -var hashKeyWithSizeRegex = regexp.MustCompile("^([a-f0-9]{64})(?:-([1-9][0-9]*))?$") // lruItem is the type of the values stored in SizedLRU to keep track of items. type lruItem struct { @@ -58,6 +56,9 @@ type lruItem struct { // Size of the blob on disk (possibly with header + compression). sizeOnDisk int64 + // A random string (of digits, for now) that is included in the filename. + random string + // If true, the blob is a raw CAS file (no header, uncompressed) // with a ".v1" filename suffix. legacy bool @@ -131,12 +132,10 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) kind = cache.RAW } - f := filepath.Join(dir, c.FileLocation(kind, value.legacy, hash, value.size)) + f := filepath.Join(dir, c.FileLocation(kind, value.legacy, hash, value.size, value.random)) - err := os.Remove(f) - if err != nil { - log.Printf("ERROR: failed to remove evicted cache file: %s", f) - } + // Run in a goroutine so we can release the lock sooner. + go removeFile(f) } c.lru = NewSizedLRU(maxSizeBytes, onEvict) @@ -153,18 +152,45 @@ func New(dir string, maxSizeBytes int64, storageMode string, proxy cache.Proxy) return c, nil } -func (c *Cache) FileLocation(kind cache.EntryKind, legacy bool, hash string, size int64) string { - if kind != cache.CAS { - return path.Join(kind.String(), hash[:2], hash) +func removeFile(f string) { + err := os.Remove(f) + if err != nil { + log.Printf("ERROR: failed to remove evicted cache file: %s", f) } +} - if legacy || c.storageMode == casblob.Identity { - return fmt.Sprintf("cas.v2/%s/%s.v1", hash[:2], hash) +func (c *Cache) FileLocationBase(kind cache.EntryKind, legacy bool, hash string, size int64) string { + if kind == cache.RAW { + return path.Join("raw.v2", hash[:2], hash) + } + + if kind == cache.AC { + return path.Join("ac.v2", hash[:2], hash) + } + + if legacy { + return path.Join("cas.v2", hash[:2], hash) } return fmt.Sprintf("cas.v2/%s/%s-%d", hash[:2], hash, size) } +func (c *Cache) FileLocation(kind cache.EntryKind, legacy bool, hash string, size int64, random string) string { + if kind == cache.RAW { + return path.Join("raw.v2", hash[:2], hash+"-"+random) + } + + if kind == cache.AC { + return path.Join("ac.v2", hash[:2], hash+"-"+random) + } + + if legacy { + return fmt.Sprintf("cas.v2/%s/%s-%s.v1", hash[:2], hash, random) + } + + return fmt.Sprintf("cas.v2/%s/%s-%d-%s", hash[:2], hash, size, random) +} + func (c *Cache) migrateDirectories() error { err := migrateDirectory(c.dir, cache.AC) if err != nil { @@ -174,7 +200,10 @@ func (c *Cache) migrateDirectories() error { if err != nil { return err } - // Note: there are no old "RAW" directories (yet). + err = migrateDirectory(c.dir, cache.RAW) + if err != nil { + return err + } return nil } @@ -244,7 +273,10 @@ func migrateDirectory(baseDir string, kind cache.EntryKind) error { } src := filepath.Join(sourceDir, oldName) - dest := filepath.Join(targetDir, oldName[:2], oldName) + + // Add a not-so-random "random" string. This should be OK + // since there is probably only be one file for this hash. + dest := filepath.Join(targetDir, oldName[:2], oldName+"-222444666") if kind == cache.CAS { dest += ".v1" } @@ -304,10 +336,12 @@ func migrateV1Subdir(oldDir string, destDir string, kind cache.EntryKind) error return os.Remove(oldDir) } + var hashKeyRegexWithRandom = regexp.MustCompile("^([a-f0-9]{64})-([0-9]+)$") + for _, item := range listing { oldPath := path.Join(oldDir, item.Name()) - if !hashKeyRegex.MatchString(item.Name()) { + if !hashKeyRegexWithRandom.MatchString(item.Name()) { return fmt.Errorf("Unexpected file: %s", oldPath) } @@ -366,6 +400,9 @@ func migrateCASFile(src string, dest string, hash string) error { func (c *Cache) loadExistingFiles() error { log.Printf("Loading existing files in %s.\n", c.dir) + // --[".v1"] + re := regexp.MustCompile("^([a-f0-9]{64})(?:-([1-9][0-9]*))?-([0-9]+)(\\.v1)?$") + // Walk the directory tree var files []nameAndInfo err := filepath.Walk(c.dir, func(name string, info os.FileInfo, err error) error { @@ -374,9 +411,17 @@ func (c *Cache) loadExistingFiles() error { return err } - if !info.IsDir() { + if info.IsDir() { + return nil + } + + if info.Mode()&os.ModeSetgid == os.ModeSetgid { + log.Println("Removing incomplete file:", name) + os.Remove(name) + } else { files = append(files, nameAndInfo{name: name, info: info}) } + return nil }) if err != nil { @@ -393,19 +438,16 @@ func (c *Cache) loadExistingFiles() error { for _, f := range files { relPath := f.name[len(c.dir)+1:] - legacy := strings.HasSuffix(f.name, ".v1") - fields := strings.Split(relPath, "/") file := fields[len(fields)-1] - if legacy { - file = strings.TrimSuffix(file, ".v1") - } - sm := hashKeyWithSizeRegex.FindStringSubmatch(file) - if len(sm) == 0 || len(sm[1]) == 0 { - return fmt.Errorf("Unrecognized file: %q", file) + sm := re.FindStringSubmatch(file) + + if len(sm) != 5 { + return fmt.Errorf("Unrecognized file: %q", relPath) } + hash := sm[1] sizeOnDisk := f.info.Size() @@ -414,13 +456,20 @@ func (c *Cache) loadExistingFiles() error { size, err = strconv.ParseInt(sm[2], 10, 64) } + random := sm[3] + if len(random) == 0 { + return fmt.Errorf("Unrecognized file (no random string): %q", file) + } + + legacy := sm[4] == ".v1" + var lookupKey string if strings.HasPrefix(relPath, "cas.v2/") { lookupKey = "cas/" + hash - } else if strings.HasPrefix(relPath, "ac/") { + } else if strings.HasPrefix(relPath, "ac.v2/") { lookupKey = "ac/" + hash - } else if strings.HasPrefix(relPath, "raw/") { + } else if strings.HasPrefix(relPath, "raw.v2/") { lookupKey = "raw/" + hash } else { return fmt.Errorf("Unrecognised file in cache dir: %q", relPath) @@ -430,6 +479,7 @@ func (c *Cache) loadExistingFiles() error { size: size, sizeOnDisk: sizeOnDisk, legacy: legacy, + random: random, }) if !ok { err = os.Remove(filepath.Join(c.dir, relPath)) @@ -467,6 +517,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) key := cache.LookupKey(kind, hash) var tf *os.File // Tempfile. + var blobFile string // Cleanup intermediate state if something went wrong and we // did not successfully commit. @@ -475,7 +526,10 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) defer func() { // No lock required to remove stray tempfiles. if removeTempfile { - os.Remove(tf.Name()) + os.Remove(blobFile) + } else if blobFile != "" { + // Mark the file as "complete". + os.Chmod(blobFile, tempfile.EndMode) } if unreserve { @@ -515,10 +569,11 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) legacy := kind == cache.CAS && c.storageMode == casblob.Identity // Final destination, if all goes well. - filePath := path.Join(c.dir, c.FileLocation(kind, legacy, hash, size)) + filePath := path.Join(c.dir, c.FileLocationBase(kind, legacy, hash, size)) // We will download to this temporary file. - tf, err := tfc.Create(filePath) + tf, random, err := tfc.Create(filePath, legacy) + blobFile = tf.Name() if err != nil { return err } @@ -531,7 +586,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) } if c.proxy != nil { - rc, err := os.Open(tf.Name()) + rc, err := os.Open(blobFile) if err != nil { log.Println("Failed to proxy Put:", err) } else { @@ -540,7 +595,7 @@ func (c *Cache) Put(kind cache.EntryKind, hash string, size int64, r io.Reader) } } - unreserve, removeTempfile, err = c.commit(key, legacy, tf.Name(), filePath, size, size, sizeOnDisk) + unreserve, removeTempfile, err = c.commit(key, legacy, blobFile, size, size, sizeOnDisk, random) // Might be nil. return err @@ -588,7 +643,7 @@ func (c *Cache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string } // This must be called when the lock is not held. -func (c *Cache) commit(key string, legacy bool, tempfile string, finalPath string, reservedSize int64, logicalSize int64, sizeOnDisk int64) (unreserve bool, removeTempfile bool, err error) { +func (c *Cache) commit(key string, legacy bool, tempfile string, reservedSize int64, logicalSize int64, sizeOnDisk int64, random string) (unreserve bool, removeTempfile bool, err error) { unreserve = reservedSize > 0 removeTempfile = true @@ -604,21 +659,20 @@ func (c *Cache) commit(key string, legacy bool, tempfile string, finalPath strin } unreserve = false - if !c.lru.Add(key, lruItem{size: logicalSize, sizeOnDisk: sizeOnDisk, legacy: legacy}) { + newItem := lruItem{ + size: logicalSize, + sizeOnDisk: sizeOnDisk, + legacy: legacy, + random: random, + } + + if !c.lru.Add(key, newItem) { err = fmt.Errorf("INTERNAL ERROR: failed to add: %s, size %d (on disk: %d): %w", key, logicalSize, sizeOnDisk, err) log.Println(err.Error()) return unreserve, removeTempfile, err } - err = syscall.Rename(tempfile, finalPath) - if err != nil { - log.Printf("INTERNAL ERROR: failed to rename \"%s\" to \"%s\": %v", - tempfile, finalPath, err) - log.Println("Removing", key) - c.lru.Remove(key) - return unreserve, removeTempfile, err - } removeTempfile = false // Commit successful if we made it this far! \o/ @@ -640,14 +694,27 @@ func (c *Cache) availableOrTryProxy(kind cache.EntryKind, hash string, size int6 c.mu.Unlock() // We expect a cache hit below. locked = false - blobPath := path.Join(c.dir, c.FileLocation(kind, item.legacy, hash, item.size)) + blobPath := path.Join(c.dir, c.FileLocation(kind, item.legacy, hash, item.size, item.random)) if !isSizeMismatch(size, item.size) { var f *os.File f, err = os.Open(blobPath) + if err != nil && os.IsNotExist(err) { + // Another request replaced the file before we could open it? + // Enter slow path. + + c.mu.Lock() + item, available = c.lru.Get(key) + if available { + blobPath = path.Join(c.dir, c.FileLocation(kind, item.legacy, hash, item.size, item.random)) + f, err = os.Open(blobPath) + } + c.mu.Unlock() + } + if err != nil { // Race condition, was the item purged after we released the lock? - log.Printf("Warning: expected %s to exist on disk, undersized cache?", blobPath) + log.Printf("Warning: expected %q to exist on disk, undersized cache?", blobPath) } else if kind == cache.CAS { if item.legacy { // The file is uncompressed, without a casblob header. @@ -764,6 +831,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, key := cache.LookupKey(kind, hash) var tf *os.File // Tempfile we will write to. + var blobFile string // Cleanup intermediate state if something went wrong and we // did not successfully commit. @@ -772,7 +840,10 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, defer func() { // No lock required to remove stray tempfiles. if removeTempfile { - os.Remove(tf.Name()) + os.Remove(blobFile) + } else if blobFile != "" { + // Mark the file as "complete". + os.Chmod(blobFile, tempfile.EndMode) } if unreserve { @@ -819,14 +890,14 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, legacy := kind == cache.CAS && c.storageMode == casblob.Identity - blobPath := path.Join(c.dir, c.FileLocation(kind, legacy, hash, foundSize)) - tf, err = tfc.Create(blobPath) + blobPathBase := path.Join(c.dir, c.FileLocationBase(kind, legacy, hash, foundSize)) + tf, random, err := tfc.Create(blobPathBase, legacy) if err != nil { return nil, -1, err } removeTempfile = true - tfName := tf.Name() + blobFile = tf.Name() var sizeOnDisk int64 sizeOnDisk, err = io.Copy(tf, r) @@ -838,7 +909,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, logicalSize := foundSize if kind == cache.CAS && c.storageMode != casblob.Identity { - logicalSize, err = casblob.GetLogicalSize(tfName) + logicalSize, err = casblob.GetLogicalSize(blobFile) if err != nil { return nil, -1, err } @@ -847,7 +918,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, } } - rcf, err := os.Open(tfName) + rcf, err := os.Open(blobFile) if err != nil { return nil, -1, err } @@ -874,7 +945,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, return nil, -1, err } - unreserve, removeTempfile, err = c.commit(key, legacy, tfName, blobPath, size, logicalSize, sizeOnDisk) + unreserve, removeTempfile, err = c.commit(key, legacy, blobFile, size, logicalSize, sizeOnDisk, random) if err != nil { rc.Close() rc = nil diff --git a/cache/disk/disk_test.go b/cache/disk/disk_test.go index 9bee14d41..9fb1b4ad5 100644 --- a/cache/disk/disk_test.go +++ b/cache/disk/disk_test.go @@ -343,25 +343,25 @@ func TestCacheExistingFiles(t *testing.T) { "hej", "9c478bf63e9500cb5db1e85ece82f18c8eb9e52e2f9135acd7f10972c8d563ba", "cas/9c478bf63e9500cb5db1e85ece82f18c8eb9e52e2f9135acd7f10972c8d563ba", - "cas.v2/9c/9c478bf63e9500cb5db1e85ece82f18c8eb9e52e2f9135acd7f10972c8d563ba", + "cas.v2/9c/9c478bf63e9500cb5db1e85ece82f18c8eb9e52e2f9135acd7f10972c8d563ba-3-123456789", }, { "världen", "d497feaa39156f4ae61317db9d2adc3a8f2ff1437fd48ccb56f814f0b7ac5fe1", "cas/d497feaa39156f4ae61317db9d2adc3a8f2ff1437fd48ccb56f814f0b7ac5fe1", - "cas.v2/d4/d497feaa39156f4ae61317db9d2adc3a8f2ff1437fd48ccb56f814f0b7ac5fe1", + "cas.v2/d4/d497feaa39156f4ae61317db9d2adc3a8f2ff1437fd48ccb56f814f0b7ac5fe1-8-123456789", }, { "foo", "733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", "ac/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", - "ac/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", + "ac/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54-123456789", }, { "bar", "733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", "raw/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", - "raw/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54", + "raw/73/733e21b37cef883579a88183eed0d00cdeea0b59e1bcd77db6957f881c3a6b54-123456789", }, } diff --git a/cache/disk/lru.go b/cache/disk/lru.go index 93c05a7e8..f8984be09 100644 --- a/cache/disk/lru.go +++ b/cache/disk/lru.go @@ -104,6 +104,12 @@ func (c *SizedLRU) Add(key Key, value lruItem) (ok bool) { uncompressedSizeDelta = value.size - ee.Value.(*entry).value.size c.ll.MoveToFront(ee) counterOverwrittenBytes.Add(float64(ee.Value.(*entry).value.sizeOnDisk)) + + prevValue := ee.Value.(*entry).value + if c.onEvict != nil { + c.onEvict(key, prevValue) + } + ee.Value.(*entry).value = value } else { sizeDelta = value.sizeOnDisk diff --git a/utils/tempfile/tempfile.go b/utils/tempfile/tempfile.go index 370b1fa7c..3efecbb06 100644 --- a/utils/tempfile/tempfile.go +++ b/utils/tempfile/tempfile.go @@ -33,25 +33,38 @@ func (c *Creator) ranqd1() string { } const flags = os.O_RDWR | os.O_CREATE | os.O_EXCL -const mode = 0666 // Before the umask is applied. -// Create attempts to create a temp file for eventualFile, and returns -// the corresponding *os.File and an error if any occurred. The temp -// file is created in the same directory as eventualFile, with permissions -// 0666 before the umask is applied. -func (c *Creator) Create(eventualFile string) (*os.File, error) { +const EndMode = 0666 +const wipMode = EndMode | os.ModeSetgid + +// Create attempts to create a file whose name is of the form +// - and with a ".v1" suffix if `legacy` is +// true. The file will be created with the setgid bit set, which +// indicates that it is not complete. The *os.File is returned +// along with the random string, and an error if something went +// wrong. +// +// Once the file has been successfully written by the caller, it +// should be chmod'ed to `EndMode` to mark it as complete. +func (c *Creator) Create(base string, legacy bool) (*os.File, string, error) { var err error var f *os.File - - base := eventualFile + ".tmp" + var name string + var random string for i := 0; i < 10000; i++ { - name := base + c.ranqd1() - f, err = os.OpenFile(name, flags, mode) + random = c.ranqd1() + if legacy { + name = base + "-" + random + ".v1" + } else { + name = base + "-" + random + } + + f, err = os.OpenFile(name, flags, wipMode) if os.IsExist(err) { continue } - return f, err + return f, random, err } - return nil, err + return nil, "", err } diff --git a/utils/tempfile/tempfile_test.go b/utils/tempfile/tempfile_test.go index c6ea40d12..e758500c8 100644 --- a/utils/tempfile/tempfile_test.go +++ b/utils/tempfile/tempfile_test.go @@ -19,14 +19,24 @@ func TestTempfileCreator(t *testing.T) { } defer os.RemoveAll(dir) - targetFile := path.Join(dir, "foo") - tf, err := tfc.Create(targetFile) + targetFileBase := path.Join(dir, "foo") + tf, random, err := tfc.Create(targetFileBase, false) if err != nil { t.Fatal(err) } defer os.Remove(tf.Name()) - expectedPrefix := targetFile + "." + if random == "" { + t.Fatalf("Expected non-empty random string in the filename: %q", + tf.Name()) + } + + if !strings.Contains(tf.Name(), random) { + t.Fatalf("Expected filename %q to contain random string %q", + tf.Name(), random) + } + + expectedPrefix := targetFileBase if !strings.HasPrefix(tf.Name(), expectedPrefix) { t.Fatalf("Expected tempfile \"%s\" to have prefix \"%s\"", tf.Name(), expectedPrefix) From 63ba6fc760cd99a65d7e5161c80dd898f00f303a Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 20 Feb 2021 00:34:51 +0100 Subject: [PATCH 27/35] [casblob] sanity check file size --- cache/disk/casblob/casblob.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index cb84d41d1..8ef14a2df 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -101,6 +101,11 @@ func readHeader(f *os.File) (*header, error) { return nil, err } foundFileSize := fileInfo.Size() + if foundFileSize <= (chunkTableOffset + 16) { + // The file must have a header with at least two chunkOffsets. + return nil, fmt.Errorf("file too small (%d) than the minimum header size (%d)", + foundFileSize, (chunkTableOffset + 16)) + } var magicNumber uint32 err = binary.Read(f, binary.LittleEndian, &magicNumber) From b12351e8659ca023de9932b787e532ab395171b8 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 20 Feb 2021 00:37:16 +0100 Subject: [PATCH 28/35] [casblob] add note that we never store empty files --- cache/disk/casblob/casblob.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index 8ef14a2df..a544f87d8 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -145,6 +145,7 @@ func readHeader(f *os.File) (*header, error) { if numOffsets < 2 { // chunkOffsets has an extra entry to specify the compressed file size. + // Note that we never store empty files. return nil, fmt.Errorf("internal error: need at least one chunk, found %d", numOffsets-1) } From 605d444713160d49b53786842e3b5603eccaad84 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 20 Feb 2021 00:44:19 +0100 Subject: [PATCH 29/35] [casblob] add missing Close on error conditions --- cache/disk/casblob/casblob.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index a544f87d8..8dec0493a 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -205,6 +205,7 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io } if expectedSize != -1 && h.uncompressedSize != expectedSize { + f.Close() return nil, fmt.Errorf("expected a blob of size %d, found %d", expectedSize, h.uncompressedSize) } @@ -312,6 +313,7 @@ func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadClo } if expectedSize != -1 && h.uncompressedSize != expectedSize { + f.Close() return nil, fmt.Errorf("expected a blob of size %d, found %d", expectedSize, h.uncompressedSize) } From c24f86183583a7d61bff65ca23db52dcac50be11 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 20 Feb 2021 00:47:18 +0100 Subject: [PATCH 30/35] [casblob] make the GetUncompressedReadCloser and GetZstdReadCloser comments more explicit --- cache/disk/casblob/casblob.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index 8dec0493a..4d7d59ca1 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -195,8 +195,10 @@ func GetLogicalSize(filename string) (int64, error) { return hdr.uncompressedSize, nil } -// Closes f if there is an error. Otherwise the caller must Close the returned -// io.ReadCloser. +// Returns an io.ReadCloser that provides uncompressed data. The caller +// must close the returned io.ReadCloser if it is non-nil. Doing so +// will automatically close f. If there is an error f will be closed, the caller +// does not need to do so. func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) { h, err := readHeader(f) if err != nil { @@ -302,8 +304,10 @@ func GetUncompressedReadCloser(f *os.File, expectedSize int64, offset int64) (io }, nil } -// Closes f if there is an error. Otherwise the caller must Close the returned -// io.ReadCloser. +// Returns an io.ReadCloser that provides zstandard compressed data. The +// caller must close the returned io.ReadCloser if it is non-nil. Doing so +// will automatically close f. If there is an error f will be closed, the caller +// does not need to do so. func GetZstdReadCloser(f *os.File, expectedSize int64, offset int64) (io.ReadCloser, error) { h, err := readHeader(f) From ed9901a3108f72d975daafa2b5850901319bede5 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 20 Feb 2021 01:20:22 +0100 Subject: [PATCH 31/35] [casblob] GetLegacyZstdReadCloser should pass errors to readers If there is an error while reading the input file, we should pass that error on to readers so they know not to use incomplete files. --- cache/disk/casblob/casblob.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index 4d7d59ca1..266be3823 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -403,8 +403,10 @@ func GetLegacyZstdReadCloser(f *os.File) (io.ReadCloser, error) { // TODO: consider implementing something with a timeout? _, err := enc.ReadFrom(f) if err != nil { - // We can't do anything here except log an error. - log.Println("Error while compressing file:", err) + log.Println("Error while reading/compressing file:", err) + + // Reading from pr will now receive this error: + pw.CloseWithError(err) } encoderPool.Put(enc) From 311fb8e4f34653a5030249f06401a55b5e7676b1 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sat, 20 Feb 2021 21:34:05 +0100 Subject: [PATCH 32/35] [casblob] read and write chunk offset table with one binary.Read/Write call --- cache/disk/casblob/casblob.go | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index 266be3823..f3f4d6de4 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -155,14 +155,14 @@ func readHeader(f *os.File) (*header, error) { frameSize, metadataSize) } - prevOffset := int64(-1) h.chunkOffsets = make([]int64, numOffsets, numOffsets) - for i := 0; int64(i) < numOffsets; i++ { - err = binary.Read(f, binary.LittleEndian, &h.chunkOffsets[i]) - if err != nil { - return nil, err - } + err = binary.Read(f, binary.LittleEndian, h.chunkOffsets) + if err != nil { + return nil, err + } + prevOffset := int64(-1) + for i := 0; int64(i) < numOffsets; i++ { if h.chunkOffsets[i] <= prevOffset { return nil, fmt.Errorf("offset table values should increase: %d -> %d", @@ -449,20 +449,7 @@ func (h *header) write(f *os.File) error { return err } - return h.writeChunkTable(f) -} - -func (h *header) writeChunkTable(f *os.File) error { - var err error - - for _, o := range h.chunkOffsets { - err = binary.Write(f, binary.LittleEndian, o) - if err != nil { - return err - } - } - - return nil + return binary.Write(f, binary.LittleEndian, h.chunkOffsets) } func (b *readCloserWrapper) Read(p []byte) (int, error) { @@ -597,7 +584,7 @@ func WriteAndClose(r io.Reader, f *os.File, t CompressionType, hash string, size return -1, err } - err = h.writeChunkTable(f) + err = binary.Write(f, binary.LittleEndian, h.chunkOffsets) if err != nil { return -1, err } From ff4c85319782a267d6d8269bd329be02d5282557 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sun, 21 Feb 2021 00:44:23 +0100 Subject: [PATCH 33/35] [casblob] proxy backends should return the logical size for Get We can peek at the first part of the data to figure out the logical size, then let the disk cache layer decide if it wants to use the blob or not. --- cache/disk/casblob/casblob.go | 42 +++++++++++++++++++++++++++++------ cache/disk/disk.go | 24 +++++--------------- cache/httpproxy/BUILD.bazel | 1 + cache/httpproxy/httpproxy.go | 10 ++++++++- cache/s3proxy/BUILD.bazel | 1 + cache/s3proxy/s3proxy.go | 11 ++++++--- 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/cache/disk/casblob/casblob.go b/cache/disk/casblob/casblob.go index f3f4d6de4..edd09bbb7 100644 --- a/cache/disk/casblob/casblob.go +++ b/cache/disk/casblob/casblob.go @@ -180,19 +180,47 @@ func readHeader(f *os.File) (*header, error) { return &h, nil } -func GetLogicalSize(filename string) (int64, error) { - f, err := os.Open(filename) +// Extract the logical size of a v2 cas blob from rc, and return that +// size along with an equivalent io.ReadCloser to rc. +func ExtractLogicalSize(rc io.ReadCloser) (io.ReadCloser, int64, error) { + + // Read the first part of the header: magic number (4 bytes), + // frame size (4 bytes), uncompressed size (8 bytes). + interesting := 16 + earlyHeader := make([]byte, interesting, interesting) + + n, err := io.ReadFull(rc, earlyHeader) if err != nil { - return -1, err + return nil, -1, err + } + if n != 16 { + return nil, -1, fmt.Errorf("Tried to read 16 header bytes, only read %d", n) } - defer f.Close() - hdr, err := readHeader(f) + var uncompressedSize int64 + br := bytes.NewReader(earlyHeader[8:]) + err = binary.Read(br, binary.LittleEndian, &uncompressedSize) if err != nil { - return -1, fmt.Errorf("Failed to read %s: %w", filename, err) + return nil, -1, err } + if uncompressedSize <= 0 { + return nil, -1, fmt.Errorf("Expected blob to have positive size, found %d", + uncompressedSize) + } + + return &multiReadCloser{ + Reader: io.MultiReader(bytes.NewReader(earlyHeader), rc), + rc: rc, + }, uncompressedSize, nil +} + +type multiReadCloser struct { + io.Reader // This will be a MultiReader. + rc io.ReadCloser +} - return hdr.uncompressedSize, nil +func (m *multiReadCloser) Close() error { + return m.rc.Close() } // Returns an io.ReadCloser that provides uncompressed data. The caller diff --git a/cache/disk/disk.go b/cache/disk/disk.go index a6a6fff38..328c39d8e 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -876,7 +876,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, return nil, -1, nil } - r, foundSize, err := c.proxy.Get(kind, hash) + r, foundLogicalSize, err := c.proxy.Get(kind, hash) if r != nil { defer r.Close() } @@ -884,7 +884,7 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, return nil, -1, err } - if kind != cache.CAS && isSizeMismatch(size, foundSize) { + if isSizeMismatch(size, foundLogicalSize) { return nil, -1, nil } @@ -906,18 +906,6 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, return nil, -1, err } - logicalSize := foundSize - - if kind == cache.CAS && c.storageMode != casblob.Identity { - logicalSize, err = casblob.GetLogicalSize(blobFile) - if err != nil { - return nil, -1, err - } - if isSizeMismatch(size, logicalSize) { - return nil, -1, nil - } - } - rcf, err := os.Open(blobFile) if err != nil { return nil, -1, err @@ -936,23 +924,23 @@ func (c *Cache) get(kind cache.EntryKind, hash string, size int64, offset int64, } } else { // Compressed CAS blob. if zstd { - rc, err = casblob.GetZstdReadCloser(rcf, logicalSize, offset) + rc, err = casblob.GetZstdReadCloser(rcf, foundLogicalSize, offset) } else { - rc, err = casblob.GetUncompressedReadCloser(rcf, logicalSize, offset) + rc, err = casblob.GetUncompressedReadCloser(rcf, foundLogicalSize, offset) } } if err != nil { return nil, -1, err } - unreserve, removeTempfile, err = c.commit(key, legacy, blobFile, size, logicalSize, sizeOnDisk, random) + unreserve, removeTempfile, err = c.commit(key, legacy, blobFile, size, foundLogicalSize, sizeOnDisk, random) if err != nil { rc.Close() rc = nil foundSize = -1 } - return rc, logicalSize, err + return rc, foundLogicalSize, err } // Contains returns true if the `hash` key exists in the cache, and diff --git a/cache/httpproxy/BUILD.bazel b/cache/httpproxy/BUILD.bazel index 2c9a6be6e..6d8c89bbe 100644 --- a/cache/httpproxy/BUILD.bazel +++ b/cache/httpproxy/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//cache:go_default_library", + "//cache/disk/casblob:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", ], diff --git a/cache/httpproxy/httpproxy.go b/cache/httpproxy/httpproxy.go index ff9c775a7..baee1120a 100644 --- a/cache/httpproxy/httpproxy.go +++ b/cache/httpproxy/httpproxy.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/buchgr/bazel-remote/cache" + "github.com/buchgr/bazel-remote/cache/disk/casblob" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -32,6 +33,7 @@ type remoteHTTPProxyCache struct { accessLogger cache.Logger errorLogger cache.Logger requestURL func(hash string, kind cache.EntryKind) string + v2mode bool } var ( @@ -95,6 +97,7 @@ func New(baseURL *url.URL, storageMode string, remote *http.Client, baseURL: strings.TrimRight(baseURL.String(), "/"), accessLogger: accessLogger, errorLogger: errorLogger, + v2mode: storageMode == "zstd", } if storageMode == "zstd" { @@ -189,6 +192,11 @@ func (r *remoteHTTPProxyCache) Get(kind cache.EntryKind, hash string) (io.ReadCl } } + if kind == cache.CAS && r.v2mode { + cacheHits.Inc() + return casblob.ExtractLogicalSize(rsp.Body) + } + sizeBytesStr := rsp.Header.Get("Content-Length") if sizeBytesStr == "" { err = errors.New("Missing Content-Length header") @@ -205,7 +213,7 @@ func (r *remoteHTTPProxyCache) Get(kind cache.EntryKind, hash string) (io.ReadCl cacheHits.Inc() - return rsp.Body, sizeBytes, err + return rsp.Body, sizeBytes, nil } func (r *remoteHTTPProxyCache) Contains(kind cache.EntryKind, hash string) (bool, int64) { diff --git a/cache/s3proxy/BUILD.bazel b/cache/s3proxy/BUILD.bazel index 14612eb2a..fa2f91d27 100644 --- a/cache/s3proxy/BUILD.bazel +++ b/cache/s3proxy/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//cache:go_default_library", + "//cache/disk/casblob:go_default_library", "//config:go_default_library", "@com_github_minio_minio_go_v7//:go_default_library", "@com_github_minio_minio_go_v7//pkg/credentials:go_default_library", diff --git a/cache/s3proxy/s3proxy.go b/cache/s3proxy/s3proxy.go index 66b44c6db..dd04d20bd 100644 --- a/cache/s3proxy/s3proxy.go +++ b/cache/s3proxy/s3proxy.go @@ -9,6 +9,7 @@ import ( "path" "github.com/buchgr/bazel-remote/cache" + "github.com/buchgr/bazel-remote/cache/disk/casblob" "github.com/buchgr/bazel-remote/config" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -209,7 +210,7 @@ func (c *s3Cache) Put(kind cache.EntryKind, hash string, size int64, rc io.ReadC func (c *s3Cache) Get(kind cache.EntryKind, hash string) (io.ReadCloser, int64, error) { - object, info, _, err := c.mcore.GetObject( + rc, info, _, err := c.mcore.GetObject( context.Background(), c.bucket, // bucketName c.objectKey(hash, kind), // objectName @@ -229,7 +230,11 @@ func (c *s3Cache) Get(kind cache.EntryKind, hash string) (io.ReadCloser, int64, logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), nil) - return object, info.Size, nil + if kind == cache.CAS && c.v2mode { + return casblob.ExtractLogicalSize(rc) + } + + return rc, info.Size, nil } func (c *s3Cache) Contains(kind cache.EntryKind, hash string) (bool, int64) { @@ -246,7 +251,7 @@ func (c *s3Cache) Contains(kind cache.EntryKind, hash string) (bool, int64) { exists = (err == nil) if err != nil { err = errNotFound - } else if kind != cache.CAS { + } else if kind != cache.CAS || !c.v2mode { size = s.Size } From 564d20e43fe180c66f5fee214457b1d9f8bf09f2 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Sun, 21 Feb 2021 00:45:47 +0100 Subject: [PATCH 34/35] [casblob] test a basic assumption about the sizeof int --- cache/disk/casblob/BUILD.bazel | 8 +++++++- cache/disk/casblob/casblob_test.go | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 cache/disk/casblob/casblob_test.go diff --git a/cache/disk/casblob/BUILD.bazel b/cache/disk/casblob/BUILD.bazel index d181a70b9..804466665 100644 --- a/cache/disk/casblob/BUILD.bazel +++ b/cache/disk/casblob/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -11,3 +11,9 @@ go_library( "@com_github_mostynb_zstdpool_syncpool//:go_default_library", ], ) + +go_test( + name = "go_default_test", + srcs = ["casblob_test.go"], + embed = [":go_default_library"], +) diff --git a/cache/disk/casblob/casblob_test.go b/cache/disk/casblob/casblob_test.go new file mode 100644 index 000000000..b3e39af5d --- /dev/null +++ b/cache/disk/casblob/casblob_test.go @@ -0,0 +1,15 @@ +package casblob_test + +import ( + "testing" + "unsafe" +) + +func TestLenSize(t *testing.T) { + slice := []int{} + if unsafe.Sizeof(len(slice)) > 8 { + // If this fails, then we have a bunch of potential truncation + // errors all over the place. + t.Errorf("len() returns a value larger than 8 bytes") + } +} From 48e8649a433a54c599e875b4d54e0d18e0dfed94 Mon Sep 17 00:00:00 2001 From: Mostyn Bramley-Moore Date: Fri, 26 Feb 2021 20:38:35 +0100 Subject: [PATCH 35/35] [disk] accept ascii characters in random string of blob filenames --- cache/disk/disk.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cache/disk/disk.go b/cache/disk/disk.go index 328c39d8e..51ef149d5 100644 --- a/cache/disk/disk.go +++ b/cache/disk/disk.go @@ -400,8 +400,10 @@ func migrateCASFile(src string, dest string, hash string) error { func (c *Cache) loadExistingFiles() error { log.Printf("Loading existing files in %s.\n", c.dir) - // --[".v1"] - re := regexp.MustCompile("^([a-f0-9]{64})(?:-([1-9][0-9]*))?-([0-9]+)(\\.v1)?$") + // compressed CAS items: -- + // uncompressed CAS items: --.v1 + // AC and RAW items: - + re := regexp.MustCompile("^([a-f0-9]{64})(?:-([1-9][0-9]*))?-([0-9a-zA-Z]+)(\\.v1)?$") // Walk the directory tree var files []nameAndInfo