diff --git a/pkg/x/http/rangecontrol/benchmark_test.go b/pkg/x/http/rangecontrol/benchmark_test.go new file mode 100644 index 0000000..52bbf4e --- /dev/null +++ b/pkg/x/http/rangecontrol/benchmark_test.go @@ -0,0 +1,84 @@ +package rangecontrol + +import "testing" + +var ( + benchmarkByteRanges []ByteRange + benchmarkContentRangeString string +) + +func BenchmarkParseRange(b *testing.B) { + cases := []struct { + name string + value string + size int64 + }{ + {"single", "bytes=0-499", 1 << 20}, + {"open_ended", "bytes=500-", 1 << 20}, + {"suffix", "bytes=-500", 1 << 20}, + {"multi", "bytes=0-499,500-999,-500", 1 << 20}, + {"spaced_multi", "bytes=0-0, 1-1, 1048570-", 1 << 20}, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + ranges, err := ParseRange(tc.value, tc.size) + if err != nil { + b.Fatal(err) + } + benchmarkByteRanges = ranges + } + }) + } +} + +func BenchmarkParse(b *testing.B) { + cases := []struct { + name string + value string + size uint64 + }{ + {"single", "bytes=0-499", 1 << 20}, + {"open_ended", "bytes=500-", 1 << 20}, + {"suffix", "bytes=-500", 1 << 20}, + {"multi", "bytes=0-499,500-999,-500", 1 << 20}, + {"spaced_multi", "bytes=0-0, 1-1, 1048570-", 1 << 20}, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + ranges, err := Parse(tc.value, tc.size) + if err != nil { + b.Fatal(err) + } + benchmarkByteRanges = ranges + } + }) + } +} + +func BenchmarkContentRange(b *testing.B) { + cases := []struct { + name string + r ByteRange + size uint64 + }{ + {"closed", ByteRange{Start: 0, End: 499}, 1 << 20}, + {"open_ended", ByteRange{Start: 500, End: -1}, 1 << 20}, + {"clamped", ByteRange{Start: 0, End: 2 << 20}, 1 << 20}, + {"unknown_size", ByteRange{Start: 0, End: 499}, 0}, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + benchmarkContentRangeString = tc.r.ContentRange(tc.size) + } + }) + } +} diff --git a/pkg/x/http/rangecontrol/request_range.go b/pkg/x/http/rangecontrol/request_range.go index c4e4efe..3094cce 100644 --- a/pkg/x/http/rangecontrol/request_range.go +++ b/pkg/x/http/rangecontrol/request_range.go @@ -50,7 +50,7 @@ func (r ByteRange) MimeHeader(contentType string, size uint64) textproto.MIMEHea } // Parse parses a Range header and returns. -func Parse(rangeHeader string) ([]ByteRange, error) { +func Parse(rangeHeader string, totalSize uint64) ([]ByteRange, error) { if rangeHeader == "" { return nil, nil } @@ -67,11 +67,6 @@ func Parse(rangeHeader string) ([]ByteRange, error) { for _, part := range parts { part = strings.TrimSpace(part) - // suffix-range: "-500" (not supported without total size) - if strings.HasPrefix(part, "-") { - return nil, ErrInvalidRange - } - dash := strings.IndexByte(part, '-') if dash < 0 { return nil, ErrInvalidRange @@ -80,6 +75,26 @@ func Parse(rangeHeader string) ([]ByteRange, error) { startStr := part[:dash] endStr := part[dash+1:] + // suffix-range: "-500" + if strings.HasPrefix(part, "-") { + end, err := strconv.ParseInt(endStr, 10, 64) + if err != nil || end < 0 { + return nil, ErrInvalidRange + } + + start := int64(totalSize) - end + // If totalSize is unknown (0) or suffix exceeds totalSize, skip + if start < 0 { + continue + } + + ranges = append(ranges, ByteRange{ + Start: start, + End: int64(totalSize) - 1, + }) + continue + } + start, err := strconv.ParseInt(startStr, 10, 64) if err != nil || start < 0 { return nil, ErrInvalidRange @@ -88,7 +103,8 @@ func Parse(rangeHeader string) ([]ByteRange, error) { var end int64 if endStr == "" { // open-ended range: "100-" - end = -1 + // end = -1 + end = int64(totalSize) - 1 } else { end, err = strconv.ParseInt(endStr, 10, 64) if err != nil || end < start { @@ -105,6 +121,92 @@ func Parse(rangeHeader string) ([]ByteRange, error) { return ranges, nil } +var ( + // ErrNoOverlap is returned by ParseRange if first-byte-pos of + // all of the byte-range-spec values is greater than the content size. + ErrNoOverlap = errors.New("invalid range: failed to overlap") + + // ErrInvalid is returned by ParseRange on invalid input. + ErrInvalid = errors.New("invalid range") +) + +// ParseRange parses a Range header string as per RFC 7233. +// ErrNoOverlap is returned if none of the ranges overlap. +// ErrInvalid is returned if s is invalid range. +func ParseRange(s string, size int64) ([]ByteRange, error) { // nolint:gocognit + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, ErrInvalid + } + var ranges []ByteRange + noOverlap := false + for _, ra := range strings.Split(s[len(b):], ",") { + ra = textproto.TrimString(ra) + if ra == "" { + continue + } + i := strings.Index(ra, "-") + if i < 0 { + return nil, ErrInvalid + } + start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:]) + var r ByteRange + if start == "" { + // If no start is specified, end specifies the + // range start relative to the end of the file, + // and we are dealing with + // which has to be a non-negative integer as per + // RFC 7233 Section 2.1 "Byte-Ranges". + if end == "" || end[0] == '-' { + return nil, ErrInvalid + } + i, err := strconv.ParseInt(end, 10, 64) + if i < 0 || err != nil { + return nil, ErrInvalid + } + if i > size { + i = size + } + r.Start = size - i + r.End = size - r.Start + } else { + i, err := strconv.ParseInt(start, 10, 64) + if err != nil || i < 0 { + return nil, ErrInvalid + } + if i >= size { + // If the range begins after the size of the content, + // then it does not overlap. + noOverlap = true + continue + } + r.Start = i + if end == "" { + // If no end is specified, range extends to end of the file. + r.End = size - r.Start + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || r.Start > i { + return nil, ErrInvalid + } + if i >= size { + i = size - 1 + } + r.End = i - r.Start + 1 + } + } + ranges = append(ranges, r) + } + if noOverlap && len(ranges) == 0 { + // The specified ranges did not overlap with the content. + return nil, ErrNoOverlap + } + return ranges, nil +} + func SortRanges(ranges []ByteRange) { sort.Slice(ranges, func(i, j int) bool { if ranges[i].Start != ranges[j].Start { @@ -127,6 +229,8 @@ func MergeRanges(ranges []ByteRange) []ByteRange { return ranges } + SortRanges(ranges) + merged := make([]ByteRange, 0, len(ranges)) cur := ranges[0] diff --git a/pkg/x/http/rangecontrol/request_range_test.go b/pkg/x/http/rangecontrol/request_range_test.go index 57ff1fc..01fade8 100644 --- a/pkg/x/http/rangecontrol/request_range_test.go +++ b/pkg/x/http/rangecontrol/request_range_test.go @@ -15,19 +15,21 @@ func TestParse(t *testing.T) { {"empty header", "", nil, false}, {"valid single closed", "bytes=0-499", []ByteRange{{Start: 0, End: 499}}, false}, {"valid multiple", "bytes=0-0, 1-1", []ByteRange{{Start: 0, End: 0}, {Start: 1, End: 1}}, false}, - {"open-ended", "bytes=100-", []ByteRange{{Start: 100, End: -1}}, false}, + {"open-ended with known size", "bytes=100-", []ByteRange{{Start: 100, End: 1048575}}, false}, {"spaces and multiple", "bytes=0-10,20-30", []ByteRange{{Start: 0, End: 10}, {Start: 20, End: 30}}, false}, {"invalid prefix", "byt=0-1", nil, true}, - {"suffix-range not supported", "bytes=-500", nil, true}, + // {"suffix-range not supported", "bytes=-500", nil, true}, {"no dash", "bytes=500", nil, true}, {"non-numeric start", "bytes=a-5", nil, true}, {"end less than start", "bytes=10-5", nil, true}, + {"suffix-range and closed range", "bytes=0-499,-500", []ByteRange{{Start: 0, End: 499}, {Start: 1048076, End: 1048575}}, false}, } + totalSize := uint64(1048576) for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - got, err := Parse(tc.header) + got, err := Parse(tc.header, totalSize) if tc.wantErr { if err == nil { t.Fatalf("expected error but got nil") diff --git a/server/middleware/caching/caching_fillrange.go b/server/middleware/caching/caching_fillrange.go index 445cb80..a7788de 100644 --- a/server/middleware/caching/caching_fillrange.go +++ b/server/middleware/caching/caching_fillrange.go @@ -6,6 +6,7 @@ import ( "math" "net/http" "strconv" + "strings" "github.com/omalloc/tavern/internal/protocol" "github.com/omalloc/tavern/pkg/iobuf" @@ -117,6 +118,9 @@ func (f *fillRange) fill(c *Caching, req *http.Request, rawRange string) *http.R if c.md != nil { objSize = c.md.Size } + if objSize == uint64(math.MaxInt64) && hasUnresolvableRange(rawRange) { + return req + } rng, err := xhttp.SingleRange(rawRange, objSize) if err != nil { @@ -188,6 +192,35 @@ func (f *fillRange) fill(c *Caching, req *http.Request, rawRange string) *http.R return req.WithContext(context.WithValue(req.Context(), fillRangeKey{}, fill)) } +// hasUnresolvableRange returns true if the range header contains suffix ranges +// or open-ended ranges that cannot be resolved without knowing the object size. +func hasUnresolvableRange(rawRange string) bool { + const prefix = "bytes=" + if !strings.HasPrefix(rawRange, prefix) { + return false + } + + for _, part := range strings.Split(strings.TrimPrefix(rawRange, prefix), ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + dash := strings.IndexByte(part, '-') + if dash < 0 { + continue + } + // Suffix range: starts with "-" (e.g., "-512") + if dash == 0 { + return true + } + // Open-ended range: ends with "-" (e.g., "100-") + if dash == len(part)-1 { + return true + } + } + return false +} + func parseFillPercent(h http.Header, def uint64) uint64 { fp := h.Get(protocol.InternalFillRangePercent) if fp != "" { diff --git a/server/middleware/caching/caching_fillrange_test.go b/server/middleware/caching/caching_fillrange_test.go new file mode 100644 index 0000000..66a4d07 --- /dev/null +++ b/server/middleware/caching/caching_fillrange_test.go @@ -0,0 +1,105 @@ +package caching + +import ( + "io" + "net/http" + "testing" + + "github.com/omalloc/tavern/api/defined/v1/storage/object" + "github.com/omalloc/tavern/contrib/log" +) + +func newTestFillRange() *fillRange { + return NewFillRangeProcessor( + WithFillRangePercent(100), + WithChunkSize(256), + ).(*fillRange) +} + +func newTestCaching(md *object.Metadata) *Caching { + return &Caching{ + log: log.NewHelper(log.NewStdLogger(io.Discard)), + md: md, + } +} + +func newFillRangeRequest(t *testing.T, rawRange string) *http.Request { + t.Helper() + + req, err := http.NewRequest(http.MethodGet, "http://example.com/file", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", rawRange) + return req +} + +func TestFillRangeUnknownSizeUnresolvableRangePassesThrough(t *testing.T) { + tests := []string{ + // suffix ranges + "bytes=-512", + "bytes= -512", + "bytes=0-99,-512", + "bytes=0-99, -512", + // open-ended ranges + "bytes=100-", + "bytes=0-", + // mixed open-ended + "bytes=0-99,200-", + } + + for _, rawRange := range tests { + t.Run(rawRange, func(t *testing.T) { + f := newTestFillRange() + req := newFillRangeRequest(t, rawRange) + + got := f.fill(newTestCaching(nil), req, rawRange) + if got != req { + t.Fatalf("fill() returned a different request for unknown-size unresolvable range") + } + if got.Header.Get("Range") != rawRange { + t.Fatalf("Range = %q, want %q", got.Header.Get("Range"), rawRange) + } + if got.Context().Value(fillRangeKey{}) != nil { + t.Fatal("fillRange context present, want nil") + } + }) + } +} + +func TestFillRangeKnownSizeSuffixRangeIsProcessed(t *testing.T) { + f := newTestFillRange() + req := newFillRangeRequest(t, "bytes=-512") + + got := f.fill(newTestCaching(&object.Metadata{Size: 1000}), req, req.Header.Get("Range")) + if got == req { + t.Fatal("fill() returned original request, want request with fillRange context") + } + if got.Header.Get("Range") == "bytes=-512" { + t.Fatal("Range was not rewritten for known-size suffix range") + } + + fill, ok := got.Context().Value(fillRangeKey{}).(*fillRangeContext) + if !ok { + t.Fatalf("fillRange context type = %T, want *fillRangeContext", got.Context().Value(fillRangeKey{})) + } + if fill.rawStart != 488 || fill.rawEnd != 999 { + t.Fatalf("raw range = %d-%d, want 488-999", fill.rawStart, fill.rawEnd) + } +} + +func TestFillRangeUnknownSizeRegularRangeStillProcesses(t *testing.T) { + f := newTestFillRange() + req := newFillRangeRequest(t, "bytes=0-99") + + got := f.fill(newTestCaching(nil), req, req.Header.Get("Range")) + if got == req { + t.Fatal("fill() returned original request, want request with fillRange context") + } + if got.Context().Value(fillRangeKey{}) == nil { + t.Fatal("fillRange context missing") + } + if got.Header.Get("Range") != "bytes=0-255" { + t.Fatalf("Range = %q, want %q", got.Header.Get("Range"), "bytes=0-255") + } +} diff --git a/server/middleware/caching/internal.go b/server/middleware/caching/internal.go index 55aff5a..99b1668 100644 --- a/server/middleware/caching/internal.go +++ b/server/middleware/caching/internal.go @@ -22,8 +22,8 @@ import ( "github.com/omalloc/tavern/api/defined/v1/storage/object" "github.com/omalloc/tavern/contrib/log" "github.com/omalloc/tavern/internal/protocol" - "github.com/omalloc/tavern/pkg/traces" "github.com/omalloc/tavern/pkg/iobuf" + "github.com/omalloc/tavern/pkg/traces" xhttp "github.com/omalloc/tavern/pkg/x/http" "github.com/omalloc/tavern/proxy" ) @@ -100,7 +100,7 @@ func (c *Caching) setXCache(resp *http.Response) { // debug header if trace := c.req.Header.Get(protocol.InternalTraceKey); trace != "" { - resp.Header.Set(protocol.InternalStoreUrl, strconv.FormatInt(int64(c.cacheStatus), 10)) + resp.Header.Set(protocol.InternalStoreUrl, c.id.Key()) resp.Header.Set(protocol.InternalSwapfile, c.id.WPath(c.bucket.Path())) } } @@ -219,7 +219,7 @@ func getContents(c *Caching, reqChunks []uint32, from uint32) (io.ReadCloser, in rawRange := c.req.Header.Get("Range") rng, _ := xhttp.SingleRange(rawRange, c.md.Size) tailChunkSize := uint64(reqChunks[len(reqChunks)-1]+1)*uint64(partSize) - 1 - if tailChunkSize > uint64(rng.End) { + if rng != nil && tailChunkSize > uint64(rng.End) { tailChunkSize = uint64(rng.End) } toByte := min(c.md.Size-1, tailChunkSize) diff --git a/server/middleware/multirange/multirange.go b/server/middleware/multirange/multirange.go index 1884f5e..e5bbddf 100644 --- a/server/middleware/multirange/multirange.go +++ b/server/middleware/multirange/multirange.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "math" "mime/multipart" "net/http" "strconv" @@ -38,41 +39,56 @@ func Middleware(c *configv1.Middleware) (middleware.Middleware, func(), error) { return origin.RoundTrip(req) } - // unsafeRanges, err := xhttp.UnsatisfiableMultiRange(rawRange) - // if err != nil || len(unsafeRanges) <= 1 { - // return origin.RoundTrip(req) - // } - - byteRanges, err := rangecontrol.Parse(rawRange) + // 检测是否有多个 Range，如果只有一个 Range，则直接传递给下游 + rangeSet, err := rangecontrol.Parse(rawRange, math.MaxInt32) if err != nil { - // Range 解析失败，返回 416 headers := make(http.Header) headers.Set("X-Error", err.Error()) return nil, xhttp.NewBizError(http.StatusRequestedRangeNotSatisfiable, headers) } - mergedByteRanges := rangecontrol.MergeRanges(byteRanges) - if len(mergedByteRanges) <= 1 { + // Single suffix range: pass through to next layer (P0) + if len(rangeSet) <= 1 { return origin.RoundTrip(req) } header := make(http.Header) - head, err1 := prefetchResource(req, origin) if err1 != nil { return nil, err1 } if head.Body != nil { + _ = head.Body.Close() } objSize := uint64(head.ContentLength) // ContentLength 不一定存在 cr, err := rangecontrol.ParseContentRange(head.Header.Get("Content-Range")) - if err == nil { + if err == nil && cr.Size > 0 { // 如果源站支持 Range 头响应，则使用其总大小 objSize = uint64(cr.Size) } + // If ranges were silently dropped (objSize unknown), reject. + if objSize <= 0 { + headers := make(http.Header) + headers.Set("X-Error", "unable to resolve range without object size") + return nil, xhttp.NewBizError(http.StatusRequestedRangeNotSatisfiable, headers) + } + + byteRanges, err := rangecontrol.Parse(rawRange, objSize) + if err != nil { + // Range 解析失败，返回 416 + headers := make(http.Header) + headers.Set("X-Error", err.Error()) + return nil, xhttp.NewBizError(http.StatusRequestedRangeNotSatisfiable, headers) + } + + mergedByteRanges := rangecontrol.MergeRanges(byteRanges) + if len(mergedByteRanges) <= 1 { + return origin.RoundTrip(req) + } + ctype := head.Header.Get("Content-Type") xhttp.CopyHeadersWithout(header, head.Header, "Content-Length", "Content-Range", "Content-Type") diff --git a/server/middleware/multirange/multirange_test.go b/server/middleware/multirange/multirange_test.go new file mode 100644 index 0000000..467d22b --- /dev/null +++ b/server/middleware/multirange/multirange_test.go @@ -0,0 +1,161 @@ +package multirange + +import ( + "io" + "net/http" + "testing" + + configv1 "github.com/omalloc/tavern/api/defined/v1/middleware" + xhttp "github.com/omalloc/tavern/pkg/x/http" +) + +type recordingRoundTripper struct { + calls int + rangeHeader string +} + +func (r *recordingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + r.calls++ + r.rangeHeader = req.Header.Get("Range") + return &http.Response{ + StatusCode: http.StatusNoContent, + Header: make(http.Header), + Body: io.NopCloser(http.NoBody), + }, nil +} + +func newTestRoundTripper(t *testing.T, origin http.RoundTripper) http.RoundTripper { + t.Helper() + + mw, cleanup, err := Middleware(&configv1.Middleware{}) + if err != nil { + t.Fatalf("Middleware() error = %v", err) + } + t.Cleanup(cleanup) + + return mw(origin) +} + +func TestMiddlewarePassesSingleSuffixRangeThrough(t *testing.T) { + tests := []string{ + "bytes=-512", + "bytes= -512", + } + + for _, rawRange := range tests { + t.Run(rawRange, func(t *testing.T) { + origin := &recordingRoundTripper{} + rt := newTestRoundTripper(t, origin) + + req, err := http.NewRequest(http.MethodGet, "http://example.com/file", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", rawRange) + + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip() error = %v", err) + } + if resp.Body != nil { + _ = resp.Body.Close() + } + if origin.calls != 1 { + t.Fatalf("origin calls = %d, want 1", origin.calls) + } + if origin.rangeHeader != rawRange { + t.Fatalf("origin Range = %q, want %q", origin.rangeHeader, rawRange) + } + }) + } +} + +func TestMiddlewarePassesSingleRangeThrough(t *testing.T) { + origin := &recordingRoundTripper{} + rt := newTestRoundTripper(t, origin) + + req, err := http.NewRequest(http.MethodGet, "http://example.com/file", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=0-499") + + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip() error = %v", err) + } + if resp.Body != nil { + _ = resp.Body.Close() + } + if origin.calls != 1 { + t.Fatalf("origin calls = %d, want 1", origin.calls) + } +} + +func TestMiddlewareRejectsInvalidRange(t *testing.T) { + tests := []string{ + "bytes=abc", + "bytes=-abc", + "bytes=-", + } + + for _, rawRange := range tests { + t.Run(rawRange, func(t *testing.T) { + origin := &recordingRoundTripper{} + rt := newTestRoundTripper(t, origin) + + req, err := http.NewRequest(http.MethodGet, "http://example.com/file", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", rawRange) + + resp, err := rt.RoundTrip(req) + if err == nil { + t.Fatal("RoundTrip() error = nil, want error") + } + if resp != nil { + t.Fatalf("RoundTrip() response = %#v, want nil", resp) + } + if origin.calls != 0 { + t.Fatalf("origin calls = %d, want 0", origin.calls) + } + bizErr, ok := xhttp.ParseBizError(err) + if !ok { + t.Fatalf("error type = %T, want xhttp.BizError", err) + } + if bizErr.Code() != http.StatusRequestedRangeNotSatisfiable { + t.Fatalf("error code = %d, want %d", bizErr.Code(), http.StatusRequestedRangeNotSatisfiable) + } + }) + } +} + +func TestMiddlewareDoesNotPassMixedSuffixMultiRangeThrough(t *testing.T) { + origin := &recordingRoundTripper{} + rt := newTestRoundTripper(t, origin) + + req, err := http.NewRequest(http.MethodGet, "http://example.com/file", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=0-99,-512") + + resp, err := rt.RoundTrip(req) + if err == nil { + t.Fatal("RoundTrip() error = nil, want error") + } + if resp != nil { + t.Fatalf("RoundTrip() response = %#v, want nil", resp) + } + if origin.calls != 1 { + t.Fatalf("origin calls = %d, want 1 (HEAD request)", origin.calls) + } + bizErr, ok := xhttp.ParseBizError(err) + if !ok { + t.Fatalf("error type = %T, want xhttp.BizError", err) + } + if bizErr.Code() != http.StatusRequestedRangeNotSatisfiable { + t.Fatalf("error code = %d, want %d", bizErr.Code(), http.StatusRequestedRangeNotSatisfiable) + } +} diff --git a/storage/bucket/disk/disk.go b/storage/bucket/disk/disk.go index d4f56dd..089d27f 100644 --- a/storage/bucket/disk/disk.go +++ b/storage/bucket/disk/disk.go @@ -16,9 +16,7 @@ import ( "sync" "time" - "github.com/omalloc/tavern/pkg/iobuf" "github.com/paulbellamy/ratecounter" - "github.com/prometheus/client_golang/prometheus" "github.com/omalloc/tavern/api/defined/v1/storage" @@ -26,6 +24,7 @@ import ( "github.com/omalloc/tavern/contrib/log" "github.com/omalloc/tavern/pkg/algorithm/heavykeeper" "github.com/omalloc/tavern/pkg/algorithm/lru" + "github.com/omalloc/tavern/pkg/iobuf" "github.com/omalloc/tavern/storage/indexdb" ) @@ -364,6 +363,15 @@ func (d *diskBucket) Store(ctx context.Context, meta *object.Metadata) error { } start := time.Now() + + // merge old chunks + oldmd, _ := d.indexdb.Get(ctx, meta.ID.Bytes()) + if oldmd != nil { + oldmd.Chunks.Range(func(x uint32) { + meta.Chunks.Set(x) + }) + } + if err := d.indexdb.Set(ctx, meta.ID.Bytes(), meta); err != nil { indexdbOperationDuration.With(prometheus.Labels{"op": "set", "bucket": d.ID()}).Observe(time.Since(start).Seconds()) return err diff --git a/tests/all-features/range/range_suffix_test.go b/tests/all-features/range/range_suffix_test.go new file mode 100644 index 0000000..25c700e --- /dev/null +++ b/tests/all-features/range/range_suffix_test.go @@ -0,0 +1,126 @@ +package range_test + +import ( + "fmt" + "net/http" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + + "github.com/omalloc/tavern/pkg/e2e" +) + +const suffixRangeLength = 512 + +func TestSuffixRangeMissPassesOriginalRange(t *testing.T) { + f := e2e.GenFile(t, 2<<20) + + tests := []string{ + "bytes=-512", + "bytes= -512", + } + + for _, rawRange := range tests { + t.Run(rawRange, func(t *testing.T) { + url := fmt.Sprintf("http://sendya.me.gslb.com/cases/range/suffix/miss/%s", uuid.NewString()) + originRanges := make(chan string, 1) + case1 := e2e.New(url, e2e.RespCallbackFile(f, func(w http.ResponseWriter, r *http.Request) { + originRanges <- r.Header.Get("Range") + })) + defer case1.Close() + + resp, err := case1.Do(func(r *http.Request) { + r.Header.Set("Range", rawRange) + }) + if !assert.NoError(t, err, "response should not error") { + return + } + if resp == nil { + t.Fatal("response should not be nil") + } + + hashBody := e2e.HashBody(resp) + start, length, contentRange := suffixRangeExpectation(f) + hashFile := e2e.HashFile(f.Path, start, length) + + assert.Equal(t, http.StatusPartialContent, resp.StatusCode, "response should be code 206") + assert.Contains(t, resp.Header.Get("X-Cache"), "MISS", "cache should be MISS") + assert.Equal(t, contentRange, resp.Header.Get("Content-Range"), "response should be Content-Range") + assert.Equal(t, int64(length), resp.ContentLength, "size should be suffix length") + assert.Equal(t, hashFile, hashBody, "response body should be suffix bytes") + + select { + case got := <-originRanges: + assert.Equal(t, rawRange, got, "origin should receive original suffix range") + default: + t.Fatal("origin was not called") + } + + e2e.Purge(t, url) + }) + } +} + +func TestSuffixRangeHitUsesCachedObjectSize(t *testing.T) { + f := e2e.GenFile(t, 2<<20) + url := fmt.Sprintf("http://sendya.me.gslb.com/cases/range/suffix/hit/%s", uuid.NewString()) + + t.Run("warm full object", func(t *testing.T) { + case1 := e2e.New(url, e2e.RespSimpleFile(f)) + defer case1.Close() + + resp, err := case1.Do(func(r *http.Request) {}) + if !assert.NoError(t, err, "response should not error") { + return + } + if resp == nil { + t.Fatal("response should not be nil") + } + + size := e2e.DiscardBody(resp, 5*1024) + + assert.Equal(t, http.StatusOK, resp.StatusCode, "response should be code 200") + assert.Contains(t, resp.Header.Get("X-Cache"), "MISS", "cache should be MISS") + assert.Equal(t, f.Size, size, "size should be file size") + }) + + t.Run("suffix range HIT", func(t *testing.T) { + case1 := e2e.New(url, e2e.WrongHit(t)) + defer case1.Close() + + resp, err := case1.Do(func(r *http.Request) { + r.Header.Set("Range", "bytes=-512") + }) + if !assert.NoError(t, err, "response should not error") { + return + } + if resp == nil { + t.Fatal("response should not be nil") + } + + hashBody := e2e.HashBody(resp) + start, length, contentRange := suffixRangeExpectation(f) + hashFile := e2e.HashFile(f.Path, start, length) + + assert.Equal(t, http.StatusPartialContent, resp.StatusCode, "response should be code 206") + assert.Contains(t, resp.Header.Get("X-Cache"), "HIT", "cache should be HIT") + assert.Equal(t, contentRange, resp.Header.Get("Content-Range"), "response should be Content-Range") + assert.Equal(t, int64(length), resp.ContentLength, "size should be suffix length") + assert.Equal(t, hashFile, hashBody, "response body should be suffix bytes") + }) + + t.Run("PURGE", func(t *testing.T) { + e2e.Purge(t, url) + }) +} + +func suffixRangeExpectation(f *e2e.MockFile) (start, length int, contentRange string) { + length = suffixRangeLength + if f.Size < length { + length = f.Size + } + start = f.Size - length + contentRange = fmt.Sprintf("bytes %d-%d/%d", start, f.Size-1, f.Size) + return start, length, contentRange +} diff --git a/tests/mockserver/main.go b/tests/mockserver/main.go index f7f4625..8f07e17 100644 --- a/tests/mockserver/main.go +++ b/tests/mockserver/main.go @@ -91,7 +91,7 @@ func main() { rawRange := r.Header.Get("Range") - byteRange, err := rangecontrol.Parse(rawRange) + byteRange, err := rangecontrol.Parse(rawRange, 0) if err != nil { _, _ = io.Copy(w, reader) return