pkg-proxy/internal/handler/metadata_gzip_test.go

package handler

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/git-pkgs/cooldown"
)

// compressibleDocuments sets upstream documents with enough versions that
// gzip makes them smaller.
func compressibleDocuments(u *validatorUpstream) {
	var npm, composer []string
	for i := range 50 {
		v := fmt.Sprintf("1.0.%d", i)
		npm = append(npm, `"`+v+`":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-`+v+`.tgz"}}`)
		composer = append(composer, `{"version":"`+v+`","dist":{"type":"zip","url":"https://example.com/pkg-`+v+`.zip"}}`)
	}
	u.set(
		`{"name":"left-pad","versions":{`+strings.Join(npm, ",")+`}}`,
		`{"packages":{"vendor/pkg":[`+strings.Join(composer, ",")+`]}}`,
	)
}

func gunzip(t *testing.T, rec *httptest.ResponseRecorder) string {
	t.Helper()
	zr, err := gzip.NewReader(bytes.NewReader(rec.Body.Bytes()))
	if err != nil {
		t.Fatalf("body is not gzip: %v", err)
	}
	body, err := io.ReadAll(zr)
	if err != nil {
		t.Fatalf("reading gzip body: %v", err)
	}
	return string(body)
}

var acceptGzip = http.Header{"Accept-Encoding": {"gzip, deflate, br"}}

// TestRewrittenMetadataGzip checks that a client accepting gzip gets the
// cached compressed copy of rewritten metadata, on the first fetch and on
// the rewrite fast path, with an ETag of its own so it never matches the
// uncompressed representation.
func TestRewrittenMetadataGzip(t *testing.T) {
	for _, c := range validatorCases {
		for _, ttl := range []time.Duration{time.Hour, 0} {
			t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) {
				u := newValidatorUpstream(t)
				compressibleDocuments(u)
				handler := c.handler(validatorProxy(t, u, ttl), u)

				identity := serve(handler, c.path, nil)
				for range 2 {
					expectGzipOf(t, serve(handler, c.path, acceptGzip), identity)
				}
			})
		}
	}
}

// expectGzipOf checks that rec is the gzip representation of the
// uncompressed response identity.
func expectGzipOf(t *testing.T, rec, identity *httptest.ResponseRecorder) {
	t.Helper()
	if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "gzip" {
		t.Fatalf("status %d, Content-Encoding %q; want 200 with gzip", rec.Code, rec.Header().Get("Content-Encoding"))
	}
	if rec.Body.Len() >= identity.Body.Len() {
		t.Errorf("gzip body is %d bytes, uncompressed %d", rec.Body.Len(), identity.Body.Len())
	}
	if got := gunzip(t, rec); got != identity.Body.String() {
		t.Errorf("decompressed body differs from the uncompressed response")
	}
	if got := rec.Header().Get("Vary"); got != "Accept-Encoding" {
		t.Errorf("Vary = %q, want Accept-Encoding", got)
	}
	if etag := rec.Header().Get("ETag"); etag == "" || etag == identity.Header().Get("ETag") {
		t.Errorf("gzip ETag %q must differ from the uncompressed ETag %q", etag, identity.Header().Get("ETag"))
	}
	if rec.Header().Get("Last-Modified") != identity.Header().Get("Last-Modified") {
		t.Errorf("Last-Modified %q, want %q", rec.Header().Get("Last-Modified"), identity.Header().Get("Last-Modified"))
	}
}

// TestRewrittenMetadataGzipRevalidation checks that each representation
// revalidates with its own ETag, also when a reverse proxy weakened it.
func TestRewrittenMetadataGzipRevalidation(t *testing.T) {
	for _, c := range validatorCases {
		t.Run(c.ecosystem, func(t *testing.T) {
			u := newValidatorUpstream(t)
			compressibleDocuments(u)
			handler := c.handler(validatorProxy(t, u, time.Hour), u)
			gzipETag := serve(handler, c.path, acceptGzip).Header().Get("ETag")
			identityETag := serve(handler, c.path, nil).Header().Get("ETag")

			for _, ifNoneMatch := range []string{gzipETag, "W/" + gzipETag} {
				rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {ifNoneMatch}})
				expectNotModified(t, rec, gzipETag)
				if got := rec.Header().Get("Vary"); got != "Accept-Encoding" {
					t.Errorf("304 Vary = %q, want Accept-Encoding", got)
				}
			}
			if rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {identityETag}}); rec.Code != http.StatusOK {
				t.Errorf("gzip request revalidating the uncompressed ETag: status %d, want 200", rec.Code)
			}
			if rec := serve(handler, c.path, http.Header{"If-None-Match": {gzipETag}}); rec.Code != http.StatusOK {
				t.Errorf("uncompressed request revalidating the gzip ETag: status %d, want 200", rec.Code)
			}
		})
	}
}

// TestRewrittenMetadataWithoutGzip checks that clients that do not accept
// gzip get the uncompressed document, which still varies on Accept-Encoding.
func TestRewrittenMetadataWithoutGzip(t *testing.T) {
	for _, c := range validatorCases {
		t.Run(c.ecosystem, func(t *testing.T) {
			u := newValidatorUpstream(t)
			compressibleDocuments(u)
			handler := c.handler(validatorProxy(t, u, time.Hour), u)

			for _, acceptEncoding := range []string{"", "identity", "gzip;q=0", "br"} {
				header := http.Header{}
				if acceptEncoding != "" {
					header.Set("Accept-Encoding", acceptEncoding)
				}
				rec := serve(handler, c.path, header)
				if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" {
					t.Errorf("Accept-Encoding %q: status %d, Content-Encoding %q; want 200 uncompressed", acceptEncoding, rec.Code, rec.Header().Get("Content-Encoding"))
				}
				if got := rec.Header().Get("Vary"); got != "Accept-Encoding" {
					t.Errorf("Accept-Encoding %q: Vary = %q, want Accept-Encoding", acceptEncoding, got)
				}
			}
		})
	}
}

// TestRewrittenMetadataGzipOnlyFromRewriteCache checks that rewrites made
// outside the cache, with it disabled or with cooldown on, stay uncompressed:
// compressing them on every request is better left to a reverse proxy.
func TestRewrittenMetadataGzipOnlyFromRewriteCache(t *testing.T) {
	setups := map[string]func(*Proxy){
		"cache disabled": func(p *Proxy) { p.SetMetadataRewriteCacheSize(0) },
		"cooldown without cache": func(p *Proxy) {
			p.SetMetadataRewriteCacheSize(0)
			p.Cooldown = &cooldown.Config{Default: "3d"}
		},
	}
	for _, c := range validatorCases {
		for name, setup := range setups {
			t.Run(c.ecosystem+"/"+name, func(t *testing.T) {
				u := newValidatorUpstream(t)
				compressibleDocuments(u)
				proxy := validatorProxy(t, u, time.Hour)
				setup(proxy)
				handler := c.handler(proxy, u)

				for range 2 {
					if rec := serve(handler, c.path, acceptGzip); rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" {
						t.Errorf("status %d, Content-Encoding %q; want 200 uncompressed", rec.Code, rec.Header().Get("Content-Encoding"))
					}
				}
			})
		}
	}
}

// TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller checks that the cache keeps
// only a compressed copy of a rewrite, counted against its size, unless
// compressing does not make the rewrite smaller, and that the plain document
// is still there for callers that need it.
func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) {
	c := newRewriteCache(1 << 20)
	identity := func(b []byte) ([]byte, error) { return b, nil }
	large := []byte(strings.Repeat(`{"version":"1.0.0"},`, 100))

	for _, doc := range [][]byte{large, []byte("abc")} {
		key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc)
		if _, err := c.rewrite(t.Context(), key, doc, nil, identity); err != nil {
			t.Fatal(err)
		}
	}

	m, ok := c.get(rewriteCacheKey("npm", "http://proxy", string(large[:3]), large))
	if !ok || m.gzipped == nil {
		t.Fatalf("large rewrite: cached %v, gzip copy %d bytes; want a gzip copy", ok, len(m.gzipped))
	}
	if m.plain != nil {
		t.Errorf("large rewrite: cached %d plain bytes besides the gzip copy, want none", len(m.plain))
	}
	if c.size != int64(len(m.gzipped)+3) {
		t.Errorf("cache size = %d, want the gzip copy and the small rewrite (%d)", c.size, len(m.gzipped)+3)
	}
	if !bytes.Equal(m.content(), large) || m.size != len(large) {
		t.Errorf("large rewrite content = %d bytes, size %d; want the %d bytes rewritten", len(m.content()), m.size, len(large))
	}
	if small, ok := c.get(rewriteCacheKey("npm", "http://proxy", "abc", []byte("abc"))); !ok || small.gzipped != nil || string(small.content()) != "abc" {
		t.Errorf("small rewrite: cached %v, gzip copy %d bytes, content %q; want abc uncompressed", ok, len(small.gzipped), small.content())
	}
}

func TestAcceptsGzip(t *testing.T) {
	cases := map[string]bool{
		"":                        false,
		"identity":                false,
		"br, deflate":             false,
		"gzip":                    true,
		"GZIP":                    true,
		"x-gzip":                  true,
		"deflate, gzip;q=1.0":     true,
		"gzip; q=0.5":             true,
		"gzip;q=0":                false,
		"gzip;q=0.000":            false,
		"*":                       true,
		"*;q=0":                   false,
		"br;q=1, *;q=0.5":         true,
		"gzip;q=0, *":             false,
		"gzip;q=invalid, deflate": false,
	}
	for header, want := range cases {
		r := httptest.NewRequest(http.MethodGet, "/", nil)
		if header != "" {
			r.Header.Set("Accept-Encoding", header)
		}
		if got := acceptsGzip(r); got != want {
			t.Errorf("acceptsGzip(%q) = %v, want %v", header, got, want)
		}
	}
}