mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-11-09 15:49:16 -05:00
25f62da322
The case-variant lookup filtered on lower(name), which no index covers, so a legacy row's first lookup scanned every metadata_cache row in its ecosystem. Reclaim already skips a queued path a row still points at, so each row now queues just its own legacy blob once it has moved off it. On case-insensitive storage a case variant still at its legacy path can lose its blob that way, and is then fetched again.
351 lines
12 KiB
Go
351 lines
12 KiB
Go
package handler
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/git-pkgs/proxy/internal/database"
|
|
"github.com/git-pkgs/proxy/internal/storage"
|
|
"github.com/git-pkgs/registries/fetch"
|
|
)
|
|
|
|
func TestMetadataStoragePathKeepsCaseApart(t *testing.T) {
|
|
for _, pair := range [][2]string{
|
|
{"github.com/Sirupsen/logrus", "github.com/sirupsen/logrus"},
|
|
{"JSONStream", "jsonstream"},
|
|
} {
|
|
a, b := metadataStoragePath("go", pair[0]), metadataStoragePath("go", pair[1])
|
|
if strings.EqualFold(a, b) {
|
|
t.Errorf("%q and %q map to paths %q and %q, which a case-insensitive filesystem treats as one", pair[0], pair[1], a, b)
|
|
}
|
|
}
|
|
for _, key := range []string{"CON", "nul", "trailing.", "a/../b"} {
|
|
p := metadataStoragePath("npm", key)
|
|
if strings.Contains(p, key) {
|
|
t.Errorf("path %q contains the raw key %q", p, key)
|
|
}
|
|
}
|
|
// Keys that already are digests keep their path, so existing entries
|
|
// stay valid.
|
|
sum := sha256.Sum256([]byte("generic"))
|
|
digest := hex.EncodeToString(sum[:])
|
|
if got, want := metadataStoragePath("generic", digest), "_metadata/generic/"+digest+"/metadata"; got != want {
|
|
t.Errorf("digest key path = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
// fileStorageProxy returns a proxy whose metadata is cached on the local
|
|
// filesystem, as with file:// storage.
|
|
func fileStorageProxy(t *testing.T, client *http.Client) (*Proxy, *database.DB) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
db, err := database.Create(filepath.Join(dir, "test.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = db.Close() })
|
|
store, err := storage.OpenBucket(context.Background(), "file://"+filepath.Join(dir, "cache"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = store.Close() })
|
|
proxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil)))
|
|
proxy.HTTPClient = client
|
|
proxy.CacheMetadata = true
|
|
proxy.MetadataTTL = time.Hour
|
|
return proxy, db
|
|
}
|
|
|
|
// TestMetadataKeysDifferingInCaseCachedApart caches metadata for two keys
|
|
// that differ only in case on file storage. On a case-insensitive filesystem
|
|
// they shared one file, and the first key was served the second's document.
|
|
func TestMetadataKeysDifferingInCaseCachedApart(t *testing.T) {
|
|
var requests atomic.Int32
|
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests.Add(1)
|
|
_, _ = io.WriteString(w, "document for "+r.URL.Path)
|
|
}))
|
|
defer upstream.Close()
|
|
proxy, _ := fileStorageProxy(t, upstream.Client())
|
|
|
|
keys := []string{"github.com/Sirupsen/logrus", "github.com/sirupsen/logrus"}
|
|
for range 2 {
|
|
for _, key := range keys {
|
|
body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/"+key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want := "document for /" + key; string(body) != want {
|
|
t.Errorf("%s: served %q, want %q", key, body, want)
|
|
}
|
|
}
|
|
}
|
|
if got := requests.Load(); got != 2 {
|
|
t.Errorf("upstream requests = %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
// seedLegacyMetadata stores blob at key's legacy path and a row pointing at
|
|
// it, fetched at fetchedAt. The row's digest is that of document, or unset
|
|
// when document is empty.
|
|
func seedLegacyMetadata(t *testing.T, proxy *Proxy, db *database.DB, key, blob, document string, fetchedAt time.Time) string {
|
|
t.Helper()
|
|
legacy := legacyMetadataStoragePath("go", key)
|
|
if _, _, err := proxy.Storage.Store(context.Background(), legacy, strings.NewReader(blob)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entry := &database.MetadataCacheEntry{
|
|
Ecosystem: "go",
|
|
Name: key,
|
|
StoragePath: legacy,
|
|
ETag: sql.NullString{String: `"v1"`, Valid: true},
|
|
Size: sql.NullInt64{Int64: int64(len(blob)), Valid: true},
|
|
FetchedAt: sql.NullTime{Time: fetchedAt, Valid: true},
|
|
}
|
|
if document != "" {
|
|
sum := sha256.Sum256([]byte(document))
|
|
entry.ContentDigest = sql.NullString{String: "sha256:" + hex.EncodeToString(sum[:]), Valid: true}
|
|
}
|
|
if err := db.UpsertMetadataCache(entry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return legacy
|
|
}
|
|
|
|
func pendingDeletes(t *testing.T, db *database.DB) []string {
|
|
t.Helper()
|
|
due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return due
|
|
}
|
|
|
|
func metadataRowPath(t *testing.T, db *database.DB, key string) string {
|
|
t.Helper()
|
|
entry, err := db.GetMetadataCache("go", key)
|
|
if err != nil || entry == nil {
|
|
t.Fatalf("cache row for %s: %v %v", key, entry, err)
|
|
}
|
|
return entry.StoragePath
|
|
}
|
|
|
|
// TestLegacyMetadataMatchingBlobServedWhenUpstreamDown checks that a row at
|
|
// its legacy path whose blob matches its digest is still served as a stale
|
|
// fallback, and is moved to the hashed path.
|
|
func TestLegacyMetadataMatchingBlobServedWhenUpstreamDown(t *testing.T) {
|
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusBadGateway)
|
|
}))
|
|
defer upstream.Close()
|
|
proxy, db := fileStorageProxy(t, upstream.Client())
|
|
|
|
const key = "github.com/Sirupsen/logrus"
|
|
legacy := seedLegacyMetadata(t, proxy, db, key, "old", "old", time.Now().Add(-2*time.Hour))
|
|
|
|
for range 2 {
|
|
body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x")
|
|
if err != nil {
|
|
t.Fatalf("legacy document not served while upstream is down: %v", err)
|
|
}
|
|
if string(body) != "old" {
|
|
t.Fatalf("served %q, want %q", body, "old")
|
|
}
|
|
}
|
|
if got, want := metadataRowPath(t, db, key), metadataStoragePath("go", key); got != want {
|
|
t.Errorf("storage path = %q, want %q", got, want)
|
|
}
|
|
if due := pendingDeletes(t, db); !slices.Contains(due, legacy) {
|
|
t.Errorf("pending deletes = %v, want %q", due, legacy)
|
|
}
|
|
}
|
|
|
|
// TestLegacyMetadataMatchingBlobServedFromCache checks that a fresh row at
|
|
// its legacy path whose blob matches is served without asking upstream.
|
|
func TestLegacyMetadataMatchingBlobServedFromCache(t *testing.T) {
|
|
var requests atomic.Int32
|
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests.Add(1)
|
|
_, _ = io.WriteString(w, "fresh")
|
|
}))
|
|
defer upstream.Close()
|
|
proxy, db := fileStorageProxy(t, upstream.Client())
|
|
|
|
const key = "github.com/Sirupsen/logrus"
|
|
seedLegacyMetadata(t, proxy, db, key, "old", "old", time.Now())
|
|
|
|
body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(body) != "old" || requests.Load() != 0 {
|
|
t.Errorf("served %q after %d upstream requests, want the cached document", body, requests.Load())
|
|
}
|
|
}
|
|
|
|
// TestLegacyMetadataMismatchedBlobNotServed checks that a legacy blob that
|
|
// does not match the row's digest, as when another key's document
|
|
// overwrote it, is neither served nor used as a fallback.
|
|
func TestLegacyMetadataMismatchedBlobNotServed(t *testing.T) {
|
|
var down atomic.Bool
|
|
var requests atomic.Int32
|
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests.Add(1)
|
|
if down.Load() {
|
|
w.WriteHeader(http.StatusBadGateway)
|
|
return
|
|
}
|
|
if r.Header.Get("If-None-Match") != "" {
|
|
t.Errorf("revalidated a document that was dropped")
|
|
}
|
|
_, _ = io.WriteString(w, "fresh")
|
|
}))
|
|
defer upstream.Close()
|
|
proxy, db := fileStorageProxy(t, upstream.Client())
|
|
|
|
const key = "github.com/Sirupsen/logrus"
|
|
seedLegacyMetadata(t, proxy, db, key, "other key's document", "old", time.Now())
|
|
down.Store(true)
|
|
if body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x"); err == nil {
|
|
t.Fatalf("served %q, want an error while upstream is down", body)
|
|
}
|
|
|
|
down.Store(false)
|
|
body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(body) != "fresh" {
|
|
t.Errorf("served %q, want %q", body, "fresh")
|
|
}
|
|
if got, want := metadataRowPath(t, db, key), metadataStoragePath("go", key); got != want {
|
|
t.Errorf("storage path = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
// caseFoldingStorage stores paths differing only in case as one file, as a
|
|
// case-insensitive filesystem does.
|
|
type caseFoldingStorage struct{ storage.Storage }
|
|
|
|
func (s caseFoldingStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
|
|
return s.Storage.Store(ctx, strings.ToLower(path), r)
|
|
}
|
|
|
|
func (s caseFoldingStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) {
|
|
return s.Storage.Open(ctx, strings.ToLower(path))
|
|
}
|
|
|
|
func (s caseFoldingStorage) Exists(ctx context.Context, path string) (bool, error) {
|
|
return s.Storage.Exists(ctx, strings.ToLower(path))
|
|
}
|
|
|
|
func (s caseFoldingStorage) Delete(ctx context.Context, path string) error {
|
|
return s.Storage.Delete(ctx, strings.ToLower(path))
|
|
}
|
|
|
|
func (s caseFoldingStorage) Size(ctx context.Context, path string) (int64, error) {
|
|
return s.Storage.Size(ctx, strings.ToLower(path))
|
|
}
|
|
|
|
// TestLegacyMetadataCaseVariantRefetchedAfterReclaim checks that on
|
|
// case-insensitive storage, a key whose legacy blob was shared with a case
|
|
// variant and deleted after that variant moved is fetched again rather than
|
|
// served.
|
|
func TestLegacyMetadataCaseVariantRefetchedAfterReclaim(t *testing.T) {
|
|
var requests atomic.Int32
|
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests.Add(1)
|
|
_, _ = io.WriteString(w, "fresh")
|
|
}))
|
|
defer upstream.Close()
|
|
proxy, db := fileStorageProxy(t, upstream.Client())
|
|
proxy.Storage = caseFoldingStorage{proxy.Storage}
|
|
|
|
const upper, lower = "github.com/Sirupsen/logrus", "github.com/sirupsen/logrus"
|
|
seedLegacyMetadata(t, proxy, db, upper, "doc", "doc", time.Now())
|
|
seedLegacyMetadata(t, proxy, db, lower, "doc", "doc", time.Now())
|
|
|
|
if body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", upper, upstream.URL+"/x"); err != nil || string(body) != "doc" {
|
|
t.Fatalf("FetchOrCacheMetadata(%s) = %q, %v; want the migrated document", upper, body, err)
|
|
}
|
|
// Reclaim deletes the queued legacy blob, which is also lower's.
|
|
for _, path := range pendingDeletes(t, db) {
|
|
if err := proxy.Storage.Delete(context.Background(), path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", lower, upstream.URL+"/x")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(body) != "fresh" || requests.Load() != 1 {
|
|
t.Errorf("served %q after %d upstream requests, want a fresh fetch", body, requests.Load())
|
|
}
|
|
if got, want := metadataRowPath(t, db, lower), metadataStoragePath("go", lower); got != want {
|
|
t.Errorf("storage path = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
// TestLegacyMetadataPathRefetched checks that a row at its legacy path with
|
|
// no digest to check its blob against is fetched again, moved to the hashed
|
|
// path, and its old blob queued for deletion.
|
|
func TestLegacyMetadataPathRefetched(t *testing.T) {
|
|
var requests atomic.Int32
|
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests.Add(1)
|
|
_, _ = io.WriteString(w, "fresh")
|
|
}))
|
|
defer upstream.Close()
|
|
proxy, db := fileStorageProxy(t, upstream.Client())
|
|
|
|
const key = "github.com/Sirupsen/logrus"
|
|
legacy := legacyMetadataStoragePath("go", key)
|
|
if _, _, err := proxy.Storage.Store(context.Background(), legacy, bytes.NewReader([]byte("old"))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{
|
|
Ecosystem: "go",
|
|
Name: key,
|
|
StoragePath: legacy,
|
|
Size: sql.NullInt64{Int64: 3, Valid: true},
|
|
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(body) != "fresh" || requests.Load() != 1 {
|
|
t.Fatalf("served %q after %d upstream requests, want a fresh fetch", body, requests.Load())
|
|
}
|
|
entry, err := db.GetMetadataCache("go", key)
|
|
if err != nil || entry == nil {
|
|
t.Fatalf("cache row: %v %v", entry, err)
|
|
}
|
|
if entry.StoragePath != metadataStoragePath("go", key) {
|
|
t.Errorf("storage path = %q, want %q", entry.StoragePath, metadataStoragePath("go", key))
|
|
}
|
|
due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Contains(due, legacy) {
|
|
t.Errorf("pending deletes = %v, want %q", due, legacy)
|
|
}
|
|
}
|