package server import ( "context" "io" "net/http" "net/http/httptest" "net/url" "sync/atomic" "testing" "time" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/registries/fetch" ) // npmRetentionUpstream serves a packument and a tarball for lodash and // @babel/core, counting tarball downloads. func npmRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { t.Helper() var tarballs atomic.Int64 var upstream *httptest.Server upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/lodash", "/@babel%2fcore", "/@babel%2Fcore", "/@babel/core": name := "lodash" tarball := upstream.URL + "/lodash/-/lodash-1.0.0.tgz" if r.URL.Path != "/lodash" { name = "@babel/core" tarball = upstream.URL + "/@babel/core/-/core-1.0.0.tgz" } w.Header().Set("Content-Type", "application/json") _, _ = io.WriteString(w, `{"name":"`+name+`","versions":{"1.0.0":{"name":"`+name+ `","version":"1.0.0","dist":{"tarball":"`+tarball+`"}}}}`) case "/lodash/-/lodash-1.0.0.tgz", "/@babel/core/-/core-1.0.0.tgz": tarballs.Add(1) _, _ = io.WriteString(w, "tarball bytes") default: http.NotFound(w, r) } })) t.Cleanup(upstream.Close) return upstream, &tarballs } func newNPMRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { t.Helper() db, store := setupEvictionTest(t) // The upstream's own client reaches the loopback test server, which the // default fetcher refuses. fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) proxy.HTTPClient = upstream.Client() srv := httptest.NewServer(handler.NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) t.Cleanup(srv.Close) return srv, db } func getOK(t *testing.T, url string) { t.Helper() resp, err := http.Get(url) if err != nil { t.Fatalf("GET %s: %v", url, err) } body, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) } } func TestNPMRegistersForRetention(t *testing.T) { if _, ok := retention.Default.Lookup("npm"); !ok { t.Fatal("npm is not registered for retention") } if key, ok := retention.Default.KeyForDBEcosystem("npm"); !ok || key != "npm" { t.Errorf("KeyForDBEcosystem(npm) = %q, %v, want npm, true", key, ok) } } // An npm tarball served from the cache stays while it is being used, is // evicted once its rule runs out, and comes back from upstream on the next // download. func TestNPMRetentionEvictsAndRefetches(t *testing.T) { upstream, tarballs := npmRetentionUpstream(t) srv, db := newNPMRetentionProxy(t, upstream) const versionPURL, filename = "pkg:npm/lodash@1.0.0", "lodash-1.0.0.tgz" tarballURL := srv.URL + "/lodash/-/" + filename getOK(t, tarballURL) getOK(t, tarballURL) if got := tarballs.Load(); got != 1 { t.Fatalf("upstream tarball downloads = %d, want 1 (second from cache)", got) } art, err := db.GetArtifact(versionPURL, filename) if err != nil || !art.LastAccessedAt.Valid { t.Fatalf("cache hit did not record an access: %+v, %v", art, err) } path := art.StoragePath.String rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} now := time.Now() sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) if !isCached(t, db, "npm", "lodash") { t.Fatal("artifact used today was evicted") } sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now.Add(2*retentionDay)) if isCached(t, db, "npm", "lodash") { t.Fatal("artifact unused for two days kept under a one-day rule") } if got := queuedPaths(t, db); len(got) != 1 || got[0] != path { t.Errorf("queue = %v, want [%s]", got, path) } getOK(t, tarballURL) if got := tarballs.Load(); got != 2 { t.Errorf("upstream tarball downloads = %d, want 2 after eviction", got) } // The refetched record keeps the access time of the evicted copy; age it // past the rule so only the fresh fetch time can keep it. backdate(t, db, "last_accessed_at", versionPURL, time.Now().Add(-30*retentionDay)) sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) if !isCached(t, db, "npm", "lodash") { t.Error("refetched artifact evicted again by the next sweep") } } // backdate sets one timestamp column of an artifact's records. func backdate(t *testing.T, db *database.DB, column, versionPURL string, at time.Time) { t.Helper() query := db.Rebind(`UPDATE artifacts SET ` + column + ` = ? WHERE version_purl = ?`) if _, err := db.Exec(query, at, versionPURL); err != nil { t.Fatalf("backdating %s: %v", column, err) } } // A package rule written the way people usually write npm PURLs has to name // the package PURL the handler stores, scoped packages included. func TestNPMRetentionPackageRuleMatchesStoredPackage(t *testing.T) { upstream, _ := npmRetentionUpstream(t) srv, db := newNPMRetentionProxy(t, upstream) getOK(t, srv.URL+"/@babel/core/-/core-1.0.0.tgz") var stored string if err := db.Get(&stored, `SELECT package_purl FROM versions WHERE purl LIKE 'pkg:npm/%core@1.0.0'`); err != nil { t.Fatalf("reading stored package PURL: %v", err) } for _, key := range []string{"pkg:npm/@babel/core", "pkg:npm/%40babel/core"} { cfg := config.Default() cfg.Storage.Retention.Packages = map[string]string{key: "0"} rules, err := cfg.RetentionRules(retention.Default) if err != nil { t.Fatalf("RetentionRules(%s): %v", key, err) } if _, ok := rules.Packages[stored]; !ok { t.Errorf("rule %s resolved to %v, want the stored package %s", key, rules.Packages, stored) } } // With both packages long unused, the "0" rule keeps @babel/core while // the ecosystem rule evicts lodash. getOK(t, srv.URL+"/lodash/-/lodash-1.0.0.tgz") old := time.Now().Add(-30 * retentionDay) for _, versionPURL := range []string{"pkg:npm/lodash@1.0.0", "pkg:npm/%40babel/core@1.0.0"} { backdate(t, db, "fetched_at", versionPURL, old) backdate(t, db, "last_accessed_at", versionPURL, old) } cfg := config.Default() cfg.Storage.Retention.Ecosystems = map[string]string{"npm": "1d"} cfg.Storage.Retention.Packages = map[string]string{"pkg:npm/@babel/core": "0"} rules, err := cfg.RetentionRules(retention.Default) if err != nil { t.Fatalf("RetentionRules: %v", err) } sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) babel, err := db.GetArtifact("pkg:npm/%40babel/core@1.0.0", "core-1.0.0.tgz") if err != nil { t.Fatalf("GetArtifact(@babel/core): %v", err) } if !babel.StoragePath.Valid { t.Error("@babel/core evicted despite its never-evict rule") } if isCached(t, db, "npm", "lodash") { t.Error("lodash kept past the npm rule") } } // rewriteToUpstream sends every request to the test upstream, so the mirror // command's fixed registry.npmjs.org tarball URLs reach it. type rewriteToUpstream struct { target *url.URL base http.RoundTripper } func (r rewriteToUpstream) RoundTrip(req *http.Request) (*http.Response, error) { req = req.Clone(req.Context()) req.URL.Scheme = r.target.Scheme req.URL.Host = r.target.Host req.Host = r.target.Host return r.base.RoundTrip(req) } // A package preloaded with the mirror command falls under the npm rule: // the mirror records it under its own ecosystem spelling and filename. func TestNPMRetentionCoversMirroredPackages(t *testing.T) { upstream, tarballs := npmRetentionUpstream(t) target, err := url.Parse(upstream.URL) if err != nil { t.Fatal(err) } db, store := setupEvictionTest(t) client := &http.Client{Transport: rewriteToUpstream{target: target, base: upstream.Client().Transport}} fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) t.Cleanup(func() { _ = fetcher.Close() }) proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) m := mirror.New(proxy, db, store, discardLogger(), 1) progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: []string{"pkg:npm/lodash@1.0.0"}}) if err != nil || progress.Failed != 0 || tarballs.Load() != 1 { t.Fatalf("mirror run = %+v, %v, tarball downloads %d; want one successful download", progress, err, tarballs.Load()) } backdate(t, db, "fetched_at", "pkg:npm/lodash@1.0.0", time.Now().Add(-30*retentionDay)) rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) if isCached(t, db, "npm", "lodash") { t.Error("never-served mirrored npm artifact kept past its rule") } }