From 67571a2ac8f5b415e326d4c5ec502115488d976e Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 25 Jun 2026 13:03:50 -0400 Subject: [PATCH 01/70] Log actual database config instead of sqlite path The startup log and /stats endpoint always reported Database.Path, which is the sqlite default even when PROXY_DATABASE_DRIVER=postgres is set and a postgres URL is in use. Add DatabaseConfig.String() that returns the sqlite path or the postgres URL with the password redacted, and use it in both places. Fixes #173 --- internal/config/config.go | 15 +++++++++++++++ internal/config/config_test.go | 21 +++++++++++++++++++++ internal/server/server.go | 4 ++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index e84e887..16928dc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -219,6 +219,21 @@ type DatabaseConfig struct { URL string `json:"url" yaml:"url"` } +// String returns a human-readable description of the configured database +// suitable for logging. For postgres the password in the connection URL is +// redacted; if the URL cannot be parsed only the driver name is returned to +// avoid leaking credentials. +func (d DatabaseConfig) String() string { + if d.Driver == "postgres" { + u, err := url.Parse(d.URL) + if err != nil || u.Host == "" { + return "postgres" + } + return u.Redacted() + } + return d.Path +} + // LogConfig configures logging. type LogConfig struct { // Level is the minimum log level: "debug", "info", "warn", "error". diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2f4fdd2..bb3ec74 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -676,3 +676,24 @@ func TestValidateDirectServeBaseURL(t *testing.T) { t.Errorf("unexpected error for valid direct_serve_base_url: %v", err) } } + +func TestDatabaseConfigString(t *testing.T) { + tests := []struct { + name string + cfg DatabaseConfig + want string + }{ + {"sqlite", DatabaseConfig{Driver: "sqlite", Path: "./cache/proxy.db"}, "./cache/proxy.db"}, + {"default driver", DatabaseConfig{Path: "/var/lib/proxy.db"}, "/var/lib/proxy.db"}, + {"postgres no password", DatabaseConfig{Driver: "postgres", URL: "postgres://user@localhost:5432/proxy"}, "postgres://user@localhost:5432/proxy"}, + {"postgres redacts password", DatabaseConfig{Driver: "postgres", URL: "postgres://user:secret@localhost:5432/proxy?sslmode=disable"}, "postgres://user:xxxxx@localhost:5432/proxy?sslmode=disable"}, + {"postgres unparseable url", DatabaseConfig{Driver: "postgres", URL: "host=localhost user=foo password=bar"}, "postgres"}, + {"postgres ignores sqlite path", DatabaseConfig{Driver: "postgres", URL: "postgres://localhost/db", Path: "./cache/proxy.db"}, "postgres://localhost/db"}, + } + + for _, tt := range tests { + if got := tt.cfg.String(); got != tt.want { + t.Errorf("%s: String() = %q, want %q", tt.name, got, tt.want) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index cb99648..0a46a37 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -302,7 +302,7 @@ func (s *Server) Start() error { "base_url", s.cfg.BaseURL, "ui_url", s.cfg.UIBaseURL, "storage", s.storage.URL(), - "database", s.cfg.Database.Path) + "database", s.cfg.Database.String()) go s.updateCacheStatsMetrics() go s.startEvictionLoop(bgCtx) @@ -927,7 +927,7 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { TotalSize: size, TotalSizeHuman: formatSize(size), StorageURL: s.storage.URL(), - DatabasePath: s.cfg.Database.Path, + DatabasePath: s.cfg.Database.String(), } w.Header().Set("Content-Type", "application/json") From b2011c0a546331f0f853f5de5fa1fb025da2a896 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 26 Jun 2026 07:42:30 -0400 Subject: [PATCH 02/70] Log actual database config instead of sqlite path (#176) The startup log and /stats endpoint always reported Database.Path, which is the sqlite default even when PROXY_DATABASE_DRIVER=postgres is set and a postgres URL is in use. Add DatabaseConfig.String() that returns the sqlite path or the postgres URL with the password redacted, and use it in both places. Fixes #173 --- internal/config/config.go | 15 +++++++++++++++ internal/config/config_test.go | 21 +++++++++++++++++++++ internal/server/server.go | 4 ++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index e84e887..16928dc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -219,6 +219,21 @@ type DatabaseConfig struct { URL string `json:"url" yaml:"url"` } +// String returns a human-readable description of the configured database +// suitable for logging. For postgres the password in the connection URL is +// redacted; if the URL cannot be parsed only the driver name is returned to +// avoid leaking credentials. +func (d DatabaseConfig) String() string { + if d.Driver == "postgres" { + u, err := url.Parse(d.URL) + if err != nil || u.Host == "" { + return "postgres" + } + return u.Redacted() + } + return d.Path +} + // LogConfig configures logging. type LogConfig struct { // Level is the minimum log level: "debug", "info", "warn", "error". diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2f4fdd2..bb3ec74 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -676,3 +676,24 @@ func TestValidateDirectServeBaseURL(t *testing.T) { t.Errorf("unexpected error for valid direct_serve_base_url: %v", err) } } + +func TestDatabaseConfigString(t *testing.T) { + tests := []struct { + name string + cfg DatabaseConfig + want string + }{ + {"sqlite", DatabaseConfig{Driver: "sqlite", Path: "./cache/proxy.db"}, "./cache/proxy.db"}, + {"default driver", DatabaseConfig{Path: "/var/lib/proxy.db"}, "/var/lib/proxy.db"}, + {"postgres no password", DatabaseConfig{Driver: "postgres", URL: "postgres://user@localhost:5432/proxy"}, "postgres://user@localhost:5432/proxy"}, + {"postgres redacts password", DatabaseConfig{Driver: "postgres", URL: "postgres://user:secret@localhost:5432/proxy?sslmode=disable"}, "postgres://user:xxxxx@localhost:5432/proxy?sslmode=disable"}, + {"postgres unparseable url", DatabaseConfig{Driver: "postgres", URL: "host=localhost user=foo password=bar"}, "postgres"}, + {"postgres ignores sqlite path", DatabaseConfig{Driver: "postgres", URL: "postgres://localhost/db", Path: "./cache/proxy.db"}, "postgres://localhost/db"}, + } + + for _, tt := range tests { + if got := tt.cfg.String(); got != tt.want { + t.Errorf("%s: String() = %q, want %q", tt.name, got, tt.want) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index cb99648..0a46a37 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -302,7 +302,7 @@ func (s *Server) Start() error { "base_url", s.cfg.BaseURL, "ui_url", s.cfg.UIBaseURL, "storage", s.storage.URL(), - "database", s.cfg.Database.Path) + "database", s.cfg.Database.String()) go s.updateCacheStatsMetrics() go s.startEvictionLoop(bgCtx) @@ -927,7 +927,7 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { TotalSize: size, TotalSizeHuman: formatSize(size), StorageURL: s.storage.URL(), - DatabasePath: s.cfg.Database.Path, + DatabasePath: s.cfg.Database.String(), } w.Header().Set("Content-Type", "application/json") From ca2514991d335739b77dc6d7976b48b3b38a696c Mon Sep 17 00:00:00 2001 From: wickedOne Date: Fri, 26 Jun 2026 13:51:08 +0200 Subject: [PATCH 03/70] fix(composer): resolve *-dev versions from the ~dev metadata file (#174) --- internal/handler/composer.go | 98 ++++++++++++++++++++++--------- internal/handler/composer_test.go | 97 ++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 28 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 065ddf9..e4dec41 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1,6 +1,7 @@ package handler import ( + "context" "encoding/json" "errors" "fmt" @@ -307,37 +308,28 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) h.proxy.Logger.Info("composer download request", "package", packageName, "version", version, "filename", filename) - // We need to fetch the metadata to get the actual download URL - // since Packagist URLs include a hash - metaURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) + // We need to fetch the metadata to get the actual download URL since + // Packagist URLs include a hash. Packagist serves dev versions (e.g. + // "3.x-dev", "dev-master") from a separate "~dev" metadata file, while + // tagged releases live in the regular file. Try the file most likely to + // contain this version first, then fall back to the other so that both + // stable and dev versions resolve correctly. + metaURLs := h.metadataURLsForVersion(vendor, pkg, version) - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, metaURL, nil) - if err != nil { - http.Error(w, "failed to create request", http.StatusInternalServerError) - return + var downloadURL string + for _, metaURL := range metaURLs { + url, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) + if err != nil { + h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL) + http.Error(w, "failed to fetch metadata", http.StatusBadGateway) + return + } + if url != "" { + downloadURL = url + break + } } - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - h.proxy.Logger.Error("failed to fetch metadata", "error", err) - http.Error(w, "failed to fetch metadata", http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - http.Error(w, "package not found", http.StatusNotFound) - return - } - - var metadata map[string]any - if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { - http.Error(w, "failed to parse metadata", http.StatusInternalServerError) - return - } - - // Find the download URL for this version - downloadURL := h.findDownloadURL(metadata, packageName, version) if downloadURL == "" { http.Error(w, "version not found", http.StatusNotFound) return @@ -353,6 +345,56 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) ServeArtifact(w, result) } +// isDevVersion reports whether a Composer version string refers to a +// development (unstable, branch) version rather than a tagged release. +// Composer formats these as either "dev-" (e.g. "dev-master") or +// "-dev" (e.g. "3.x-dev"). +func isDevVersion(version string) bool { + return strings.HasPrefix(version, "dev-") || strings.HasSuffix(version, "-dev") +} + +// metadataURLsForVersion returns the upstream metadata URLs to consult for a +// given version, in priority order. Dev versions are served from the "~dev" +// file, tagged releases from the regular file; the other file is included as a +// fallback so an unexpected classification still resolves. +func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string { + stable := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) + dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", h.repoURL, vendor, pkg) + + if isDevVersion(version) { + return []string{dev, stable} + } + return []string{stable, dev} +} + +// findDownloadURLFromMetadata fetches a metadata document and returns the dist +// URL for the given version, or an empty string if the version is not present. +// An error is returned only on transport failure; a missing document (non-200) +// or a missing version both yield an empty string so the caller can fall back. +func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil) + if err != nil { + return "", err + } + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", nil + } + + var metadata map[string]any + if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { + return "", err + } + + return h.findDownloadURL(metadata, packageName, version), nil +} + // findDownloadURL finds the dist URL for a specific version in metadata. func (h *ComposerHandler) findDownloadURL(metadata map[string]any, packageName, version string) string { packages, ok := metadata["packages"].(map[string]any) diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index baf13b6..9898664 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -1,8 +1,11 @@ package handler import ( + "context" "encoding/json" "log/slog" + "net/http" + "net/http/httptest" "strings" "testing" "time" @@ -465,6 +468,100 @@ func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) { } } +// TestComposerDownloadDevVersionUsesDevMetadata is a regression test for the +// bug that made it impossible to install a *-dev dependency from dist. +// +// Packagist serves development versions (e.g. "3.x-dev", "dev-master") from a +// separate "{package}~dev.json" metadata file; the regular "{package}.json" +// file contains only tagged releases. The download handler used to fetch only +// the regular file, so it could never find the dist URL for a dev version and +// returned 404 — causing Composer to silently fall back to a git clone. +// +// This test serves both files from a mock upstream and asserts that: +// - the OLD behavior (regular file only) cannot resolve the dev version, and +// - the FIXED behavior (consulting the ~dev file) does. +func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { + const ( + pkg = "phpmd/phpmd" + vendor = "phpmd" + name = "phpmd" + version = "3.x-dev" + distURL = "https://api.github.com/repos/phpmd/phpmd/zipball/2a9217f60aaf27bf6ddad9188f254d020ab70745" + ) + + // Regular metadata: tagged releases only — no dev versions. + stableBody := `{ + "packages": { + "phpmd/phpmd": [ + {"version": "2.15.0", "dist": {"url": "https://example.com/2.15.0.zip", "type": "zip"}} + ] + } + }` + + // ~dev metadata: where the 3.x-dev version actually lives. + devBody := `{ + "packages": { + "phpmd/phpmd": [ + {"version": "3.x-dev", "dist": {"url": "` + distURL + `", "type": "zip"}} + ] + } + }` + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/p2/phpmd/phpmd.json": + _, _ = w.Write([]byte(stableBody)) + case "/p2/phpmd/phpmd~dev.json": + _, _ = w.Write([]byte(devBody)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + h := &ComposerHandler{ + proxy: testProxy(), + repoURL: srv.URL, + proxyURL: "http://localhost:8080", + } + + ctx := context.Background() + + // OLD behavior: fetching only the regular file fails to resolve the dev + // version, which is what produced the 404 before the fix. + stableURL := srv.URL + "/p2/phpmd/phpmd.json" + got, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version) + if err != nil { + t.Fatalf("unexpected error fetching regular metadata: %v", err) + } + if got != "" { + t.Fatalf("regular metadata unexpectedly contained dev version %q (got %q); "+ + "the test no longer reproduces the original bug", version, got) + } + + // FIXED behavior: the handler consults the ~dev file (it is first in the + // candidate list for dev versions) and resolves the dist URL. + urls := h.metadataURLsForVersion(vendor, name, version) + if len(urls) == 0 || !strings.HasSuffix(urls[0], "/p2/phpmd/phpmd~dev.json") { + t.Fatalf("dev version should consult the ~dev metadata file first, got %v", urls) + } + + var resolved string + for _, u := range urls { + resolved, err = h.findDownloadURLFromMetadata(ctx, u, pkg, version) + if err != nil { + t.Fatalf("unexpected error fetching metadata %q: %v", u, err) + } + if resolved != "" { + break + } + } + + if resolved != distURL { + t.Errorf("dev version dist URL = %q, want %q", resolved, distURL) + } +} + func TestComposerRewriteMetadataCooldown(t *testing.T) { now := time.Now() old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339) From f4a8c5606a003f737e687552da44943885579813 Mon Sep 17 00:00:00 2001 From: Bruno Clermont Date: Fri, 26 Jun 2026 07:54:00 -0400 Subject: [PATCH 04/70] Fix package redirection to '/ui/packages' (#169) cause I get 404 every time --- internal/server/templates/pages/packages_list.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/server/templates/pages/packages_list.html b/internal/server/templates/pages/packages_list.html index 5ef42c7..95897db 100644 --- a/internal/server/templates/pages/packages_list.html +++ b/internal/server/templates/pages/packages_list.html @@ -100,7 +100,7 @@ document.getElementById('ecosystem-filter').addEventListener('change', function( const params = new URLSearchParams(); if (ecosystem) params.set('ecosystem', ecosystem); if (sortBy) params.set('sort', sortBy); - window.location.href = '/packages?' + params.toString(); + window.location.href = '/ui/packages?' + params.toString(); }); document.getElementById('sort-by').addEventListener('change', function(e) { @@ -109,7 +109,7 @@ document.getElementById('sort-by').addEventListener('change', function(e) { const params = new URLSearchParams(); if (ecosystem) params.set('ecosystem', ecosystem); if (sortBy) params.set('sort', sortBy); - window.location.href = '/packages?' + params.toString(); + window.location.href = '/ui/packages?' + params.toString(); }); {{end}} From bb214bb7b2c771c8c8d36213027456491f47b648 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 29 Jun 2026 14:43:49 +0100 Subject: [PATCH 05/70] Bump git-pkgs and third-party dependencies (#177) --- go.mod | 28 +++++++++++------------ go.sum | 70 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index a1b22d6..471ab53 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,12 @@ require ( github.com/CycloneDX/cyclonedx-go v0.11.0 github.com/git-pkgs/archives v0.3.0 github.com/git-pkgs/cooldown v0.1.1 - github.com/git-pkgs/enrichment v0.3.0 - github.com/git-pkgs/purl v0.1.12 - github.com/git-pkgs/registries v0.6.1 + github.com/git-pkgs/enrichment v0.4.1 + github.com/git-pkgs/purl v0.1.13 + github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.2.6 - github.com/git-pkgs/vulns v0.1.5 + github.com/git-pkgs/vulns v0.1.6 github.com/go-chi/chi/v5 v5.3.0 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 @@ -24,7 +24,7 @@ require ( golang.org/x/sync v0.21.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.52.0 + modernc.org/sqlite v1.53.0 ) require ( @@ -115,7 +115,7 @@ require ( github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ecosyste-ms/ecosystems-go v0.1.1 // indirect + github.com/ecosyste-ms/ecosystems-go v0.2.0 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/fatih/color v1.18.0 // indirect @@ -125,7 +125,7 @@ require ( github.com/fzipp/gocyclo v0.6.0 // indirect github.com/ghostiam/protogetter v0.3.20 // indirect github.com/git-pkgs/packageurl-go v0.3.1 // indirect - github.com/git-pkgs/pom v0.1.4 // indirect + github.com/git-pkgs/pom v0.1.5 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-critic/go-critic v0.14.3 // indirect github.com/go-logr/logr v1.4.3 // indirect @@ -217,7 +217,7 @@ require ( github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect - github.com/oapi-codegen/runtime v1.2.0 // indirect + github.com/oapi-codegen/runtime v1.4.1 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -291,15 +291,15 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.49.0 // indirect + golang.org/x/crypto v0.51.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect - golang.org/x/mod v0.33.0 // indirect - golang.org/x/net v0.52.0 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/net v0.54.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.35.0 // indirect - golang.org/x/tools v0.42.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/tools v0.45.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect @@ -307,7 +307,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect - modernc.org/libc v1.72.3 // indirect + modernc.org/libc v1.73.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect diff --git a/go.sum b/go.sum index c1a5bb8..8d8c96d 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ecosyste-ms/ecosystems-go v0.1.1 h1:YYiBK9TCCTeE+BtmpN2FssaRFcmF+T0v4LrupIOjehQ= -github.com/ecosyste-ms/ecosystems-go v0.1.1/go.mod h1:VczXs1CO9nL8XbL1NwvgmwIaqzMsAxcsXnTpRtwi9gU= +github.com/ecosyste-ms/ecosystems-go v0.2.0 h1:Nhpg54C+St8Sd/mf8bNJmQqx35ZgHw33TfFoxaXMQI8= +github.com/ecosyste-ms/ecosystems-go v0.2.0/go.mod h1:CCdzT1iAZirbEZAbFSnWpK88eKKaIWex7gjtZ0UudXA= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= @@ -248,22 +248,22 @@ github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= -github.com/git-pkgs/enrichment v0.3.0 h1:tsATMAwR/qcSIw4JV37uH2TBgTv415RfJC7w3nzWfD4= -github.com/git-pkgs/enrichment v0.3.0/go.mod h1:MBv5nhHzjwLxeSgx2+7waCcpReUjhCD+9B0bvufpMO0= +github.com/git-pkgs/enrichment v0.4.1 h1:A8BKs0XwvpF1sF5qviZy4fkJAe18qB9OgpbRnmwnT34= +github.com/git-pkgs/enrichment v0.4.1/go.mod h1:stHqZUitV9ZkwACqHzBysLMSe6T4QZn81hxTdSroNhM= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= -github.com/git-pkgs/pom v0.1.4 h1:C6st+XSbF75eKuwfdkDZZtYHoTcaWRIEQYar5VtszUo= -github.com/git-pkgs/pom v0.1.4/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= -github.com/git-pkgs/purl v0.1.12 h1:qCskrEU1LWQhCkIVZd992W5++Bsxazvx2Cx1/65qCvU= -github.com/git-pkgs/purl v0.1.12/go.mod h1:ofp4mHsR0cUeVONQaf33n6Wxg2QTEvtUdRfCedI8ouA= -github.com/git-pkgs/registries v0.6.1 h1:xZfVZQmffIfdeJthn5o2EozbVJ6gBeImYwKQnfdKUfU= -github.com/git-pkgs/registries v0.6.1/go.mod h1:a3BP/56VW3O/CFRqiJCtSy+OqRrSH25wF1PWHP76ka0= +github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= +github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= +github.com/git-pkgs/purl v0.1.13 h1:at8BU6vnP5oonHFHAPA064BzgRqij+SZcOUDgNT2DC8= +github.com/git-pkgs/purl v0.1.13/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= +github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+u0a8= +github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8= github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= -github.com/git-pkgs/vulns v0.1.5 h1:mtX88/27toFl+B95kaH5QbAdOCQ3YIDGjJrlrrnqQTE= -github.com/git-pkgs/vulns v0.1.5/go.mod h1:bZFikfrR/5gC0ZMwXh7qcEu2gpKfXMBhVsy4kF12Ae0= +github.com/git-pkgs/vulns v0.1.6 h1:8RRSgdlxp4JMU0Zykr63XTOMo5CyZKwt/PwaQxrx9Yg= +github.com/git-pkgs/vulns v0.1.6/go.mod h1:TsZC4MjoCkKJslgmbcmRCnytwnFcjESC2N8b0a2xDWc= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= @@ -510,8 +510,10 @@ github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= github.com/nunnatsa/ginkgolinter v0.23.0 h1:x3o4DGYOWbBMP/VdNQKgSj+25aJKx2Pe6lHr8gBcgf8= github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= -github.com/oapi-codegen/runtime v1.2.0 h1:RvKc1CVS1QeKSNzO97FBQbSMZyQ8s6rZd+LpmzwHMP4= -github.com/oapi-codegen/runtime v1.2.0/go.mod h1:Y7ZhmmlE8ikZOmuHRRndiIm7nf3xcVv+YMweKgG1DT0= +github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= +github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= +github.com/oapi-codegen/runtime v1.4.1 h1:9nwLoI+KrWxzbBcp0jO/R8uXqbik/HUyCvPeU68Y/qo= +github.com/oapi-codegen/runtime v1.4.1/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= @@ -736,8 +738,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= @@ -753,8 +755,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -769,8 +771,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -821,8 +823,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -837,8 +839,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= -golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= @@ -880,20 +882,20 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= -modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= -modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= -modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= +modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= +modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= -modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= +modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= -modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -902,8 +904,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.52.0 h1:p4dhYh2tXZCiyaqHwRVJDjIGKWyXayiQpThxgDzJaxo= -modernc.org/sqlite v1.52.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 787b76dfe7d503e19de0dc0cfd9736ab471386ac Mon Sep 17 00:00:00 2001 From: Philipp Dieter Date: Tue, 30 Jun 2026 13:25:49 +0200 Subject: [PATCH 06/70] Fix Composer 404 on minified metadata and add debug logging - Expand minified Composer v2 metadata in findDownloadURLFromMetadata so versions that inherit dist from a previous entry resolve correctly - Add Debug()-level log statements in the download resolution path (handleDownload, findDownloadURLFromMetadata) so -log-level debug produces meaningful output --- internal/handler/composer.go | 39 +++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index e4dec41..bc3bc1d 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -316,6 +316,10 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) // stable and dev versions resolve correctly. metaURLs := h.metadataURLsForVersion(vendor, pkg, version) + h.proxy.Logger.Debug("resolving download URL", + "package", packageName, "version", version, + "metadata_urls", metaURLs) + var downloadURL string for _, metaURL := range metaURLs { url, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) @@ -331,10 +335,17 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) } if downloadURL == "" { + h.proxy.Logger.Debug("version not found in any metadata source", + "package", packageName, "version", version, + "tried_urls", metaURLs) http.Error(w, "version not found", http.StatusNotFound) return } + h.proxy.Logger.Debug("resolved download URL", + "package", packageName, "version", version, + "download_url", downloadURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL) if err != nil { h.proxy.Logger.Error("failed to get artifact", "error", err) @@ -372,6 +383,9 @@ func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) [] // An error is returned only on transport failure; a missing document (non-200) // or a missing version both yield an empty string so the caller can fall back. func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, error) { + h.proxy.Logger.Debug("fetching upstream metadata for download lookup", + "url", metaURL, "package", packageName, "version", version) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil) if err != nil { return "", err @@ -383,6 +397,9 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU } defer func() { _ = resp.Body.Close() }() + h.proxy.Logger.Debug("upstream metadata response", + "url", metaURL, "status", resp.StatusCode) + if resp.StatusCode != http.StatusOK { return "", nil } @@ -392,7 +409,27 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU return "", err } - return h.findDownloadURL(metadata, packageName, version), nil + // Expand minified Composer v2 format so that inherited fields (including + // dist) are present on every version entry. Without this, versions that + // inherit dist from a previous entry will appear to have no download URL. + if metadata["minified"] == "composer/2.0" { + h.proxy.Logger.Debug("expanding minified metadata", "url", metaURL) + if packages, ok := metadata["packages"].(map[string]any); ok { + for pkgName, versions := range packages { + versionList, ok := versions.([]any) + if !ok { + continue + } + packages[pkgName] = expandMinifiedVersions(versionList) + } + } + } + + url := h.findDownloadURL(metadata, packageName, version) + h.proxy.Logger.Debug("download URL lookup result", + "url", metaURL, "package", packageName, "version", version, + "download_url", url) + return url, nil } // findDownloadURL finds the dist URL for a specific version in metadata. From 81d19b7632bbc0cbb5a24cdc29f1ad6534cc1e1e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:13:19 +0000 Subject: [PATCH 07/70] Bump zizmorcore/zizmor-action from 0.5.6 to 0.5.7 Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.5.6 to 0.5.7. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/5f14fd08f7cf1cb1609c1e344975f152c7ee938d...192e21d79ab29983730a13d1382995c2307fbcaa) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index b404599..cce6e4f 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 + uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 From a8c0562c97fe20619e75b5d0295889f36178055b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:13:48 +0000 Subject: [PATCH 08/70] Bump actions/setup-go from 6.4.0 to 6.5.0 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 6.4.0 to 6.5.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/4a3601121dd01d1626a1e23e37211e3254c1c06c...924ae3a1cded613372ab5595356fb5720e22ba16) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: 6.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release.yml | 2 +- .github/workflows/swagger.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1a2abb..700f28e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version: ${{ matrix.go-version }} @@ -40,7 +40,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version: '1.25' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a7d8ae5..f7833fb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Set up Go - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod cache: false diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index b6e086a..f947c7e 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -17,7 +17,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version: '1.25' From 0ae63066e22c3991cecb4f58954a2f89f07c36fa Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 7 Jul 2026 17:48:36 +0100 Subject: [PATCH 09/70] Make upstream HTTP timeout configurable Add http_timeout config option (and PROXY_HTTP_TIMEOUT env var) to set the per-request timeout on the shared HTTP client used by protocol handlers for upstream metadata fetches and pass-through file requests. Defaults to the previous hardcoded 30s; "0" disables the timeout. Fixes #187 --- config.example.yaml | 5 +++ docs/configuration.md | 12 +++++++ internal/config/config.go | 44 ++++++++++++++++++++++++ internal/config/config_test.go | 63 ++++++++++++++++++++++++++++++++++ internal/server/server.go | 1 + 5 files changed, 125 insertions(+) diff --git a/config.example.yaml b/config.example.yaml index 62b4105..d1ed9a1 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -9,6 +9,11 @@ listen: ":8080" # so users know what to point their package manager at. base_url: "http://localhost:8080" +# Timeout for individual upstream HTTP requests made by protocol handlers +# (metadata fetches, pass-through file requests). Uses Go duration syntax. +# Set to "0" to disable the timeout. Default: "30s". +# http_timeout: "30s" + # Public URL where the web UI is reached. Defaults to base_url when unset. # Set this separately when the UI is served on a different hostname than the # package endpoints — for example, the UI on a public domain behind auth while diff --git a/docs/configuration.md b/docs/configuration.md index f220279..dcdec24 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -276,6 +276,18 @@ metadata_max_size: "100MB" # default Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`. +## Upstream HTTP timeout + +Protocol handlers use a shared HTTP client for upstream requests such as metadata fetches and pass-through file downloads. `http_timeout` sets that client's per-request timeout. Raise it if slow upstreams or large metadata responses cause `context deadline exceeded` errors. + +```yaml +http_timeout: "30s" # default +``` + +Or via environment variable: `PROXY_HTTP_TIMEOUT=2m`. + +Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout). + ## Mirror API The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP: diff --git a/internal/config/config.go b/internal/config/config.go index 16928dc..2d275c7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -110,6 +110,12 @@ type Config struct { // size return ErrMetadataTooLarge. Default: "100MB". MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"` + // HTTPTimeout is the timeout for individual upstream HTTP requests made + // by protocol handlers (metadata fetches, pass-through file requests). + // Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s". + // Set to "0" to disable the timeout entirely. + HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"` + // MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP. // Disabled by default to prevent unauthenticated users from triggering downloads. MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` @@ -460,6 +466,9 @@ func (c *Config) LoadFromEnv() { if v := os.Getenv("PROXY_METADATA_MAX_SIZE"); v != "" { c.MetadataMaxSize = v } + if v := os.Getenv("PROXY_HTTP_TIMEOUT"); v != "" { + c.HTTPTimeout = v + } if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY"); v != "" { c.Gradle.BuildCache.ReadOnly = v == "true" || v == "1" } @@ -567,6 +576,10 @@ func (c *Config) Validate() error { return err } + if err := validateHTTPTimeout(c.HTTPTimeout); err != nil { + return err + } + if err := c.Health.Validate(); err != nil { return err } @@ -636,6 +649,7 @@ func (g *GradleBuildCacheConfig) Validate() error { const ( defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default + defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default defaultMetadataMaxSize = 100 << 20 defaultGradleBuildCacheMaxUploadSize = 100 << 20 defaultGradleBuildCacheSweepInterval = 10 * time.Minute @@ -656,6 +670,20 @@ func (c *Config) ParseMaxSize() int64 { return size } +func validateHTTPTimeout(s string) error { + if s == "" || s == "0" { + return nil + } + d, err := time.ParseDuration(s) + if err != nil { + return fmt.Errorf("invalid http_timeout %q: %w", s, err) + } + if d < 0 { + return fmt.Errorf("invalid http_timeout %q: must be non-negative", s) + } + return nil +} + func validateMetadataMaxSize(s string) error { if s == "" { return nil @@ -683,6 +711,22 @@ func (c *Config) ParseMetadataMaxSize() int64 { return size } +// ParseHTTPTimeout returns the upstream HTTP client timeout. +// Returns 30s if unset, 0 (no timeout) if explicitly set to "0". +func (c *Config) ParseHTTPTimeout() time.Duration { + if c.HTTPTimeout == "" { + return defaultHTTPTimeout + } + if c.HTTPTimeout == "0" { + return 0 + } + d, err := time.ParseDuration(c.HTTPTimeout) + if err != nil || d < 0 { + return defaultHTTPTimeout + } + return d +} + // ParseMetadataTTL returns the metadata TTL duration. // Returns 5 minutes if unset, 0 if explicitly disabled. func (c *Config) ParseMetadataTTL() time.Duration { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bb3ec74..9c34023 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -524,6 +524,69 @@ func TestValidateHealthStorageProbeInterval(t *testing.T) { } } +func TestParseHTTPTimeout(t *testing.T) { + tests := []struct { + name string + timeout string + want time.Duration + }{ + {"empty defaults to 30s", "", 30 * time.Second}, + {"explicit zero disables", "0", 0}, + {"2 minutes", "2m", 2 * time.Minute}, + {"90 seconds", "90s", 90 * time.Second}, + {"invalid defaults to 30s", "not-a-duration", 30 * time.Second}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.HTTPTimeout = tt.timeout + got := cfg.ParseHTTPTimeout() + if got != tt.want { + t.Errorf("ParseHTTPTimeout() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestValidateHTTPTimeout(t *testing.T) { + cfg := Default() + cfg.HTTPTimeout = "not-a-duration" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for invalid http_timeout") + } + + cfg.HTTPTimeout = "-5s" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for negative http_timeout") + } + + cfg.HTTPTimeout = "2m" + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for valid http_timeout: %v", err) + } + + cfg.HTTPTimeout = "0" + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for zero http_timeout: %v", err) + } + + cfg.HTTPTimeout = "" + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for empty http_timeout: %v", err) + } +} + +func TestLoadHTTPTimeoutFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_HTTP_TIMEOUT", "90s") + cfg.LoadFromEnv() + + if cfg.HTTPTimeout != "90s" { + t.Errorf("HTTPTimeout = %q, want %q", cfg.HTTPTimeout, "90s") + } +} + func TestLoadMetadataTTLFromEnv(t *testing.T) { cfg := Default() t.Setenv("PROXY_METADATA_TTL", "10m") diff --git a/internal/server/server.go b/internal/server/server.go index 0a46a37..5aac4db 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -166,6 +166,7 @@ func (s *Server) Start() error { Packages: s.cfg.Cooldown.Packages, } proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) + proxy.HTTPClient.Timeout = s.cfg.ParseHTTPTimeout() proxy.Cooldown = cd proxy.CacheMetadata = s.cfg.CacheMetadata proxy.MetadataTTL = s.cfg.ParseMetadataTTL() From 1a18dd4635f3eb6289992ef54783579a4b51199c Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 7 Jul 2026 17:53:35 +0100 Subject: [PATCH 10/70] Bump golang.org/x/net to v0.56.0 Clears GHSA-5cv4-jp36-h3mw / CVE-2026-25680 (HTML parser DoS, only reachable via golangci-lint tooling deps) and GO-2026-5026 (idna Punycode validation, reachable via http.Client.Do). Also pulls x/crypto, x/sys and x/text forward as required by x/net. --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 471ab53..9576fda 100644 --- a/go.mod +++ b/go.mod @@ -291,14 +291,14 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.36.0 // indirect - golang.org/x/net v0.54.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect golang.org/x/tools v0.45.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect diff --git a/go.sum b/go.sum index 8d8c96d..93bed13 100644 --- a/go.sum +++ b/go.sum @@ -738,8 +738,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= @@ -771,8 +771,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -807,8 +807,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -823,8 +823,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From df3ad9184821b60efcbb5198d7b65b6dd2949d1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:13:56 +0000 Subject: [PATCH 11/70] Bump docker/build-push-action from 7.2.0 to 7.3.0 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 7.2.0 to 7.3.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/f9f3042f7e2789586610d6e8b85c8f03e5195baf...53b7df96c91f9c12dcc8a07bcb9ccacbed38856a) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 7.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 93b0a1e..a4e105d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,7 +38,7 @@ jobs: images: ghcr.io/${{ github.repository }} - name: Build and push Docker image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a with: context: . push: true From f7b5dd2028e80f420cdf1284ccb74f3b5ce4f6b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:14:04 +0000 Subject: [PATCH 12/70] Bump golang.org/x/sync from 0.21.0 to 0.22.0 Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.21.0 to 0.22.0. - [Commits](https://github.com/golang/sync/compare/v0.21.0...v0.22.0) --- updated-dependencies: - dependency-name: golang.org/x/sync dependency-version: 0.22.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9576fda..b063dba 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 gocloud.dev v0.46.0 - golang.org/x/sync v0.21.0 + golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 93bed13..b1d35f6 100644 --- a/go.sum +++ b/go.sum @@ -784,8 +784,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From ea4eea4b113c41523a8a48f40158f0d87ba577a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:14:07 +0000 Subject: [PATCH 13/70] Bump docker/login-action from 4.2.0 to 4.3.0 Bumps [docker/login-action](https://github.com/docker/login-action) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/650006c6eb7dba73a995cc03b0b2d7f5ca915bee...c99871dec2022cc055c062a10cc1a1310835ceb4) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 93b0a1e..f9c32a5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: Log in to the Container registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 with: registry: ghcr.io username: ${{ github.actor }} From defe1ec96db676d7833dabb65e5540ebf539000f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:14:16 +0000 Subject: [PATCH 14/70] Bump goreleaser/goreleaser-action from 7.2.2 to 7.2.3 Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 7.2.2 to 7.2.3. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89...f06c13b6b1a9625abc9e6e439d9c05a8f2190e94) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-version: 7.2.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7833fb..4d4d0b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: go-version-file: go.mod cache: false - - uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 + - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 with: version: "~> v2" args: release --clean From 383d2089cf30e313d46f656c49c4fcddc573759e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:14:26 +0000 Subject: [PATCH 15/70] Bump docker/metadata-action from 6.1.0 to 6.2.0 Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 6.1.0 to 6.2.0. - [Release notes](https://github.com/docker/metadata-action/releases) - [Commits](https://github.com/docker/metadata-action/compare/80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9...dc802804100637a589fabce1cb79ff13a1411302) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-version: 6.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 93b0a1e..b2288be 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,7 +33,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 with: images: ghcr.io/${{ github.repository }} From 41465968cbaf3e52cd2b552b9c7610000224a54d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 9 Jul 2026 17:36:18 +0100 Subject: [PATCH 16/70] Validate filesystem storage paths --- internal/storage/filesystem.go | 30 +++++++++++--- internal/storage/filesystem_test.go | 62 +++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/internal/storage/filesystem.go b/internal/storage/filesystem.go index 1e5a24f..d509e9d 100644 --- a/internal/storage/filesystem.go +++ b/internal/storage/filesystem.go @@ -34,11 +34,31 @@ func NewFilesystem(root string) (*Filesystem, error) { } func (fs *Filesystem) fullPath(path string) (string, error) { - full := filepath.Clean(filepath.Join(fs.root, filepath.FromSlash(path))) - if full != fs.root && !strings.HasPrefix(full, fs.root+string(filepath.Separator)) { - return "", fmt.Errorf("%w: path escapes storage root", ErrNotFound) + localPath, err := cleanStoragePath(path) + if err != nil { + return "", err } - return full, nil + return filepath.Join(fs.root, localPath), nil +} + +func (fs *Filesystem) prefixPath(prefix string) (string, error) { + if prefix == "" { + return fs.root, nil + } + return fs.fullPath(prefix) +} + +func cleanStoragePath(path string) (string, error) { + if path == "." || strings.Contains(path, `\`) || !filepath.IsLocal(path) { + return "", fmt.Errorf("%w: invalid storage path", ErrNotFound) + } + + localPath, err := filepath.Localize(path) + if err != nil || localPath == "." || !filepath.IsLocal(localPath) { + return "", fmt.Errorf("%w: invalid storage path", ErrNotFound) + } + + return localPath, nil } func (fs *Filesystem) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { @@ -190,7 +210,7 @@ func (fs *Filesystem) UsedSpace(ctx context.Context) (int64, error) { // ListPrefix returns object metadata for paths under a prefix. func (fs *Filesystem) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { - searchRoot, err := fs.fullPath(prefix) + searchRoot, err := fs.prefixPath(prefix) if err != nil { return nil, err } diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go index 332dfbf..10d073b 100644 --- a/internal/storage/filesystem_test.go +++ b/internal/storage/filesystem_test.go @@ -243,19 +243,65 @@ func TestFilesystemLargeFile(t *testing.T) { assertLargeFileRoundTrip(t, createTestFilesystem(t)) } -func TestFilesystemRejectsTraversal(t *testing.T) { +func TestFilesystemRejectsInvalidPaths(t *testing.T) { tmp := t.TempDir() fs, err := NewFilesystem(tmp) if err != nil { t.Fatal(err) } - for _, p := range []string{"../etc/passwd", "../../etc/passwd", "a/../../etc/passwd"} { - if _, err := fs.Open(context.Background(), p); err == nil { - t.Errorf("Open(%q) should reject traversal", p) - } - if _, _, err := fs.Store(context.Background(), p, strings.NewReader("x")); err == nil { - t.Errorf("Store(%q) should reject traversal", p) - } + for _, p := range []string{ + "", + ".", + "../etc/passwd", + "../../etc/passwd", + "a/../../etc/passwd", + "/etc/passwd", + "test//file.txt", + "test/./file.txt", + "test/../file.txt", + `test\..\file.txt`, + } { + t.Run(p, func(t *testing.T) { + ctx := context.Background() + + if _, err := fs.FullPath(p); !errors.Is(err, ErrNotFound) { + t.Errorf("FullPath(%q) = %v, want ErrNotFound", p, err) + } + if _, err := fs.Open(ctx, p); err == nil { + t.Errorf("Open(%q) should reject invalid path", p) + } + if _, _, err := fs.Store(ctx, p, strings.NewReader("x")); err == nil { + t.Errorf("Store(%q) should reject invalid path", p) + } + if _, err := fs.Exists(ctx, p); err == nil { + t.Errorf("Exists(%q) should reject invalid path", p) + } + if err := fs.Delete(ctx, p); err == nil { + t.Errorf("Delete(%q) should reject invalid path", p) + } + if _, err := fs.Size(ctx, p); err == nil { + t.Errorf("Size(%q) should reject invalid path", p) + } + if _, err := fs.ListPrefix(ctx, p); p != "" && err == nil { + t.Errorf("ListPrefix(%q) should reject invalid path", p) + } + }) + } +} + +func TestFilesystemListPrefixAllowsEmptyPrefix(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + _, _, _ = fs.Store(ctx, "a.txt", strings.NewReader("aaaa")) + _, _, _ = fs.Store(ctx, "c/d.txt", strings.NewReader("ccccc")) + + objects, err := fs.ListPrefix(ctx, "") + if err != nil { + t.Fatalf("ListPrefix empty prefix failed: %v", err) + } + if len(objects) != 2 { + t.Fatalf("ListPrefix empty prefix returned %d objects, want 2", len(objects)) } } From 98150ea576a32792e27fbf2adf2bbe596fff7573 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 9 Jul 2026 17:41:05 +0100 Subject: [PATCH 17/70] Name empty path validation subtest --- internal/storage/filesystem_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go index 10d073b..de0e418 100644 --- a/internal/storage/filesystem_test.go +++ b/internal/storage/filesystem_test.go @@ -261,7 +261,12 @@ func TestFilesystemRejectsInvalidPaths(t *testing.T) { "test/../file.txt", `test\..\file.txt`, } { - t.Run(p, func(t *testing.T) { + name := p + if name == "" { + name = "empty" + } + + t.Run(name, func(t *testing.T) { ctx := context.Background() if _, err := fs.FullPath(p); !errors.Is(err, ErrNotFound) { From 5b959d14b95c1b3efd801cab570cd5e1cbfdcccb Mon Sep 17 00:00:00 2001 From: ychampion Date: Fri, 10 Jul 2026 22:59:15 +0000 Subject: [PATCH 18/70] Preserve Go proxy fallback for missing modules Return the repository-standard not-found response when an upstream module is absent. Constraint: GOPROXY advances to direct only after 404 or 410 responses. Rejected: Preserve a handler-specific response body | sibling handlers consistently use not found. Confidence: high Scope-risk: narrow Directive: Keep missing-artifact responses consistent across registry handlers. Tested: go test ./internal/handler -run TestGoModuleDownloadUpstreamErrors -count=1; prior go test -race ./...; go build ./...; golangci-lint; go vet. --- internal/handler/go.go | 7 +++++++ internal/handler/go_test.go | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/internal/handler/go.go b/internal/handler/go.go index 955a89c..cf40aa1 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -1,9 +1,12 @@ package handler import ( + "errors" "fmt" "net/http" "strings" + + "github.com/git-pkgs/registries/fetch" ) const ( @@ -100,6 +103,10 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul result, err := h.proxy.GetOrFetchArtifact(r.Context(), "golang", decodedModule, version, filename) if err != nil { + if errors.Is(err, fetch.ErrNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } h.proxy.Logger.Error("failed to get artifact", "error", err) http.Error(w, "failed to fetch module", http.StatusBadGateway) return diff --git a/internal/handler/go_test.go b/internal/handler/go_test.go index da4ea63..ae998c9 100644 --- a/internal/handler/go_test.go +++ b/internal/handler/go_test.go @@ -1,9 +1,49 @@ package handler import ( + "errors" + "net/http" + "net/http/httptest" "testing" + + "github.com/git-pkgs/registries/fetch" ) +func TestGoModuleDownloadUpstreamErrors(t *testing.T) { + tests := []struct { + name string + fetchErr error + wantStatus int + }{ + { + name: "module not found", + fetchErr: fetch.ErrNotFound, + wantStatus: http.StatusNotFound, + }, + { + name: "upstream failure", + fetchErr: errors.New("connection refused"), + wantStatus: http.StatusBadGateway, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = tt.fetchErr + handler := NewGoHandler(proxy, "http://localhost:8080") + + req := httptest.NewRequest(http.MethodGet, "/example.com/mod/@v/v1.0.0.zip", nil) + resp := httptest.NewRecorder() + handler.Routes().ServeHTTP(resp, req) + + if resp.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d", resp.Code, tt.wantStatus) + } + }) + } +} + func TestDecodeGoModule(t *testing.T) { tests := []struct { encoded string From be15a0f826333d8eba32f9a34b7fcaed67d150c0 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 13 Jul 2026 17:11:04 -0700 Subject: [PATCH 19/70] Fix cooldown overrides for scoped package PURLs --- config.example.yaml | 3 ++- docs/configuration.md | 2 ++ internal/config/config.go | 31 +++++++++++++++++++++++++++++++ internal/config/config_test.go | 28 ++++++++++++++++++++++++++++ internal/server/server.go | 2 +- 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index d1ed9a1..1176f5b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -168,7 +168,8 @@ cooldown: # npm: "7d" # cargo: "0" - # Per-package overrides (keyed by PURL) + # Per-package overrides (keyed by PURL). Keys are normalized, so npm scopes + # may use either @scope or the canonical %40scope form. # packages: # "pkg:npm/lodash": "0" # "pkg:npm/@babel/core": "14d" diff --git a/docs/configuration.md b/docs/configuration.md index dcdec24..b103253 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -234,6 +234,8 @@ cooldown: Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable. +Package PURL keys are normalized automatically. For npm scopes, both the readable form (`pkg:npm/@babel/core`) and canonical form (`pkg:npm/%40babel/core`) are accepted. If both forms configure the same package, the canonical entry wins. + Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. diff --git a/internal/config/config.go b/internal/config/config.go index 2d275c7..ec510d2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -54,10 +54,12 @@ import ( "net/url" "os" "path/filepath" + "sort" "strconv" "strings" "time" + "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" ) @@ -137,9 +139,38 @@ type CooldownConfig struct { Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"` // Packages overrides the cooldown for specific packages (keyed by PURL). + // Valid PURL keys are normalized to canonical form before use. Packages map[string]string `json:"packages" yaml:"packages"` } +// NormalizedPackages returns a copy of the package overrides with valid PURL +// keys in canonical form. An explicitly canonical key wins over an equivalent +// noncanonical key, and invalid keys are preserved unchanged. +func (c *CooldownConfig) NormalizedPackages() map[string]string { + if c == nil || c.Packages == nil { + return nil + } + + keys := make([]string, 0, len(c.Packages)) + for key := range c.Packages { + keys = append(keys, key) + } + sort.Strings(keys) + + normalized := make(map[string]string, len(c.Packages)) + for _, key := range keys { + canonical := key + if parsed, err := purl.Parse(key); err == nil { + canonical = parsed.String() + } + if _, exists := normalized[canonical]; exists && key != canonical { + continue + } + normalized[canonical] = c.Packages[key] + } + return normalized +} + // StorageConfig configures artifact storage. type StorageConfig struct { // URL is the storage backend URL. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9c34023..decc18f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -363,6 +363,34 @@ cooldown: if cfg.Cooldown.Packages["pkg:npm/@babel/core"] != "14d" { t.Errorf("Cooldown.Packages[@babel/core] = %q, want %q", cfg.Cooldown.Packages["pkg:npm/@babel/core"], "14d") } + if got := cfg.Cooldown.NormalizedPackages()["pkg:npm/%40babel/core"]; got != "14d" { + t.Errorf("normalized Cooldown.Packages[@babel/core] = %q, want %q", got, "14d") + } +} + +func TestCooldownConfigNormalizedPackages(t *testing.T) { + rawScoped := "pkg:npm/@typescript/typescript-darwin-arm64" + canonicalScoped := "pkg:npm/%40typescript/typescript-darwin-arm64" + cfg := CooldownConfig{Packages: map[string]string{ + rawScoped: "2d", + canonicalScoped: "3d", + "not-a-purl": "4d", + }} + + got := cfg.NormalizedPackages() + + if got[canonicalScoped] != "3d" { + t.Errorf("canonical scoped package duration = %q, want %q", got[canonicalScoped], "3d") + } + if _, exists := got[rawScoped]; exists { + t.Errorf("raw scoped package key %q was not canonicalized", rawScoped) + } + if got["not-a-purl"] != "4d" { + t.Errorf("invalid PURL duration = %q, want preserved value %q", got["not-a-purl"], "4d") + } + if cfg.Packages[rawScoped] != "2d" { + t.Error("NormalizedPackages mutated the source map") + } } func TestLoadCooldownFromEnv(t *testing.T) { diff --git a/internal/server/server.go b/internal/server/server.go index 5aac4db..74c82c7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -163,7 +163,7 @@ func (s *Server) Start() error { cd := &cooldown.Config{ Default: s.cfg.Cooldown.Default, Ecosystems: s.cfg.Cooldown.Ecosystems, - Packages: s.cfg.Cooldown.Packages, + Packages: s.cfg.Cooldown.NormalizedPackages(), } proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy.HTTPClient.Timeout = s.cfg.ParseHTTPTimeout() From cdf075695f00be2fc187068875b759e872218e63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:14:00 +0000 Subject: [PATCH 20/70] Bump golang from 1.26.4-alpine to 1.26.5-alpine Bumps golang from 1.26.4-alpine to 1.26.5-alpine. --- updated-dependencies: - dependency-name: golang dependency-version: 1.26.5-alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7b2795c..c2e197f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.26.4-alpine AS builder +FROM golang:1.26.5-alpine AS builder WORKDIR /src From ec3cc3579599246742c022bbec9157330eb30129 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:14:06 +0000 Subject: [PATCH 21/70] Bump github.com/go-chi/chi/v5 from 5.3.0 to 5.3.1 Bumps [github.com/go-chi/chi/v5](https://github.com/go-chi/chi) from 5.3.0 to 5.3.1. - [Release notes](https://github.com/go-chi/chi/releases) - [Changelog](https://github.com/go-chi/chi/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-chi/chi/compare/v5.3.0...v5.3.1) --- updated-dependencies: - dependency-name: github.com/go-chi/chi/v5 dependency-version: 5.3.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b063dba..bae89fb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.2.6 github.com/git-pkgs/vulns v0.1.6 - github.com/go-chi/chi/v5 v5.3.0 + github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 github.com/prometheus/client_golang v1.23.2 diff --git a/go.sum b/go.sum index b1d35f6..e9d1e26 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/git-pkgs/vulns v0.1.6 h1:8RRSgdlxp4JMU0Zykr63XTOMo5CyZKwt/PwaQxrx9Yg= github.com/git-pkgs/vulns v0.1.6/go.mod h1:TsZC4MjoCkKJslgmbcmRCnytwnFcjESC2N8b0a2xDWc= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= -github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= -github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= +github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= +github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog= github.com/go-critic/go-critic v0.14.3/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= From 96e113213de527dd9834e233dd77b71558030d38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:14:12 +0000 Subject: [PATCH 22/70] Bump github.com/git-pkgs/purl from 0.1.13 to 0.1.14 Bumps [github.com/git-pkgs/purl](https://github.com/git-pkgs/purl) from 0.1.13 to 0.1.14. - [Commits](https://github.com/git-pkgs/purl/compare/v0.1.13...v0.1.14) --- updated-dependencies: - dependency-name: github.com/git-pkgs/purl dependency-version: 0.1.14 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b063dba..35079fb 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/git-pkgs/archives v0.3.0 github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/enrichment v0.4.1 - github.com/git-pkgs/purl v0.1.13 + github.com/git-pkgs/purl v0.1.14 github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.2.6 diff --git a/go.sum b/go.sum index b1d35f6..f38a797 100644 --- a/go.sum +++ b/go.sum @@ -254,8 +254,8 @@ github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6 github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= -github.com/git-pkgs/purl v0.1.13 h1:at8BU6vnP5oonHFHAPA064BzgRqij+SZcOUDgNT2DC8= -github.com/git-pkgs/purl v0.1.13/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= +github.com/git-pkgs/purl v0.1.14 h1:GgqwiBNS0eKJqJ/gabUBEC10Xhpj6UX12kfNzujaeYc= +github.com/git-pkgs/purl v0.1.14/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+u0a8= github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= From df997e5825ee0e30ec38452be4da42260ec65e36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:14:14 +0000 Subject: [PATCH 23/70] Bump docker/login-action from 4.3.0 to 4.4.0 Bumps [docker/login-action](https://github.com/docker/login-action) from 4.3.0 to 4.4.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/c99871dec2022cc055c062a10cc1a1310835ceb4...af1e73f918a031802d376d3c8bbc3fe56130a9b0) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5f459b6..9c99e5d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: Log in to the Container registry - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 with: registry: ghcr.io username: ${{ github.actor }} From 41ef27907e99ee9c214ece7731ea96f25708873d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:20:18 +0000 Subject: [PATCH 24/70] Bump github.com/git-pkgs/enrichment from 0.4.1 to 0.6.0 Bumps [github.com/git-pkgs/enrichment](https://github.com/git-pkgs/enrichment) from 0.4.1 to 0.6.0. - [Commits](https://github.com/git-pkgs/enrichment/compare/v0.4.1...v0.6.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/enrichment dependency-version: 0.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 0db3f6b..b84d213 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/CycloneDX/cyclonedx-go v0.11.0 github.com/git-pkgs/archives v0.3.0 github.com/git-pkgs/cooldown v0.1.1 - github.com/git-pkgs/enrichment v0.4.1 + github.com/git-pkgs/enrichment v0.6.0 github.com/git-pkgs/purl v0.1.14 github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 @@ -115,7 +115,7 @@ require ( github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ecosyste-ms/ecosystems-go v0.2.0 // indirect + github.com/ecosyste-ms/ecosystems-go v0.3.0 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/fatih/color v1.18.0 // indirect @@ -217,7 +217,7 @@ require ( github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect - github.com/oapi-codegen/runtime v1.4.1 // indirect + github.com/oapi-codegen/runtime v1.4.2 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect diff --git a/go.sum b/go.sum index 9a93f6d..6da21d4 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ecosyste-ms/ecosystems-go v0.2.0 h1:Nhpg54C+St8Sd/mf8bNJmQqx35ZgHw33TfFoxaXMQI8= -github.com/ecosyste-ms/ecosystems-go v0.2.0/go.mod h1:CCdzT1iAZirbEZAbFSnWpK88eKKaIWex7gjtZ0UudXA= +github.com/ecosyste-ms/ecosystems-go v0.3.0 h1:eVTNKQ3PyVNkSAnk1934958Vg4rR7ho5B5MWEQaMLi4= +github.com/ecosyste-ms/ecosystems-go v0.3.0/go.mod h1:bkjiI8WoTFB1Jw0M3h8yn3KXCD/+LdETsQ3m2BNLMHQ= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= @@ -248,8 +248,8 @@ github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= -github.com/git-pkgs/enrichment v0.4.1 h1:A8BKs0XwvpF1sF5qviZy4fkJAe18qB9OgpbRnmwnT34= -github.com/git-pkgs/enrichment v0.4.1/go.mod h1:stHqZUitV9ZkwACqHzBysLMSe6T4QZn81hxTdSroNhM= +github.com/git-pkgs/enrichment v0.6.0 h1:npV6N+eFZnI64uw/B0s+WJPn6Fu6Be08bexAViZFjMg= +github.com/git-pkgs/enrichment v0.6.0/go.mod h1:ov2WDaiNoIXViqdnE1FlUjNTnB5RnkUH4et9BWPhUDY= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= @@ -512,8 +512,8 @@ github.com/nunnatsa/ginkgolinter v0.23.0 h1:x3o4DGYOWbBMP/VdNQKgSj+25aJKx2Pe6lHr github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= -github.com/oapi-codegen/runtime v1.4.1 h1:9nwLoI+KrWxzbBcp0jO/R8uXqbik/HUyCvPeU68Y/qo= -github.com/oapi-codegen/runtime v1.4.1/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/oapi-codegen/runtime v1.4.2 h1:GMxFVYLzoYLua+/KvzgSphkyK1lLTReQI9Vf4hvATKE= +github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= From cdbd1a93697b1b97a50d68c7d67c89063be86ef0 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 17 Jul 2026 18:11:47 -0700 Subject: [PATCH 25/70] Canonicalize cooldown lookup PURLs to match config NormalizedPackages runs config keys through purl.Parse().String(), which applies per-type rules like lowercasing pypi names. The handler side was building lookup keys with MakePURLString, which does not, so a config entry for pkg:pypi/Django would be normalized to pkg:pypi/django and never match the runtime key pkg:pypi/Django. Route both sides through the same canonical form: a new canonicalPackagePURL helper calls Normalize() on the constructed PURL before stringifying, and all cooldown IsAllowed call sites use it. --- docs/configuration.md | 2 +- internal/handler/cargo.go | 4 +--- internal/handler/composer.go | 4 +--- internal/handler/conda.go | 4 +--- internal/handler/gem.go | 4 +--- internal/handler/handler.go | 8 ++++++++ internal/handler/handler_test.go | 31 +++++++++++++++++++++++++++++++ internal/handler/hex.go | 3 +-- internal/handler/npm.go | 4 +--- internal/handler/nuget.go | 4 +--- internal/handler/pub.go | 4 +--- internal/handler/pypi.go | 6 ++---- 12 files changed, 50 insertions(+), 28 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b103253..8998ac2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -234,7 +234,7 @@ cooldown: Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable. -Package PURL keys are normalized automatically. For npm scopes, both the readable form (`pkg:npm/@babel/core`) and canonical form (`pkg:npm/%40babel/core`) are accepted. If both forms configure the same package, the canonical entry wins. +Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 5d7810c..bf424e2 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -8,8 +8,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -143,7 +141,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr continue } - cratePURL := purl.MakePURLString("cargo", crate.Name, "") + cratePURL := canonicalPackagePURL("cargo", crate.Name) if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering cargo version", diff --git a/internal/handler/composer.go b/internal/handler/composer.go index bc3bc1d..23deba5 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -10,8 +10,6 @@ import ( "path" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -216,7 +214,7 @@ func deepCopyValue(v any) any { // filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs // for a single package's version list. func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any { - packagePURL := purl.MakePURLString("composer", packageName, "") + packagePURL := canonicalPackagePURL("composer", packageName) filtered := versionList[:0] for _, v := range versionList { diff --git a/internal/handler/conda.go b/internal/handler/conda.go index cfa20c8..a8632e8 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -6,8 +6,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -218,7 +216,7 @@ func (h *CondaHandler) applyCooldownFiltering(body []byte) ([]byte, error) { continue } - packagePURL := purl.MakePURLString("conda", name, "") + packagePURL := canonicalPackagePURL("conda", name) if !h.proxy.Cooldown.IsAllowed("conda", packagePURL, publishedAt) { version, _ := entryMap["version"].(string) diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 9ec57e3..a3714d6 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -8,8 +8,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -266,7 +264,7 @@ func (h *GemHandler) fetchFilteredVersions(r *http.Request, name string) (map[st return nil, err } - packagePURL := purl.MakePURLString("gem", name, "") + packagePURL := canonicalPackagePURL("gem", name) filtered := make(map[string]bool) for _, v := range versions { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d06ca83..202426d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -48,6 +48,14 @@ func hasDotDotSegment(path string) bool { const defaultHTTPTimeout = 30 * time.Second +// canonicalPackagePURL returns a versionless PURL in canonical form so cooldown +// lookups match keys produced by config.CooldownConfig.NormalizedPackages. +func canonicalPackagePURL(ecosystem, name string) string { + p := purl.MakePURL(ecosystem, name, "") + _ = p.Normalize() + return p.String() +} + const contentTypeJSON = "application/json" const headerAcceptEncoding = "Accept-Encoding" diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index bbcab72..9a2b329 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/registries/fetch" @@ -1010,3 +1011,33 @@ func TestProxyCached_FreshResponse_NoWarningHeader(t *testing.T) { t.Errorf("Warning should be empty for fresh response, got %q", got) } } + +// TestCanonicalPackagePURLMatchesConfig ensures the runtime cooldown lookup key +// agrees with config.CooldownConfig.NormalizedPackages for the same package, +// so a configured override is actually found regardless of how the user wrote it. +func TestCanonicalPackagePURLMatchesConfig(t *testing.T) { + tests := []struct { + ecosystem string + requestName string + configKey string + }{ + {"npm", "@babel/core", "pkg:npm/@babel/core"}, + {"npm", "@babel/core", "pkg:npm/%40babel/core"}, + {"npm", "@typescript/typescript-darwin-arm64", "pkg:npm/@typescript/typescript-darwin-arm64"}, + {"pypi", "Django", "pkg:pypi/Django"}, + {"pypi", "django", "pkg:pypi/Django"}, + {"composer", "symfony/console", "pkg:composer/Symfony/Console"}, + {"cargo", "serde", "pkg:cargo/serde"}, + } + for _, tt := range tests { + t.Run(tt.ecosystem+"/"+tt.requestName+"<="+tt.configKey, func(t *testing.T) { + cfg := config.CooldownConfig{Packages: map[string]string{tt.configKey: "1d"}} + normalized := cfg.NormalizedPackages() + + lookup := canonicalPackagePURL(tt.ecosystem, tt.requestName) + if _, ok := normalized[lookup]; !ok { + t.Errorf("lookup key %q not found in normalized config %v", lookup, normalized) + } + }) + } +} diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 0f0c72e..6ebc176 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -10,7 +10,6 @@ import ( "strings" "time" - "github.com/git-pkgs/purl" "google.golang.org/protobuf/encoding/protowire" ) @@ -237,7 +236,7 @@ func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[st return nil, err } - packagePURL := purl.MakePURLString("hex", name, "") + packagePURL := canonicalPackagePURL("hex", name) filtered := make(map[string]bool) for _, release := range pkg.Releases { diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 0585eda..06f539e 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -9,8 +9,6 @@ import ( "sort" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -134,7 +132,7 @@ func (h *NPMHandler) applyCooldownFiltering(metadata map[string]any, versions ma return } - packagePURL := purl.MakePURLString("npm", packageName, "") + packagePURL := canonicalPackagePURL("npm", packageName) for version := range versions { publishedStr, ok := timeMap[version].(string) diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 40b8b5f..3e6373a 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -8,8 +8,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -271,7 +269,7 @@ func (h *NuGetHandler) applyCooldownFiltering(body []byte) ([]byte, error) { } } - packagePURL := purl.MakePURLString("nuget", strings.ToLower(id), "") + packagePURL := canonicalPackagePURL("nuget", strings.ToLower(id)) if !h.proxy.Cooldown.IsAllowed("nuget", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering nuget version", diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 60bbbad..2971ddf 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -7,8 +7,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -127,7 +125,7 @@ func (h *PubHandler) rewriteMetadata(name string, body []byte) ([]byte, error) { return body, nil } - packagePURL := purl.MakePURLString("pub", name, "") + packagePURL := canonicalPackagePURL("pub", name) filtered := h.filterAndRewriteVersions(name, packagePURL, versions) metadata["versions"] = filtered diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 3021d2b..0cf39fb 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -12,8 +12,6 @@ import ( "regexp" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -131,7 +129,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[st return nil } - packagePURL := purl.MakePURLString("pypi", name, "") + packagePURL := canonicalPackagePURL("pypi", name) filtered := make(map[string]bool) for version, files := range releases { @@ -262,7 +260,7 @@ func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, error) { packageName, _ := extractPyPIName(metadata) packagePURL := "" if packageName != "" { - packagePURL = purl.MakePURLString("pypi", packageName, "") + packagePURL = canonicalPackagePURL("pypi", packageName) } h.filterAndRewriteReleases(metadata, packageName, packagePURL) From 60c72be65e6f0d45593dbecdeba2f1e3ded533fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:13:49 +0000 Subject: [PATCH 26/70] Bump actions/setup-go from 6.5.0 to 7.0.0 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 6.5.0 to 7.0.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/924ae3a1cded613372ab5595356fb5720e22ba16...b7ad1dad31e06c5925ef5d2fc7ad053ef454303e) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release.yml | 2 +- .github/workflows/swagger.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 700f28e..c405f5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: ${{ matrix.go-version }} @@ -40,7 +40,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.25' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d4d0b5..f06cecd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod cache: false diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index f947c7e..e99aecf 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -17,7 +17,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.25' From d9b7a30294582b3a3c8939457805a4a2c8d3c688 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:13:53 +0000 Subject: [PATCH 27/70] Bump zizmorcore/zizmor-action from 0.5.7 to 0.6.0 Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.5.7 to 0.6.0. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/192e21d79ab29983730a13d1382995c2307fbcaa...6599ee8b7a49aef6a770f63d261d214911a7ce02) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index cce6e4f..df06c5f 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0 From c3f1577017e76dadf040d7ab225869052513bc29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:14:01 +0000 Subject: [PATCH 28/70] Bump modernc.org/sqlite from 1.53.0 to 1.54.0 Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.53.0 to 1.54.0. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.53.0...v1.54.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.54.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index b84d213..384fba3 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.53.0 + modernc.org/sqlite v1.54.0 ) require ( @@ -294,12 +294,12 @@ require ( golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect - golang.org/x/mod v0.36.0 // indirect + golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect @@ -307,7 +307,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect - modernc.org/libc v1.73.4 // indirect + modernc.org/libc v1.74.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect diff --git a/go.sum b/go.sum index 6da21d4..301a00d 100644 --- a/go.sum +++ b/go.sum @@ -755,8 +755,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -839,8 +839,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= @@ -882,20 +882,20 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= -modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= -modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= -modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= -modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= -modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= +modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -904,8 +904,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= -modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= +modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 5ae5eab0311d4f3cfab5ec5465523bcd402fbd1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:14:10 +0000 Subject: [PATCH 29/70] Bump github.com/git-pkgs/vulns from 0.1.6 to 0.2.0 Bumps [github.com/git-pkgs/vulns](https://github.com/git-pkgs/vulns) from 0.1.6 to 0.2.0. - [Commits](https://github.com/git-pkgs/vulns/compare/v0.1.6...v0.2.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/vulns dependency-version: 0.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b84d213..7b7701d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.2.6 - github.com/git-pkgs/vulns v0.1.6 + github.com/git-pkgs/vulns v0.2.0 github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 diff --git a/go.sum b/go.sum index 6da21d4..4427b92 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8= github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= -github.com/git-pkgs/vulns v0.1.6 h1:8RRSgdlxp4JMU0Zykr63XTOMo5CyZKwt/PwaQxrx9Yg= -github.com/git-pkgs/vulns v0.1.6/go.mod h1:TsZC4MjoCkKJslgmbcmRCnytwnFcjESC2N8b0a2xDWc= +github.com/git-pkgs/vulns v0.2.0 h1:S1sA0ObQ/IKW0vqK+lK+HAY37AdDHEfhaIja/fLptwg= +github.com/git-pkgs/vulns v0.2.0/go.mod h1:mjVquLlunsAFPltqjf6mbvbI4tQ1iIa1NuhKIOPKt/4= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= From 13ae3970e9b09b14cb9fbf90a136ae081db4222e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:42:03 +0000 Subject: [PATCH 30/70] Bump github.com/git-pkgs/vers from 0.2.6 to 0.3.0 Bumps [github.com/git-pkgs/vers](https://github.com/git-pkgs/vers) from 0.2.6 to 0.3.0. - [Commits](https://github.com/git-pkgs/vers/compare/v0.2.6...v0.3.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/vers dependency-version: 0.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7b7701d..563b518 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/git-pkgs/purl v0.1.14 github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 - github.com/git-pkgs/vers v0.2.6 + github.com/git-pkgs/vers v0.3.0 github.com/git-pkgs/vulns v0.2.0 github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 diff --git a/go.sum b/go.sum index 4427b92..ec10b3c 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,8 @@ github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+ github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= -github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8= -github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= +github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= +github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= github.com/git-pkgs/vulns v0.2.0 h1:S1sA0ObQ/IKW0vqK+lK+HAY37AdDHEfhaIja/fLptwg= github.com/git-pkgs/vulns v0.2.0/go.mod h1:mjVquLlunsAFPltqjf6mbvbI4tQ1iIa1NuhKIOPKt/4= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= From 532e4925fef3c7cbbd8d6d276e8444971d23c98b Mon Sep 17 00:00:00 2001 From: Tilian Honig Date: Sun, 26 Jul 2026 20:07:46 +0200 Subject: [PATCH 31/70] fix: proper handling of upstream registry 404s (#209) * fix: proper handling of upstream registry 404s * fix: consistently return 404s for all artifact types --- go.mod | 2 +- go.sum | 4 +- internal/handler/cargo.go | 3 +- internal/handler/composer.go | 3 +- internal/handler/conan.go | 6 +- internal/handler/conda.go | 3 +- internal/handler/container.go | 7 +- internal/handler/cran.go | 6 +- internal/handler/debian.go | 3 +- internal/handler/filename_download.go | 39 +++++ internal/handler/gem.go | 31 +--- internal/handler/handler.go | 22 ++- internal/handler/hex.go | 31 +--- internal/handler/julia.go | 9 +- internal/handler/maven.go | 7 +- internal/handler/notfound_ecosystems_test.go | 151 +++++++++++++++++++ internal/handler/notfound_test.go | 83 ++++++++++ internal/handler/npm.go | 4 + internal/handler/nuget.go | 3 +- internal/handler/pub.go | 3 +- internal/handler/pypi.go | 3 +- internal/handler/rpm.go | 3 +- 22 files changed, 337 insertions(+), 89 deletions(-) create mode 100644 internal/handler/filename_download.go create mode 100644 internal/handler/notfound_ecosystems_test.go create mode 100644 internal/handler/notfound_test.go diff --git a/go.mod b/go.mod index 1f2c073..6d81076 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/enrichment v0.6.0 github.com/git-pkgs/purl v0.1.14 - github.com/git-pkgs/registries v0.6.2 + github.com/git-pkgs/registries v0.6.3 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.3.0 github.com/git-pkgs/vulns v0.2.0 diff --git a/go.sum b/go.sum index 17b0651..5d16d31 100644 --- a/go.sum +++ b/go.sum @@ -256,8 +256,8 @@ github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= github.com/git-pkgs/purl v0.1.14 h1:GgqwiBNS0eKJqJ/gabUBEC10Xhpj6UX12kfNzujaeYc= github.com/git-pkgs/purl v0.1.14/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= -github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+u0a8= -github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= +github.com/git-pkgs/registries v0.6.3 h1:7sveeeMS2lgXtcqNYAA3bwaT7H9CQs0uhy6wQSxz3Js= +github.com/git-pkgs/registries v0.6.3/go.mod h1:j4o50ii/vD9Z42/nFeQASt15BPolADIejeFmmqK4njo= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index bf424e2..abc0d1f 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -191,8 +191,7 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifact(r.Context(), "cargo", name, version, filename) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch crate", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch crate") return } diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 23deba5..45935b7 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -346,8 +346,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/conan.go b/internal/handler/conan.go index 53f6428..c0476f9 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -84,8 +84,7 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch file", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch file") return } @@ -122,8 +121,7 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch file", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch file") return } diff --git a/internal/handler/conda.go b/internal/handler/conda.go index a8632e8..224c25f 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -72,8 +72,7 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/container.go b/internal/handler/container.go index 8ba5e97..0710ed1 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -2,6 +2,7 @@ package handler import ( "encoding/json" + "errors" "fmt" "io" "net/http" @@ -117,8 +118,12 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req ) if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + h.containerError(w, http.StatusNotFound, "BLOB_UNKNOWN", "blob unknown to registry") + return + } h.proxy.Logger.Error("failed to fetch blob", "error", err) - h.containerError(w, http.StatusBadGateway, "BLOB_UNKNOWN", "failed to fetch blob") + h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch blob") return } diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 0ecd2a3..2fc4fab 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -72,8 +72,7 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, version, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } @@ -107,8 +106,7 @@ func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Reques result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, storageVersion, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/debian.go b/internal/handler/debian.go index b767f6d..9a4aaab 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -81,8 +81,7 @@ func (h *DebianHandler) handlePackageDownload(w http.ResponseWriter, r *http.Req result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "deb", name, version, filename, downloadURL) if err != nil { - h.proxy.Logger.Error("failed to get debian package", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go new file mode 100644 index 0000000..bedec16 --- /dev/null +++ b/internal/handler/filename_download.go @@ -0,0 +1,39 @@ +package handler + +import ( + "net/http" + "strings" +) + +type filenameDownload struct { + ecosystem string + suffix string + parseErr string + fetchErr string + parse func(string) (name, version string) +} + +func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { + filename := r.PathValue("filename") + if filename == "" || !strings.HasSuffix(filename, d.suffix) { + http.Error(w, "invalid filename", http.StatusBadRequest) + return + } + + name, version := d.parse(filename) + if name == "" || version == "" { + http.Error(w, d.parseErr, http.StatusBadRequest) + return + } + + p.Logger.Info(d.ecosystem+" download request", + "name", name, "version", version, "filename", filename) + + result, err := p.GetOrFetchArtifact(r.Context(), d.ecosystem, name, version, filename) + if err != nil { + p.serveArtifactError(w, err, d.fetchErr) + return + } + + ServeArtifact(w, result) +} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index a3714d6..260568a 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -58,30 +58,13 @@ func (h *GemHandler) Routes() http.Handler { // handleDownload serves a gem file, fetching and caching from upstream if needed. func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { - filename := r.PathValue("filename") - if filename == "" || !strings.HasSuffix(filename, ".gem") { - http.Error(w, "invalid filename", http.StatusBadRequest) - return - } - - // Extract name and version from filename (e.g., "rails-7.1.0.gem") - name, version := h.parseGemFilename(filename) - if name == "" || version == "" { - http.Error(w, "could not parse gem filename", http.StatusBadRequest) - return - } - - h.proxy.Logger.Info("gem download request", - "name", name, "version", version, "filename", filename) - - result, err := h.proxy.GetOrFetchArtifact(r.Context(), "gem", name, version, filename) - if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch gem", http.StatusBadGateway) - return - } - - ServeArtifact(w, result) + h.proxy.handleFilenameDownload(w, r, filenameDownload{ + ecosystem: "gem", + suffix: ".gem", + parseErr: "could not parse gem filename", + fetchErr: "failed to fetch gem", + parse: h.parseGemFilename, + }) } // parseGemFilename extracts name and version from a gem filename. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 202426d..e62292f 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -251,6 +251,9 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil // Resolve download URL info, err := p.Resolver.Resolve(ctx, ecosystem, name, version) if err != nil { + if errors.Is(err, fetch.ErrNotFound) { + return nil, ErrUpstreamNotFound + } return nil, fmt.Errorf("resolving download URL: %w", err) } @@ -270,6 +273,9 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil if err != nil { metrics.RecordUpstreamFetch(ecosystem, fetchDuration) metrics.RecordUpstreamError(ecosystem, "fetch_failed") + if errors.Is(err, fetch.ErrNotFound) { + return nil, ErrUpstreamNotFound + } return nil, fmt.Errorf("fetching from upstream: %w", err) } metrics.RecordUpstreamFetch(ecosystem, fetchDuration) @@ -451,7 +457,18 @@ func JSONError(w http.ResponseWriter, status int, message string) { } // ErrUpstreamNotFound indicates the upstream returned 404. -var ErrUpstreamNotFound = fmt.Errorf("upstream: not found") +var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) + +// serveArtifactError writes response for a failed fetch: +// 404 when upstream reports artifact missing, 502 otherwise. +func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg string) { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + p.Logger.Error("failed to get artifact", "error", err) + http.Error(w, clientMsg, http.StatusBadGateway) +} // errStale304 is returned when upstream sends 304 but the cached file is missing. var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing") @@ -795,6 +812,9 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) if err != nil { + if errors.Is(err, fetch.ErrNotFound) { + return nil, ErrUpstreamNotFound + } return nil, fmt.Errorf("fetching from upstream: %w", err) } diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 6ebc176..2ff4f0f 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -53,30 +53,13 @@ func (h *HexHandler) Routes() http.Handler { // handleDownload serves a package tarball, fetching and caching from upstream if needed. func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { - filename := r.PathValue("filename") - if filename == "" || !strings.HasSuffix(filename, ".tar") { - http.Error(w, "invalid filename", http.StatusBadRequest) - return - } - - // Extract name and version from filename (e.g., "phoenix-1.7.10.tar") - name, version := h.parseTarballFilename(filename) - if name == "" || version == "" { - http.Error(w, "could not parse tarball filename", http.StatusBadRequest) - return - } - - h.proxy.Logger.Info("hex download request", - "name", name, "version", version, "filename", filename) - - result, err := h.proxy.GetOrFetchArtifact(r.Context(), "hex", name, version, filename) - if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) - return - } - - ServeArtifact(w, result) + h.proxy.handleFilenameDownload(w, r, filenameDownload{ + ecosystem: "hex", + suffix: ".tar", + parseErr: "could not parse tarball filename", + fetchErr: "failed to fetch package", + parse: h.parseTarballFilename, + }) } // parseTarballFilename extracts name and version from a hex tarball filename. diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 08b1fdf..0fed8c9 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -90,8 +90,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get registry", "error", err) - http.Error(w, "failed to fetch registry", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch registry") return } @@ -119,8 +118,7 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get package", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } @@ -141,8 +139,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch artifact", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } diff --git a/internal/handler/maven.go b/internal/handler/maven.go index c423645..10e551e 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -130,12 +130,7 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, ur } } if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch artifact", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go new file mode 100644 index 0000000..3c2cecf --- /dev/null +++ b/internal/handler/notfound_ecosystems_test.go @@ -0,0 +1,151 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { + tests := []struct { + name string + path string + handler func(p *Proxy) http.Handler + }{ + {"debian", "/pool/main/n/nginx/nginx_1.18.0-6_amd64.deb", + func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost").Routes() }}, + {"rpm", "/releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm", + func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }}, + {"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", + func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }}, + {"pypi", "/packages/packages/ab/cd/ef0123456789/requests-2.31.0-py3-none-any.whl", + func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://localhost").Routes() }}, + {"cran", "/src/contrib/ggplot2_3.4.4.tar.gz", + func(p *Proxy) http.Handler { return NewCRANHandler(p, "http://localhost").Routes() }}, + {"conda", "/conda-forge/linux-64/numpy-1.26.0-py311_0.tar.bz2", + func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }}, + {"conan", "/v1/files/zlib/1.3.1/_/_/0/recipe/conan_sources.tgz", + func(p *Proxy) http.Handler { return NewConanHandler(p, "http://localhost").Routes() }}, + {"gem", "/gems/rails-7.1.0.gem", + func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }}, + {"hex", "/tarballs/phoenix-1.7.10.tar", + func(p *Proxy) http.Handler { return NewHexHandler(p, "http://localhost").Routes() }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + srv := httptest.NewServer(tt.handler(proxy)) + defer srv.Close() + + resp, err := http.Get(srv.URL + tt.path) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode) + } + }) + } +} + +func TestJuliaPackageUpstreamNotFoundReturns404(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + dead := httptest.NewServer(http.NotFoundHandler()) + defer dead.Close() + + h := NewJuliaHandler(proxy, "http://localhost") + h.upstreamURL = dead.URL + + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + + "/package/7876af07-990d-54b4-ab0e-23690620f79a/0123456789abcdef0123456789abcdef01234567") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("want 404 for missing upstream package, got %d", resp.StatusCode) + } +} + +func TestComposerDownloadUpstreamNotFoundReturns404(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + meta := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/p2/monolog/monolog.json" { + _, _ = w.Write([]byte(`{ + "packages": { + "monolog/monolog": [ + {"version": "2.9.1", "dist": {"url": "https://example.com/monolog-2.9.1.zip", "type": "zip"}} + ] + } + }`)) + return + } + http.NotFound(w, r) + })) + defer meta.Close() + + h := &ComposerHandler{proxy: proxy, repoURL: meta.URL, proxyURL: "http://localhost"} + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/files/monolog/monolog/2.9.1/monolog-2.9.1.zip") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("want 404 for missing upstream dist, got %d", resp.StatusCode) + } +} + +func TestContainerBlobUpstreamNotFoundReturns404(t *testing.T) { + authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"token": "test-token-123"}`)) + })) + defer authServer.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = &mockFetcherWithHeaders{ + fetchFn: func(_ context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { + return nil, fetch.ErrNotFound + }, + } + + h := &ContainerHandler{ + proxy: proxy, + registryURL: "https://registry-1.docker.io", + authURL: authServer.URL, + proxyURL: "http://localhost:8080", + } + + req := httptest.NewRequest(http.MethodGet, + "/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Errorf("want 404 for missing upstream blob, got %d; body: %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "BLOB_UNKNOWN") { + t.Errorf("want BLOB_UNKNOWN error code in body, got: %s", w.Body.String()) + } +} diff --git a/internal/handler/notfound_test.go b/internal/handler/notfound_test.go new file mode 100644 index 0000000..9ea38ac --- /dev/null +++ b/internal/handler/notfound_test.go @@ -0,0 +1,83 @@ +package handler + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +func TestErrUpstreamNotFoundWrapsFetchErrNotFound(t *testing.T) { + if !errors.Is(ErrUpstreamNotFound, fetch.ErrNotFound) { + t.Fatal("ErrUpstreamNotFound does not wrap fetch.ErrNotFound") + } +} + +func TestGetOrFetchArtifactFromURL_NotFound(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "maven", "org.example:missing", "1.0", "missing-1.0.jar", + "http://upstream.test/org/example/missing/1.0/missing-1.0.jar") + + if !errors.Is(err, ErrUpstreamNotFound) { + t.Fatalf("want ErrUpstreamNotFound, got %v", err) + } +} + +func TestMavenHandler_UpstreamNotFoundReturns404(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/org/example/missing/1.0/missing-1.0.jar") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode) + } +} + +func TestMavenHandler_PluginPortalFallback(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErrByURL = map[string]error{ + "http://upstream.test/org/example/plugin/1.0/plugin-1.0.jar": fetch.ErrNotFound, + } + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("portal artifact")), + ContentType: "application/java-archive", + } + + h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/org/example/plugin/1.0/plugin-1.0.jar") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200 via plugin portal fallback, got %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if string(body) != "portal artifact" { + t.Errorf("want portal artifact body, got %q", body) + } + if fetcher.fetchedURL != "http://portal.test/org/example/plugin/1.0/plugin-1.0.jar" { + t.Errorf("fallback did not hit plugin portal, last URL: %s", fetcher.fetchedURL) + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 06f539e..a3ed910 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -263,6 +263,10 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifact(r.Context(), "npm", packageName, version, filename) if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + JSONError(w, http.StatusNotFound, "package not found") + return + } h.proxy.Logger.Error("failed to get artifact", "error", err) JSONError(w, http.StatusBadGateway, "failed to fetch package") return diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 3e6373a..4785e40 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -314,8 +314,7 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "nuget", name, version, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 2971ddf..e5ca199 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -67,8 +67,7 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifact(r.Context(), "pub", name, version, filename) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 0cf39fb..f5a0232 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -426,8 +426,7 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "pypi", name, version, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 6440d0f..a5752ec 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -83,8 +83,7 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "rpm", name, version, filename, downloadURL) if err != nil { - h.proxy.Logger.Error("failed to get rpm package", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } From cf3741162f7e3000cc9bc78152896e608e5a6549 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 26 Jul 2026 19:11:24 +0100 Subject: [PATCH 32/70] fix(upstream): honor npm and Cargo overrides (#200) * fix(upstream): honor npm and cargo overrides * Preserve npm scope separators in download URLs * Apply upstream auth to metadata requests --- internal/handler/cargo.go | 24 ++++++++-- internal/handler/cargo_test.go | 71 +++++++++++++++++++++++++++++ internal/handler/handler.go | 16 +++++++ internal/handler/npm.go | 26 +++++++++-- internal/handler/npm_test.go | 83 ++++++++++++++++++++++++++++++++++ internal/server/server.go | 10 +++- internal/server/server_test.go | 9 +++- 7 files changed, 228 insertions(+), 11 deletions(-) diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index abc0d1f..79fb750 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "strings" "time" ) @@ -28,11 +29,18 @@ type CargoHandler struct { } // NewCargoHandler creates a new cargo protocol handler. -func NewCargoHandler(proxy *Proxy, proxyURL string) *CargoHandler { +func NewCargoHandler(proxy *Proxy, proxyURL, indexURL, downloadURL string) *CargoHandler { + if strings.TrimSpace(indexURL) == "" { + indexURL = cargoUpstream + } + if strings.TrimSpace(downloadURL) == "" { + downloadURL = cargoDownloadBase + } + return &CargoHandler{ proxy: proxy, - indexURL: cargoUpstream, - downloadURL: cargoDownloadBase, + indexURL: strings.TrimSuffix(indexURL, "/"), + downloadURL: strings.TrimSuffix(downloadURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"), } } @@ -189,7 +197,15 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("cargo download request", "crate", name, "version", version, "filename", filename) - result, err := h.proxy.GetOrFetchArtifact(r.Context(), "cargo", name, version, filename) + downloadURL := fmt.Sprintf( + "%s/%s/%s", + h.downloadURL, + url.PathEscape(name), + url.PathEscape(filename), + ) + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), "cargo", name, version, filename, downloadURL, + ) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch crate") return diff --git a/internal/handler/cargo_test.go b/internal/handler/cargo_test.go index 10d3faf..895ff7b 100644 --- a/internal/handler/cargo_test.go +++ b/internal/handler/cargo_test.go @@ -2,6 +2,7 @@ package handler import ( "encoding/json" + "io" "log/slog" "net/http" "net/http/httptest" @@ -10,6 +11,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) func cargoTestProxy() *Proxy { @@ -70,6 +72,75 @@ func TestCargoConfigEndpoint(t *testing.T) { } } +func TestCargoHandlerUsesConfiguredUpstreams(t *testing.T) { + t.Run("index", func(t *testing.T) { + var requestPath, authHeader string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + authHeader = r.Header.Get("Authorization") + if authHeader != "Bearer cargo-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "text/plain") + _, _ = io.WriteString(w, `{"name":"serde","vers":"1.0.0"}`) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.AuthForURL = func(string) (string, string) { + return "Authorization", "Bearer cargo-token" + } + h := NewCargoHandler( + proxy, + "http://proxy.test", + upstream.URL+"/index/", + "https://crates.example.test/files/", + ) + + req := httptest.NewRequest(http.MethodGet, "/se/rd/serde", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if requestPath != "/index/se/rd/serde" { + t.Errorf("upstream path = %q, want %q", requestPath, "/index/se/rd/serde") + } + if authHeader != "Bearer cargo-token" { + t.Errorf("Authorization = %q, want %q", authHeader, "Bearer cargo-token") + } + }) + + t.Run("download", func(t *testing.T) { + proxy, _, _, artifactFetcher := setupTestProxy(t) + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("crate")), + ContentType: "application/gzip", + } + h := NewCargoHandler( + proxy, + "http://proxy.test", + "https://index.example.test/root/", + "https://crates.example.test/files/", + ) + + req := httptest.NewRequest(http.MethodGet, "/crates/serde/1.0.0/download", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + want := "https://crates.example.test/files/serde/serde-1.0.0.crate" + if artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + }) +} + func TestCargoIndexProxy(t *testing.T) { // Create a mock upstream index server indexContent := `{"name":"serde","vers":"1.0.0","deps":[],"cksum":"abc123"} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index e62292f..fc78dbf 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -104,6 +104,7 @@ type Proxy struct { // storage at an internal one. DirectServeBaseURL string HTTPClient *http.Client + AuthForURL func(string) (headerName, headerValue string) } // NewProxy creates a new Proxy with the given dependencies. @@ -405,6 +406,7 @@ func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamUR req.Header.Set(header, v) } } + p.applyUpstreamAuth(req) resp, err := p.HTTPClient.Do(req) if err != nil { @@ -431,6 +433,7 @@ func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL st http.Error(w, "failed to create request", http.StatusInternalServerError) return } + p.applyUpstreamAuth(req) resp, err := p.HTTPClient.Do(req) if err != nil { @@ -571,6 +574,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return nil, "", "", zeroTime, fmt.Errorf("creating request: %w", err) } req.Header.Set("Accept", accept) + p.applyUpstreamAuth(req) if entry != nil && entry.ETag.Valid { req.Header.Set("If-None-Match", entry.ETag.String) @@ -760,6 +764,7 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst accept = acceptHeaders[0] } req.Header.Set("Accept", accept) + p.applyUpstreamAuth(req) for _, header := range []string{headerAcceptEncoding, "If-Modified-Since", "If-None-Match"} { if v := r.Header.Get(header); v != "" { @@ -784,6 +789,17 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst _, _ = io.Copy(w, resp.Body) } +func (p *Proxy) applyUpstreamAuth(req *http.Request) { + if p.AuthForURL == nil { + return + } + + headerName, headerValue := p.AuthForURL(req.URL.String()) + if headerName != "" && headerValue != "" { + req.Header.Set(headerName, headerValue) + } +} + // GetOrFetchArtifactFromURL retrieves an artifact from cache or fetches from a specific URL. // This is useful for registries where download URLs are determined from metadata. func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) { diff --git a/internal/handler/npm.go b/internal/handler/npm.go index a3ed910..ee9b59c 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -25,10 +25,14 @@ type NPMHandler struct { } // NewNPMHandler creates a new npm protocol handler. -func NewNPMHandler(proxy *Proxy, proxyURL string) *NPMHandler { +func NewNPMHandler(proxy *Proxy, proxyURL, upstreamURL string) *NPMHandler { + if strings.TrimSpace(upstreamURL) == "" { + upstreamURL = npmUpstream + } + return &NPMHandler{ proxy: proxy, - upstreamURL: npmUpstream, + upstreamURL: strings.TrimSuffix(upstreamURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"), } } @@ -261,7 +265,15 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("npm download request", "package", packageName, "version", version, "filename", filename) - result, err := h.proxy.GetOrFetchArtifact(r.Context(), "npm", packageName, version, filename) + downloadURL := fmt.Sprintf( + "%s/%s/-/%s", + h.upstreamURL, + escapeNPMDownloadPackage(packageName), + url.PathEscape(filename), + ) + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), "npm", packageName, version, filename, downloadURL, + ) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, "package not found") @@ -275,6 +287,14 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifact(w, result) } +func escapeNPMDownloadPackage(packageName string) string { + scope, name, scoped := strings.Cut(packageName, "/") + if scoped && strings.HasPrefix(scope, "@") && len(scope) > 1 && name != "" && !strings.Contains(name, "/") { + return url.PathEscape(scope) + "/" + url.PathEscape(name) + } + return url.PathEscape(packageName) +} + // extractPackageName extracts the package name from the request path. // Handles both scoped (@scope/name) and unscoped (name) packages. func (h *NPMHandler) extractPackageName(r *http.Request) string { diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index bc1edde..e0257dd 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -2,13 +2,16 @@ package handler import ( "encoding/json" + "io" "log/slog" "net/http" "net/http/httptest" + "strings" "testing" "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) const testVersion100 = "1.0.0" @@ -46,6 +49,86 @@ func TestNPMExtractVersionFromFilename(t *testing.T) { } } +func TestNPMHandlerUsesConfiguredUpstream(t *testing.T) { + t.Run("metadata", func(t *testing.T) { + var requestPath, authHeader string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + authHeader = r.Header.Get("Authorization") + if authHeader != "Bearer npm-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"versions":{}}`) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.AuthForURL = func(string) (string, string) { + return "Authorization", "Bearer npm-token" + } + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/root/") + + req := httptest.NewRequest(http.MethodGet, "/testpkg", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if requestPath != "/root/testpkg" { + t.Errorf("upstream path = %q, want %q", requestPath, "/root/testpkg") + } + if authHeader != "Bearer npm-token" { + t.Errorf("Authorization = %q, want %q", authHeader, "Bearer npm-token") + } + }) + + t.Run("download", func(t *testing.T) { + proxy, _, _, artifactFetcher := setupTestProxy(t) + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/") + + req := httptest.NewRequest(http.MethodGet, "/testpkg/-/testpkg-1.0.0.tgz", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + want := "https://npm.example.test/root/testpkg/-/testpkg-1.0.0.tgz" + if artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + }) + + t.Run("scoped download", func(t *testing.T) { + proxy, _, _, artifactFetcher := setupTestProxy(t) + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/") + + req := httptest.NewRequest(http.MethodGet, "/@scope/name/-/name-1.0.0.tgz", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + want := "https://npm.example.test/root/@scope/name/-/name-1.0.0.tgz" + if artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + }) +} + func TestNPMRewriteMetadata(t *testing.T) { h := &NPMHandler{ proxy: testProxy(), diff --git a/internal/server/server.go b/internal/server/server.go index 74c82c7..13c5997 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -167,6 +167,7 @@ func (s *Server) Start() error { } proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy.HTTPClient.Timeout = s.cfg.ParseHTTPTimeout() + proxy.AuthForURL = s.authForURL proxy.Cooldown = cd proxy.CacheMetadata = s.cfg.CacheMetadata proxy.MetadataTTL = s.cfg.ParseMetadataTTL() @@ -196,8 +197,13 @@ func (s *Server) Start() error { }) // Mount protocol handlers - npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL) - cargoHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL) + npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM) + cargoHandler := handler.NewCargoHandler( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.Cargo, + s.cfg.Upstream.CargoDownload, + ) gemHandler := handler.NewGemHandler(proxy, s.cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, s.cfg.BaseURL) hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index f1de62d..2d27147 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -67,8 +67,13 @@ func newTestServer(t *testing.T) *testServer { r := chi.NewRouter() // Mount handlers - npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL) - cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL) + npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL, cfg.Upstream.NPM) + cargoHandler := handler.NewCargoHandler( + proxy, + cfg.BaseURL, + cfg.Upstream.Cargo, + cfg.Upstream.CargoDownload, + ) gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, cfg.BaseURL) pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL) From 44041d07a986de9b567c4a7e3dd5eb34aa3f5b8b Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 27 Jul 2026 12:15:57 +0100 Subject: [PATCH 33/70] Bump git-pkgs archives, enrichment, purl, registries, vulns (#215) --- go.mod | 17 +++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 6d81076..f3becd4 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.25.6 require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.11.0 - github.com/git-pkgs/archives v0.3.0 + github.com/git-pkgs/archives v0.3.1 github.com/git-pkgs/cooldown v0.1.1 - github.com/git-pkgs/enrichment v0.6.0 - github.com/git-pkgs/purl v0.1.14 - github.com/git-pkgs/registries v0.6.3 + github.com/git-pkgs/enrichment v0.6.4 + github.com/git-pkgs/purl v0.1.15 + github.com/git-pkgs/registries v0.6.4 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.3.0 - github.com/git-pkgs/vulns v0.2.0 + github.com/git-pkgs/vulns v0.2.1 github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 @@ -115,7 +115,7 @@ require ( github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ecosyste-ms/ecosystems-go v0.3.0 // indirect + github.com/ecosyste-ms/ecosystems-go v0.4.0 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/fatih/color v1.18.0 // indirect @@ -217,7 +217,8 @@ require ( github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect - github.com/oapi-codegen/runtime v1.4.2 // indirect + github.com/oapi-codegen/nullable v1.1.0 // indirect + github.com/oapi-codegen/runtime v1.6.0 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -265,7 +266,7 @@ require ( github.com/timonwong/loggercheck v0.11.0 // indirect github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect - github.com/ulikunitz/xz v0.5.15 // indirect + github.com/ulikunitz/xz v0.5.16 // indirect github.com/ultraware/funlen v0.2.0 // indirect github.com/ultraware/whitespace v0.2.0 // indirect github.com/urfave/cli/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 5d16d31..34e9116 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ecosyste-ms/ecosystems-go v0.3.0 h1:eVTNKQ3PyVNkSAnk1934958Vg4rR7ho5B5MWEQaMLi4= -github.com/ecosyste-ms/ecosystems-go v0.3.0/go.mod h1:bkjiI8WoTFB1Jw0M3h8yn3KXCD/+LdETsQ3m2BNLMHQ= +github.com/ecosyste-ms/ecosystems-go v0.4.0 h1:5A+zF+XWT8sYYYjlc2/tI1SmiDGzbHLyT9CapVc5dGA= +github.com/ecosyste-ms/ecosystems-go v0.4.0/go.mod h1:FVswCrp3DQkur1HjVqfDF/gYrDSEmiFflntcB1G0DbA= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= @@ -244,26 +244,26 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= -github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x78= -github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= +github.com/git-pkgs/archives v0.3.1 h1:GKUuw++0YXAAElxweVHiR4AaSShKKYoVQmyxlF5blG4= +github.com/git-pkgs/archives v0.3.1/go.mod h1:408oQv3FxLCtePa33zp3sg3njXnwH74vnHZFxkRqoPo= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= -github.com/git-pkgs/enrichment v0.6.0 h1:npV6N+eFZnI64uw/B0s+WJPn6Fu6Be08bexAViZFjMg= -github.com/git-pkgs/enrichment v0.6.0/go.mod h1:ov2WDaiNoIXViqdnE1FlUjNTnB5RnkUH4et9BWPhUDY= +github.com/git-pkgs/enrichment v0.6.4 h1:mGrfenttwmcUfPXRkWpB0wBJiiGj55ltniUh66Pq4bU= +github.com/git-pkgs/enrichment v0.6.4/go.mod h1:zz1vPUak/w8Jhajll0KDRN2MjKaEYeCzQTxumWnVhqY= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= -github.com/git-pkgs/purl v0.1.14 h1:GgqwiBNS0eKJqJ/gabUBEC10Xhpj6UX12kfNzujaeYc= -github.com/git-pkgs/purl v0.1.14/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= -github.com/git-pkgs/registries v0.6.3 h1:7sveeeMS2lgXtcqNYAA3bwaT7H9CQs0uhy6wQSxz3Js= -github.com/git-pkgs/registries v0.6.3/go.mod h1:j4o50ii/vD9Z42/nFeQASt15BPolADIejeFmmqK4njo= +github.com/git-pkgs/purl v0.1.15 h1:iQ3clh0Cw41rkM0rf24B7ShnN9Z+UtLMAFlNDUs+Qd4= +github.com/git-pkgs/purl v0.1.15/go.mod h1:PqCLVBDeZrZgHysR803/AntMELgIr2LFZVNCcwLH2m0= +github.com/git-pkgs/registries v0.6.4 h1:Kq/KlStjaQyE83UXT/tKuzCrIzc4keGeBjtroMqgoHA= +github.com/git-pkgs/registries v0.6.4/go.mod h1:YkGHbxHIe2Ha/ROH6zNkS5PJUUoa9g0Ti/s2XhZnrak= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= -github.com/git-pkgs/vulns v0.2.0 h1:S1sA0ObQ/IKW0vqK+lK+HAY37AdDHEfhaIja/fLptwg= -github.com/git-pkgs/vulns v0.2.0/go.mod h1:mjVquLlunsAFPltqjf6mbvbI4tQ1iIa1NuhKIOPKt/4= +github.com/git-pkgs/vulns v0.2.1 h1:tWGhOfPVDZwkM2Y9vRkMpMR+gjtlu2jhERS5JeNBoKQ= +github.com/git-pkgs/vulns v0.2.1/go.mod h1:/0gHKHQR5SWttZVEMqgOvCXssKFwAtbac/PfkhBax9o= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= @@ -512,8 +512,8 @@ github.com/nunnatsa/ginkgolinter v0.23.0 h1:x3o4DGYOWbBMP/VdNQKgSj+25aJKx2Pe6lHr github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= -github.com/oapi-codegen/runtime v1.4.2 h1:GMxFVYLzoYLua+/KvzgSphkyK1lLTReQI9Vf4hvATKE= -github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU= +github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= @@ -654,8 +654,8 @@ github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVF github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= -github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= -github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= +github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= From 90422697b81da002b7ad9ec2908c9cb6c7d7e84c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:16:27 +0100 Subject: [PATCH 34/70] Bump google.golang.org/grpc from 1.81.1 to 1.82.1 (#216) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.81.1 to 1.82.1. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.81.1...v1.82.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.82.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index f3becd4..c460ec7 100644 --- a/go.mod +++ b/go.mod @@ -303,8 +303,8 @@ require ( golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect - google.golang.org/grpc v1.81.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index 34e9116..5621b47 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,8 @@ github.com/CycloneDX/cyclonedx-go v0.11.0 h1:GokP8FiRC+foiuwWhSSLpSD5H4hSWtGnR3w github.com/CycloneDX/cyclonedx-go v0.11.0/go.mod h1:vUvbCXQsEm48OI6oOlanxstwNByXjCZ2wuleUlwGEO8= github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g= github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= @@ -702,8 +702,8 @@ go.augendre.info/fatcontext v0.9.0 h1:Gt5jGD4Zcj8CDMVzjOJITlSb9cEch54hjRRlN3qDoj go.augendre.info/fatcontext v0.9.0/go.mod h1:L94brOAT1OOUNue6ph/2HnwxoNlds9aXDF2FcUntbNw= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= @@ -857,12 +857,12 @@ google.golang.org/api v0.272.0 h1:eLUQZGnAS3OHn31URRf9sAmRk3w2JjMx37d2k8AjJmA= google.golang.org/api v0.272.0/go.mod h1:wKjowi5LNJc5qarNvDCvNQBn3rVK8nSy6jg2SwRwzIA= google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE= google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw= -google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js= -google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 h1:aJmi6DVGGIStN9Mobk/tZOOQUBbj0BPjZjjnOdoZKts= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From a5d456790d9da7455e6194cf28fc11d73c091eb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:14:36 +0100 Subject: [PATCH 35/70] Bump actions/checkout from 7.0.0 to 7.0.1 (#218) Bumps [actions/checkout](https://github.com/actions/checkout) from 7.0.0 to 7.0.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0...3d3c42e5aac5ba805825da76410c181273ba90b1) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/swagger.yml | 2 +- .github/workflows/zizmor.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c405f5d..bb3d87d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -35,7 +35,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9c99e5d..bb483f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,7 +20,7 @@ jobs: contents: read steps: - name: Check out the repo - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f06cecd..5d32181 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index e99aecf..625f944 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -12,7 +12,7 @@ jobs: swagger: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index df06c5f..d50b0a4 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -21,7 +21,7 @@ jobs: security-events: write steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false From b56a1fed658b91f5b6839ab02abee4471f22ea9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:52:35 +0100 Subject: [PATCH 36/70] Bump docker/login-action from 4.4.0 to 4.5.0 (#220) Bumps [docker/login-action](https://github.com/docker/login-action) from 4.4.0 to 4.5.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/af1e73f918a031802d376d3c8bbc3fe56130a9b0...06fb636fac595d6fb4b28a5dfcb21a6f5091859c) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bb483f1..c08cfa6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: Log in to the Container registry - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 + uses: docker/login-action@06fb636fac595d6fb4b28a5dfcb21a6f5091859c with: registry: ghcr.io username: ${{ github.actor }} From 1057ee926eb29ea87015b5f6109242cc86d27d14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:55:16 +0100 Subject: [PATCH 37/70] Bump zizmorcore/zizmor-action from 0.6.0 to 0.6.1 (#217) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.0 to 0.6.1. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/6599ee8b7a49aef6a770f63d261d214911a7ce02...6fc4b006235f201fdab3722e17240ab420d580e5) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index d50b0a4..4df0e18 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0 + uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1 From 31ecca8cf14a643ea8479109b8d169ea4842a174 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:31:21 +0100 Subject: [PATCH 38/70] Bump github.com/prometheus/client_golang from 1.23.2 to 1.24.0 (#221) Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.23.2 to 1.24.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/v1.24.0/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.23.2...v1.24.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-version: 1.24.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index c460ec7..005ec32 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 - github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_golang v1.24.0 github.com/prometheus/client_model v0.6.2 github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 @@ -225,8 +225,8 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.67.5 // indirect - github.com/prometheus/procfs v0.20.1 // indirect + github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/quasilyte/go-ruleguard v0.4.5 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect @@ -290,7 +290,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect @@ -298,7 +298,7 @@ require ( golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.46.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect diff --git a/go.sum b/go.sum index 5621b47..0801599 100644 --- a/go.sum +++ b/go.sum @@ -422,8 +422,8 @@ github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3 github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -541,14 +541,14 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= +github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= -github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= -github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= +github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= github.com/quasilyte/go-ruleguard v0.4.5/go.mod h1:Vl05zJ538vcEEwu16V/Hdu7IYZWyKSwIy4c88Ro1kRE= github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuMRoVWSkXC4uvY= @@ -726,8 +726,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= @@ -807,8 +807,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= From 9a9a82176da7988d4ded052c39f996e93c5adff8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:32:38 +0100 Subject: [PATCH 39/70] Bump modernc.org/sqlite from 1.54.0 to 1.55.0 (#219) Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.54.0 to 1.55.0. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.54.0...v1.55.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.55.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 005ec32..2f62065 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.54.0 + modernc.org/sqlite v1.55.0 ) require ( diff --git a/go.sum b/go.sum index 0801599..4f356f9 100644 --- a/go.sum +++ b/go.sum @@ -904,8 +904,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= -modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM= +modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 36f3a51c6523ca4b296dadec66bf8080263932c1 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 31 Jul 2026 17:04:01 +0100 Subject: [PATCH 40/70] Detect content types when browsing files --- go.mod | 3 +- go.sum | 6 +- internal/server/browse.go | 128 +++++++++++++++---------- internal/server/browse_bench_test.go | 25 +++++ internal/server/browse_test.go | 138 ++++++++++++++++----------- 5 files changed, 190 insertions(+), 110 deletions(-) diff --git a/go.mod b/go.mod index 2f62065..e6e0a08 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,10 @@ go 1.25.6 require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.11.0 - github.com/git-pkgs/archives v0.3.1 + github.com/git-pkgs/archives v0.4.0 github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/enrichment v0.6.4 + github.com/git-pkgs/magic v0.1.0 github.com/git-pkgs/purl v0.1.15 github.com/git-pkgs/registries v0.6.4 github.com/git-pkgs/spdx v0.1.4 diff --git a/go.sum b/go.sum index 4f356f9..c239240 100644 --- a/go.sum +++ b/go.sum @@ -244,12 +244,14 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= -github.com/git-pkgs/archives v0.3.1 h1:GKUuw++0YXAAElxweVHiR4AaSShKKYoVQmyxlF5blG4= -github.com/git-pkgs/archives v0.3.1/go.mod h1:408oQv3FxLCtePa33zp3sg3njXnwH74vnHZFxkRqoPo= +github.com/git-pkgs/archives v0.4.0 h1:KNmmIsLiSH27lUdT27EfUkQXFaLgXV5KezE81iyOIgo= +github.com/git-pkgs/archives v0.4.0/go.mod h1:tfio0OIuPKEBKHs/UCL5XBUvYmKpnvtnba2iDlfSd6g= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= github.com/git-pkgs/enrichment v0.6.4 h1:mGrfenttwmcUfPXRkWpB0wBJiiGj55ltniUh66Pq4bU= github.com/git-pkgs/enrichment v0.6.4/go.mod h1:zz1vPUak/w8Jhajll0KDRN2MjKaEYeCzQTxumWnVhqY= +github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY= +github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= diff --git a/internal/server/browse.go b/internal/server/browse.go index 504f5f1..3ea5676 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -1,6 +1,7 @@ package server import ( + "bufio" "encoding/json" "fmt" "io" @@ -10,29 +11,22 @@ import ( "github.com/git-pkgs/archives" "github.com/git-pkgs/archives/diff" + "github.com/git-pkgs/magic" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/purl" "github.com/go-chi/chi/v5" ) -const contentTypePlainText = "text/plain; charset=utf-8" +const ( + contentTypePlainText = "text/plain; charset=utf-8" + browseSniffSize = 512 +) // maxBrowseArchiveSize caps how much data openArchive will buffer for // prefix detection. Artifacts larger than this are rejected to prevent // memory exhaustion from a single request. const maxBrowseArchiveSize = 512 << 20 // 512 MB -// archiveFilename returns a filename suitable for archive format detection. -// Some ecosystems (e.g. composer) store artifacts with bare hash filenames -// that have no extension. This adds .zip when the original has no extension -// and the content is likely a zip archive. -func archiveFilename(filename string) string { - if path.Ext(filename) == "" { - return filename + ".zip" - } - return filename -} - // detectSingleRootDir returns the single top-level directory name if all files // in the archive live under one common directory (e.g. GitHub zipballs use // "repo-hash/"). Returns "" if there's no single root or the archive is flat. @@ -66,8 +60,6 @@ func detectSingleRootDir(reader archives.Reader) string { // and stripping a single top-level directory prefix (like GitHub zipballs). // For npm, the hardcoded "package/" prefix takes precedence. func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations - fname := archiveFilename(filename) - limited := io.LimitReader(content, maxBrowseArchiveSize+1) data, err := io.ReadAll(limited) if err != nil { @@ -78,17 +70,17 @@ func openArchive(filename string, content io.Reader, ecosystem string) (archives } if ecosystem == "npm" { - return archives.OpenBytesWithPrefix(fname, data, "package/") + return archives.OpenBytesWithPrefix(filename, data, "package/") } - probe, err := archives.OpenBytes(fname, data) + probe, err := archives.OpenBytes(filename, data) if err != nil { return nil, err } prefix := detectSingleRootDir(probe) _ = probe.Close() - return archives.OpenBytesWithPrefix(fname, data, prefix) + return archives.OpenBytesWithPrefix(filename, data, prefix) } // BrowseListResponse contains the file listing for a directory in an archives. @@ -361,7 +353,14 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n } defer func() { _ = fileReader.Close() }() - contentType := detectContentType(filePath) + contentType, knownPath := detectContentTypeFromPath(filePath) + var content io.Reader = fileReader + if !knownPath { + bufferedFile := bufio.NewReaderSize(fileReader, browseSniffSize) + prefix, _ := bufferedFile.Peek(browseSniffSize) + contentType = detectContentType(filePath, prefix) + content = bufferedFile + } w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Security-Policy", "sandbox") w.Header().Set("X-Content-Type-Options", "nosniff") @@ -370,85 +369,112 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename)) // Stream the file - _, _ = io.Copy(w, fileReader) + _, _ = io.Copy(w, content) } -// detectContentType returns an appropriate content type based on file extension. -func detectContentType(filename string) string { +// detectContentType returns an appropriate content type. Known filenames and +// extensions take precedence; content detection handles the remaining files. +func detectContentType(filename string, prefix []byte) string { + if contentType, ok := detectContentTypeFromPath(filename); ok { + return contentType + } + return detectContentTypeFromPrefix(prefix) +} + +func detectContentTypeFromPath(filename string) (string, bool) { ext := strings.ToLower(path.Ext(filename)) switch ext { // Text formats case ".txt", ".md", ".markdown": - return contentTypePlainText + return contentTypePlainText, true case ".html", ".htm", ".xhtml": - return contentTypePlainText + return contentTypePlainText, true case ".css": - return "text/css; charset=utf-8" + return "text/css; charset=utf-8", true case ".js", ".mjs": - return "application/javascript; charset=utf-8" + return "application/javascript; charset=utf-8", true case ".json": - return "application/json; charset=utf-8" + return "application/json; charset=utf-8", true case ".xml": - return "application/xml; charset=utf-8" + return "application/xml; charset=utf-8", true case ".yaml", ".yml": - return "text/yaml; charset=utf-8" + return "text/yaml; charset=utf-8", true case ".toml": - return "text/toml; charset=utf-8" + return "text/toml; charset=utf-8", true // Programming languages case ".go": - return "text/x-go; charset=utf-8" + return "text/x-go; charset=utf-8", true case ".rs": - return "text/x-rust; charset=utf-8" + return "text/x-rust; charset=utf-8", true case ".py": - return "text/x-python; charset=utf-8" + return "text/x-python; charset=utf-8", true case ".rb": - return "text/x-ruby; charset=utf-8" + return "text/x-ruby; charset=utf-8", true case ".java": - return "text/x-java; charset=utf-8" + return "text/x-java; charset=utf-8", true case ".c", ".h": - return "text/x-c; charset=utf-8" + return "text/x-c; charset=utf-8", true case ".cpp", ".cc", ".cxx", ".hpp": - return "text/x-c++; charset=utf-8" + return "text/x-c++; charset=utf-8", true case ".ts": - return "text/typescript; charset=utf-8" + return "text/typescript; charset=utf-8", true case ".tsx": - return "text/tsx; charset=utf-8" + return "text/tsx; charset=utf-8", true case ".jsx": - return "text/jsx; charset=utf-8" + return "text/jsx; charset=utf-8", true case ".php": - return "text/x-php; charset=utf-8" + return "text/x-php; charset=utf-8", true // Config files case ".conf", ".config", ".ini": - return contentTypePlainText + return contentTypePlainText, true case ".sh", ".bash": - return "text/x-shellscript; charset=utf-8" + return "text/x-shellscript; charset=utf-8", true case ".dockerfile": - return "text/x-dockerfile; charset=utf-8" + return "text/x-dockerfile; charset=utf-8", true // Images case ".png": - return "image/png" + return "image/png", true case ".jpg", ".jpeg": - return "image/jpeg" + return "image/jpeg", true case ".gif": - return "image/gif" + return "image/gif", true case ".svg": - return contentTypePlainText + return contentTypePlainText, true case ".ico": - return "image/x-icon" + return "image/x-icon", true // Archives case ".zip", ".tar", ".gz", ".bz2", ".xz": - return "application/octet-stream" + return "application/octet-stream", true default: - // Try to detect if it looks like text if isLikelyText(filename) { - return contentTypePlainText + return contentTypePlainText, true } + return "", false + } +} + +func detectContentTypeFromPrefix(prefix []byte) string { + result := magic.DetectPrefix(prefix) + if result.Kind == magic.KindText { + return contentTypePlainText + } + + switch result.Format { + case "png": + return "image/png" + case "jpeg": + return "image/jpeg" + case "gif": + return "image/gif" + case "pdf": + return "application/pdf" + default: return "application/octet-stream" } } diff --git a/internal/server/browse_bench_test.go b/internal/server/browse_bench_test.go index 03f3f02..16b2419 100644 --- a/internal/server/browse_bench_test.go +++ b/internal/server/browse_bench_test.go @@ -55,3 +55,28 @@ func BenchmarkOpenArchive(b *testing.B) { }) } } + +func BenchmarkDetectContentType(b *testing.B) { + cases := []struct { + name string + filename string + prefix []byte + }{ + {"known-path", "README.md", nil}, + {"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize)}, + {"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...)}, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var contentType string + for b.Loop() { + contentType = detectContentType(tc.filename, tc.prefix) + } + if contentType == "" { + b.Fatal("empty content type") + } + }) + } +} diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index f1fb993..6b1e487 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -137,29 +137,44 @@ func TestHandleBrowseFile(t *testing.T) { t.Fatalf("failed to upsert artifact: %v", err) } - // Test fetching a file - req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/README.md", nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + files := []struct { + path string + content string + contentType string + }{ + {"README.md", "# Test Package\n", contentTypePlainText}, + {"notes.data", "short text\n", contentTypePlainText}, + {"logo", "\x89PNG\r\n\x1a\nimage data", "image/png"}, + {"page", "", contentTypePlainText}, + {"misleading.txt", "\x89PNG\r\n\x1a\nimage data", contentTypePlainText}, } + for _, file := range files { + t.Run(file.path, func(t *testing.T) { + req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/"+file.path, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) - body := w.Body.String() - if body != "# Test Package\n" { - t.Errorf("unexpected file content: %q", body) - } - - // Check content type - contentType := w.Header().Get("Content-Type") - if contentType != contentTypePlainText { - t.Errorf("expected text/plain content type, got %q", contentType) + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + } + if w.Body.String() != file.content { + t.Errorf("unexpected file content: %q", w.Body.String()) + } + if got := w.Header().Get("Content-Type"); got != file.contentType { + t.Errorf("Content-Type = %q, want %q", got, file.contentType) + } + if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" { + t.Errorf("Content-Security-Policy = %q, want sandbox", got) + } + if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" { + t.Errorf("X-Content-Type-Options = %q, want nosniff", got) + } + }) } // Test fetching non-existent file - req = httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil) - w = httptest.NewRecorder() + req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil) + w := httptest.NewRecorder() ts.handler.ServeHTTP(w, req) if w.Code != http.StatusNotFound { @@ -169,34 +184,47 @@ func TestHandleBrowseFile(t *testing.T) { func TestDetectContentType(t *testing.T) { tests := []struct { + name string filename string + prefix []byte expectedCT string }{ - {"file.txt", contentTypePlainText}, - {"file.md", contentTypePlainText}, - {"file.json", "application/json; charset=utf-8"}, - {"file.js", "application/javascript; charset=utf-8"}, - {"file.go", "text/x-go; charset=utf-8"}, - {"file.py", "text/x-python; charset=utf-8"}, - {"file.rs", "text/x-rust; charset=utf-8"}, - {"file.html", contentTypePlainText}, - {"file.htm", contentTypePlainText}, - {"file.xhtml", contentTypePlainText}, - {"file.svg", contentTypePlainText}, - {"file.png", "image/png"}, - {"file.jpg", "image/jpeg"}, - {"README", contentTypePlainText}, - {"LICENSE", contentTypePlainText}, - {"Makefile", contentTypePlainText}, - {".gitignore", contentTypePlainText}, - {"file.bin", "application/octet-stream"}, + {"text extension", "file.txt", nil, contentTypePlainText}, + {"markdown extension", "file.md", nil, contentTypePlainText}, + {"JSON extension", "file.json", nil, "application/json; charset=utf-8"}, + {"JavaScript extension", "file.js", nil, "application/javascript; charset=utf-8"}, + {"Go extension", "file.go", nil, "text/x-go; charset=utf-8"}, + {"Python extension", "file.py", nil, "text/x-python; charset=utf-8"}, + {"Rust extension", "file.rs", nil, "text/x-rust; charset=utf-8"}, + {"HTML extension", "file.html", nil, contentTypePlainText}, + {"HTM extension", "file.htm", nil, contentTypePlainText}, + {"XHTML extension", "file.xhtml", nil, contentTypePlainText}, + {"SVG extension", "file.svg", nil, contentTypePlainText}, + {"PNG extension", "file.png", nil, "image/png"}, + {"JPEG extension", "file.jpg", nil, "image/jpeg"}, + {"README", "README", nil, contentTypePlainText}, + {"LICENSE", "LICENSE", nil, contentTypePlainText}, + {"Makefile", "Makefile", nil, contentTypePlainText}, + {"gitignore", ".gitignore", nil, contentTypePlainText}, + {"unknown empty", "file.bin", nil, "application/octet-stream"}, + {"extensionless PNG", "asset", []byte("\x89PNG\r\n\x1a\n"), "image/png"}, + {"extensionless JPEG", "asset", []byte("\xff\xd8\xff"), "image/jpeg"}, + {"extensionless GIF", "asset", []byte("GIF89a"), "image/gif"}, + {"extensionless PDF", "asset", []byte("%PDF-1.7"), "application/pdf"}, + {"extensionless text", "asset", []byte("plain text\n"), contentTypePlainText}, + {"extensionless HTML", "asset", []byte(""), contentTypePlainText}, + {"extensionless XML", "asset", []byte(""), contentTypePlainText}, + {"extensionless SVG", "asset", []byte(""), contentTypePlainText}, + {"extensionless ZIP", "asset", []byte("PK\x03\x04"), "application/octet-stream"}, + {"extensionless binary", "asset", []byte{0, 1, 2}, "application/octet-stream"}, + {"known path wins", "file.txt", []byte("\x89PNG\r\n\x1a\n"), contentTypePlainText}, } for _, tt := range tests { - t.Run(tt.filename, func(t *testing.T) { - got := detectContentType(tt.filename) + t.Run(tt.name, func(t *testing.T) { + got := detectContentType(tt.filename, tt.prefix) if got != tt.expectedCT { - t.Errorf("detectContentType(%q) = %q, want %q", tt.filename, got, tt.expectedCT) + t.Errorf("detectContentType(%q, %q) = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT) } }) } @@ -255,6 +283,10 @@ func createTestArchive(t *testing.T) []byte { "package/lib/index.js": "module.exports = {};", "package/lib/helper.js": "module.exports.help = () => {};", "package/test/index.test.js": "// tests", + "package/notes.data": "short text\n", + "package/logo": "\x89PNG\r\n\x1a\nimage data", + "package/page": "", + "package/misleading.txt": "\x89PNG\r\n\x1a\nimage data", } for path, content := range files { @@ -609,25 +641,19 @@ func TestHandleComparePage(t *testing.T) { } } -func TestArchiveFilename(t *testing.T) { - tests := []struct { - input string - want string - }{ - {"package.tar.gz", "package.tar.gz"}, - {"d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856.zip"}, - {"file.zip", "file.zip"}, - {"archive.tgz", "archive.tgz"}, - {"noext", "noext.zip"}, +func TestOpenArchiveDetectsExtensionlessTarGz(t *testing.T) { + reader, err := openArchive("artifact", bytes.NewReader(createTestArchive(t)), "npm") + if err != nil { + t.Fatalf("openArchive failed: %v", err) } + defer func() { _ = reader.Close() }() - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got := archiveFilename(tt.input) - if got != tt.want { - t.Errorf("archiveFilename(%q) = %q, want %q", tt.input, got, tt.want) - } - }) + files, err := reader.List() + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(files) == 0 { + t.Fatal("expected files in extensionless archive") } } From 7dbf13e3450396a7fe0973c50ea59911b4c8f7c6 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sat, 1 Aug 2026 11:22:13 +0100 Subject: [PATCH 41/70] Avoid repeated content type path lookup --- internal/server/browse.go | 11 +---------- internal/server/browse_bench_test.go | 23 +++++++++++++++-------- internal/server/browse_test.go | 9 ++++++--- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/internal/server/browse.go b/internal/server/browse.go index 3ea5676..56aa1c5 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -358,7 +358,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n if !knownPath { bufferedFile := bufio.NewReaderSize(fileReader, browseSniffSize) prefix, _ := bufferedFile.Peek(browseSniffSize) - contentType = detectContentType(filePath, prefix) + contentType = detectContentTypeFromPrefix(prefix) content = bufferedFile } w.Header().Set("Content-Type", contentType) @@ -372,15 +372,6 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n _, _ = io.Copy(w, content) } -// detectContentType returns an appropriate content type. Known filenames and -// extensions take precedence; content detection handles the remaining files. -func detectContentType(filename string, prefix []byte) string { - if contentType, ok := detectContentTypeFromPath(filename); ok { - return contentType - } - return detectContentTypeFromPrefix(prefix) -} - func detectContentTypeFromPath(filename string) (string, bool) { ext := strings.ToLower(path.Ext(filename)) diff --git a/internal/server/browse_bench_test.go b/internal/server/browse_bench_test.go index 16b2419..840bc75 100644 --- a/internal/server/browse_bench_test.go +++ b/internal/server/browse_bench_test.go @@ -58,21 +58,28 @@ func BenchmarkOpenArchive(b *testing.B) { func BenchmarkDetectContentType(b *testing.B) { cases := []struct { - name string - filename string - prefix []byte + name string + filename string + prefix []byte + knownPath bool }{ - {"known-path", "README.md", nil}, - {"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize)}, - {"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...)}, + {"known-path", "README.md", nil, true}, + {"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize), false}, + {"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...), false}, } for _, tc := range cases { b.Run(tc.name, func(b *testing.B) { b.ReportAllocs() var contentType string - for b.Loop() { - contentType = detectContentType(tc.filename, tc.prefix) + if tc.knownPath { + for b.Loop() { + contentType, _ = detectContentTypeFromPath(tc.filename) + } + } else { + for b.Loop() { + contentType = detectContentTypeFromPrefix(tc.prefix) + } } if contentType == "" { b.Fatal("empty content type") diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index 6b1e487..5240a92 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -182,7 +182,7 @@ func TestHandleBrowseFile(t *testing.T) { } } -func TestDetectContentType(t *testing.T) { +func TestBrowseContentTypePolicy(t *testing.T) { tests := []struct { name string filename string @@ -222,9 +222,12 @@ func TestDetectContentType(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := detectContentType(tt.filename, tt.prefix) + got, knownPath := detectContentTypeFromPath(tt.filename) + if !knownPath { + got = detectContentTypeFromPrefix(tt.prefix) + } if got != tt.expectedCT { - t.Errorf("detectContentType(%q, %q) = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT) + t.Errorf("content type for %q with prefix %q = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT) } }) } From 63f0efd0e99a9e5c9dd3b74becf5d418a42b3a54 Mon Sep 17 00:00:00 2001 From: wickedOne Date: Mon, 3 Aug 2026 12:14:03 +0200 Subject: [PATCH 42/70] fix(pypi): resolve name and version for PEP 658 metadata sidecars (#222) * resolve name and version for PEP 658 metadata sidecars * fix(pypi): parse Windows installer and egg filenames separately The bdist_wininst and bdist_msi layout joins the platform to the version with a '.' rather than a '-', so treating .exe/.msi like a wheel folded the platform into the version: foo-1.0.win32-py2.0.exe resolved to version "1.0.win32". Eggs shared the problem, as setuptools' hyphen escaping is not universal: aws-sdk-1.0.0-py3.11.egg resolved to name "aws", version "sdk". Give each format its own parser. Wheels keep the PEP 427 spec-guaranteed field positions, eggs locate the version relative to the py{X.Y} interpreter field, and Windows installers strip the platform and interpreter fields before splitting name from version. A PEP 658 sidecar resolves to the same name and version as the distribution it describes, so it is cached under that version. Browse and compare took the first cached artifact without checking its extension, handing openArchive plain text: a version pip had only fetched metadata for reported hasCached and then 500'd. Add firstBrowsableArtifact, replacing five duplicated selection loops, and export PyPIMetadataSuffix so the suffix has a single definition. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- internal/handler/pypi.go | 182 +++++++++++++++++++++++++++------ internal/handler/pypi_test.go | 78 +++++++++++--- internal/server/browse.go | 59 +++++------ internal/server/browse_test.go | 67 ++++++++++++ internal/server/server.go | 10 +- 5 files changed, 309 insertions(+), 87 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index f5a0232..713f6c8 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -19,7 +19,14 @@ const ( minWheelParts = 5 // name + version + python + abi + platform minSubmatchParts = 2 // full match + first capture group minPyPIPathParts = 3 // hash_prefix + hash + filename - minPythonTagLen = 2 // minimum length for a python tag (e.g., "py") + minEggParts = 3 // name + version + python tag + + // PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip + // appends to a distribution URL when the index advertises core metadata. + // A sidecar resolves to the same name and version as the distribution it + // describes, so it is cached alongside it; consumers that expect an openable + // archive must skip these. + PyPIMetadataSuffix = ".metadata" ) // PyPIHandler handles PyPI registry protocol requests. @@ -433,56 +440,163 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifact(w, result) } +// archiveExtensions are sdist formats of the form {name}-{version}{ext}. They +// carry no trailing tags, but legacy sdist names may contain hyphens. +var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".tgz", ".tar", ".zip"} + +// windowsInstallerExtensions are the legacy distutils bdist_wininst and +// bdist_msi formats, which share a filename layout. +var windowsInstallerExtensions = []string{".exe", ".msi"} + // parseFilename extracts package name and version from a PyPI filename. -// Handles both wheels and sdists: +// Handles wheels, sdists and legacy bdist formats: // - requests-2.31.0-py3-none-any.whl // - requests-2.31.0.tar.gz +// - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg +// - numpy-1.8.0.win32-py2.7.exe func (h *PyPIHandler) parseFilename(filename string) (name, version string) { - // Try wheel format first: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl - if strings.HasSuffix(filename, ".whl") { - base := strings.TrimSuffix(filename, ".whl") - parts := strings.Split(base, "-") - if len(parts) >= minWheelParts { - // Find where version ends (version followed by python tag) - for i := 1; i < len(parts)-2; i++ { - // Check if this looks like a python tag (py2, py3, cp39, etc) - if isPythonTag(parts[i]) { - name = strings.Join(parts[:i-1], "-") - version = parts[i-1] - return - } - } + // PEP 658/714 core-metadata sidecars are the distribution filename plus + // ".metadata"; they describe the same name and version. Without this, pip's + // metadata-only fetches fall back to a hash-derived package identifier. + filename = strings.TrimSuffix(filename, PyPIMetadataSuffix) + + switch { + case strings.HasSuffix(filename, ".whl"): + return parseWheelFilename(strings.TrimSuffix(filename, ".whl")) + case strings.HasSuffix(filename, ".egg"): + return parseEggFilename(strings.TrimSuffix(filename, ".egg")) + } + + for _, ext := range windowsInstallerExtensions { + if strings.HasSuffix(filename, ext) { + return parseWindowsInstallerFilename(strings.TrimSuffix(filename, ext)) } } - // Try sdist formats: {name}-{version}.tar.gz, {name}-{version}.zip - for _, ext := range []string{".tar.gz", ".tar.bz2", ".zip", ".tar"} { + for _, ext := range archiveExtensions { if strings.HasSuffix(filename, ext) { - base := strings.TrimSuffix(filename, ext) - // Find last hyphen followed by version - for i := len(base) - 1; i >= 0; i-- { - if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) { - return base[:i], base[i+1:] - } - } + return splitNameVersion(strings.TrimSuffix(filename, ext)) } } return "", "" } -func isPythonTag(s string) bool { - if len(s) < minPythonTagLen { - return false +// parseWheelFilename parses the PEP 427 layout +// {name}-{version}(-{build})?-{python}-{abi}-{platform}, base being the +// filename without its ".whl" suffix. The spec escapes every hyphen in the name +// and version to '_', so the first two fields are authoritative even when the +// optional build tag is present. +func parseWheelFilename(base string) (name, version string) { + parts := strings.Split(base, "-") + if len(parts) < minWheelParts { + return "", "" } - // Python tags start with py, cp, pp, ip, jy - prefixes := []string{"py", "cp", "pp", "ip", "jy"} - for _, p := range prefixes { - if strings.HasPrefix(s, p) { - return true + + return parts[0], parts[1] +} + +// parseEggFilename parses the setuptools bdist_egg layout +// {name}-{version}-py{X.Y}(-{platform})?, base being the filename without its +// ".egg" suffix. setuptools escapes hyphens in the name and version to '_', but +// eggs built by other tooling do not always, so the version is located relative +// to the interpreter field rather than assumed to be the second field. +func parseEggFilename(base string) (name, version string) { + parts := strings.Split(base, "-") + // Scan from the end: the trailing platform fields never look like an + // interpreter tag, so the last match is the real one even when the package + // name itself carries a "py{N}" component. Stop before index 1, since a tag + // any earlier would leave no room for both a name and a version. + for i := len(parts) - 1; i >= minEggParts-1; i-- { + if !isEggPythonTag(parts[i]) || !isVersionField(parts[i-1]) { + continue + } + + return strings.Join(parts[:i-1], "-"), parts[i-1] + } + + // No interpreter field: {name}-{version}. + return splitNameVersion(base) +} + +// parseWindowsInstallerFilename parses the distutils bdist_wininst and +// bdist_msi layout {name}-{version}.{platform}(-py{X.Y})?, base being the +// filename without its ".exe" or ".msi" suffix. The platform is joined to the +// version with a '.' rather than a '-' and may itself contain a hyphen +// ("win-amd64"), so both trailing fields are stripped before the name and +// version are split apart. +func parseWindowsInstallerFilename(base string) (name, version string) { + if i := strings.LastIndex(base, "-py"); i >= 0 && isDottedNumber(base[i+len("-py"):]) { + base = base[:i] + } + + // The platform is the final '.'-separated field. Requiring it to start with + // a non-digit keeps a dotted version from being truncated when a filename + // carries no platform tag. + i := strings.LastIndex(base, ".") + if i < 0 || i+1 >= len(base) || isVersionStart(base[i+1]) { + return "", "" + } + + return splitFullname(base[:i]) +} + +// splitFullname splits the distutils fullname {name}-{version} that precedes a +// Windows installer's platform field. Unlike an sdist, a wininst fullname may +// carry a trailing build variant ("cx_Oracle-5.1.2-11g"), which belongs to +// neither the name nor the version, so the first purely numeric field wins and +// anything after it is discarded. +func splitFullname(fullname string) (name, version string) { + parts := strings.Split(fullname, "-") + for i := 1; i < len(parts); i++ { + if isDottedNumber(parts[i]) { + return strings.Join(parts[:i], "-"), parts[i] } } - return false + + // No purely numeric field, e.g. a prerelease version like "1.0b1". + return splitNameVersion(fullname) +} + +// splitNameVersion splits a {name}-{version} pair at the last hyphen that +// starts a version, leaving hyphens inside the name intact. +func splitNameVersion(base string) (name, version string) { + for i := len(base) - 1; i >= 0; i-- { + if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) { + return base[:i], base[i+1:] + } + } + + return "", "" +} + +// isEggPythonTag reports whether field is the py{X.Y} interpreter field that +// setuptools places directly after the version in an egg filename. +func isEggPythonTag(field string) bool { + const prefix = "py" + + return len(field) > len(prefix) && strings.HasPrefix(field, prefix) && isVersionStart(field[len(prefix)]) +} + +// isVersionField reports whether field can be a version, i.e. it is non-empty +// and starts with a digit as every PEP 440 release segment does. +func isVersionField(field string) bool { + return field != "" && isVersionStart(field[0]) +} + +// isDottedNumber reports whether s is a dotted numeric version such as "2.7". +func isDottedNumber(s string) bool { + if s == "" || !isVersionStart(s[0]) { + return false + } + + for i := range len(s) { + if !isVersionStart(s[i]) && s[i] != '.' { + return false + } + } + + return true } func isVersionStart(c byte) bool { diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index 2b58960..5ae76ca 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -28,13 +28,58 @@ func TestPyPIParseFilename(t *testing.T) { {"aws-sdk-1.0.0.tar.gz", "aws-sdk", "1.0.0"}, {"zipp-3.17.0.zip", "zipp", "3.17.0"}, + // Additional sdist archive formats + {"lxml-4.9.3.tar.xz", "lxml", "4.9.3"}, + {"docutils-0.20.1.tgz", "docutils", "0.20.1"}, + {"psycopg2-2.9.9.tar.bz2", "psycopg2", "2.9.9"}, + // Wheel formats {"requests-2.31.0-py3-none-any.whl", "requests", "2.31.0"}, {"numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.whl", "numpy", "1.26.2"}, {"cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", "cryptography", "41.0.5"}, + // Wheels with a build tag must not fold the tag into the version + {"foo-1.0-1-py3-none-any.whl", "foo", "1.0"}, + {"tensorflow-2.15.0-2-cp311-cp311-manylinux_2_17_x86_64.whl", "tensorflow", "2.15.0"}, + + // PEP 658 core-metadata sidecars resolve to the distribution they describe + {"backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", "backports_asyncio_runner", "1.2.0"}, + {"requests-2.31.0-py3-none-any.whl.metadata", "requests", "2.31.0"}, + {"requests-2.31.0.tar.gz.metadata", "requests", "2.31.0"}, + + // Eggs: {name}-{version}-py{X.Y}(-{platform})?.egg. Unescaped hyphens in + // the name must not be mistaken for the field separator before the version. + {"numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg", "numpy", "1.8.0"}, + {"aws-sdk-1.0.0-py3.11.egg", "aws-sdk", "1.0.0"}, + {"aws-sdk-1.0.0-py2.7-macosx-10.9-x86_64.egg", "aws-sdk", "1.0.0"}, + {"aws-sdk-1.0.0.egg", "aws-sdk", "1.0.0"}, + // A "py{N}" component inside the name is not the interpreter field, so + // the interpreter must be located from the end of the filename. + {"django-rest-py3-1.0-py3.6.egg", "django-rest-py3", "1.0"}, + + // Windows installers: {name}-{version}.{platform}(-py{X.Y})?.{exe,msi}. + // The platform is not part of the version, and may contain a hyphen. + {"foo-1.0.win32-py2.0.exe", "foo", "1.0"}, + {"pywin32-223.win32-py2.7.exe", "pywin32", "223"}, + {"numpy-1.8.0.win-amd64-py2.7.exe", "numpy", "1.8.0"}, + {"aws-sdk-1.0.0.win32-py2.7.exe", "aws-sdk", "1.0.0"}, + {"pywin32-223.win32.exe", "pywin32", "223"}, + {"cx_Oracle-5.1.2.win32-py2.7.msi", "cx_Oracle", "5.1.2"}, + {"numpy-1.8.0.win-amd64.msi", "numpy", "1.8.0"}, + // A trailing build variant belongs to neither the name nor the version. + {"cx_Oracle-5.1.2-11g.win32-py2.7.exe", "cx_Oracle", "5.1.2"}, + // A prerelease version has no purely numeric field to anchor on. + {"foo-1.0b1.win32-py2.7.exe", "foo", "1.0b1"}, + // Invalid {"invalid", "", ""}, + {"invalid.metadata", "", ""}, + {"backports.ssl_match_hostname-3.4.0.2-py2.7.whl", "", ""}, + {"invalid.exe", "", ""}, + {"foo-1.0.exe", "", ""}, + // An egg with an interpreter field but no version must not promote the + // trailing component of a hyphenated name to the version. + {"aws-sdk-py2.7.egg", "", ""}, } for _, tt := range tests { @@ -94,25 +139,24 @@ func TestPyPIRewriteJSONMetadataCooldown(t *testing.T) { } } -func TestIsPythonTag(t *testing.T) { - tests := []struct { - tag string - want bool - }{ - {"py3", true}, - {"py2", true}, - {"cp311", true}, - {"cp37", true}, - {"pp39", true}, - {"none", false}, - {"any", false}, - {"manylinux", false}, +// TestPyPIParseFilenameNoHashFallback guards the identifier used for caching: +// a filename that parses to an empty name makes handleDownload fall back to a +// "_hash_" package name, which surfaces as a bogus PURL in the package +// overview. +func TestPyPIParseFilenameNoHashFallback(t *testing.T) { + h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}} + + filenames := []string{ + "backports_asyncio_runner-1.2.0-py3-none-any.whl", + "backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", + "backports_asyncio_runner-1.2.0.tar.gz", } - for _, tt := range tests { - got := isPythonTag(tt.tag) - if got != tt.want { - t.Errorf("isPythonTag(%q) = %v, want %v", tt.tag, got, tt.want) + for _, filename := range filenames { + name, version := h.parseFilename(filename) + if name != "backports_asyncio_runner" || version != "1.2.0" { + t.Errorf("parseFilename(%q) = (%q, %q), want (%q, %q)", + filename, name, version, "backports_asyncio_runner", "1.2.0") } } } diff --git a/internal/server/browse.go b/internal/server/browse.go index 56aa1c5..c60cba3 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -13,6 +13,7 @@ import ( "github.com/git-pkgs/archives/diff" "github.com/git-pkgs/magic" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/purl" "github.com/go-chi/chi/v5" ) @@ -27,6 +28,31 @@ const ( // memory exhaustion from a single request. const maxBrowseArchiveSize = 512 << 20 // 512 MB +// firstBrowsableArtifact returns the first cached artifact that can be opened as +// an archive, or nil if the version has none. +// +// A version's artifact list is not all archives: a PEP 658 core-metadata sidecar +// resolves to the same name and version as the distribution it describes, so it +// is cached under that version too. Sidecars are plain text, and because '-' +// sorts before '.' one can even precede the real distribution in the +// filename-ordered list, so selecting blindly would hand openArchive a file it +// cannot parse. +func firstBrowsableArtifact(artifacts []database.Artifact) *database.Artifact { + for i := range artifacts { + if artifacts[i].StoragePath.Valid && !isMetadataSidecar(artifacts[i].Filename) { + return &artifacts[i] + } + } + + return nil +} + +// isMetadataSidecar reports whether filename is a core-metadata sidecar rather +// than a distribution archive. +func isMetadataSidecar(filename string) bool { + return strings.HasSuffix(filename, handler.PyPIMetadataSuffix) +} + // detectSingleRootDir returns the single top-level directory name if all files // in the archive live under one common directory (e.g. GitHub zipballs use // "repo-hash/"). Returns "" if there's no single root or the archive is flat. @@ -214,14 +240,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n return } - // Find the first cached artifact - var cachedArtifact *database.Artifact - for i := range artifacts { - if artifacts[i].StoragePath.Valid { - cachedArtifact = &artifacts[i] - break - } - } + cachedArtifact := firstBrowsableArtifact(artifacts) if cachedArtifact == nil { notFound(w, "artifact not cached") @@ -308,14 +327,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n return } - // Find the first cached artifact - var cachedArtifact *database.Artifact - for i := range artifacts { - if artifacts[i].StoragePath.Valid { - cachedArtifact = &artifacts[i] - break - } - } + cachedArtifact := firstBrowsableArtifact(artifacts) if cachedArtifact == nil { notFound(w, "artifact not cached") @@ -535,19 +547,8 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, } // Find cached artifacts - var fromArtifact, toArtifact *database.Artifact - for i := range fromArtifacts { - if fromArtifacts[i].StoragePath.Valid { - fromArtifact = &fromArtifacts[i] - break - } - } - for i := range toArtifacts { - if toArtifacts[i].StoragePath.Valid { - toArtifact = &toArtifacts[i] - break - } - } + fromArtifact := firstBrowsableArtifact(fromArtifacts) + toArtifact := firstBrowsableArtifact(toArtifacts) if fromArtifact == nil || toArtifact == nil { notFound(w, "one or both versions not cached") diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index 5240a92..6ea0f7e 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -829,3 +829,70 @@ func createTarGzArchive(t *testing.T, files map[string]string) []byte { } return buf.Bytes() } + +// TestFirstBrowsableArtifact guards artifact selection against PEP 658 +// core-metadata sidecars. A sidecar resolves to the same version as the +// distribution it describes, so it is cached under that version, but it is plain +// text and openArchive cannot parse it. +func TestFirstBrowsableArtifact(t *testing.T) { + cached := func(filename string) database.Artifact { + return database.Artifact{ + Filename: filename, + StoragePath: sql.NullString{String: "pypi/" + filename, Valid: true}, + } + } + uncached := func(filename string) database.Artifact { + return database.Artifact{Filename: filename} + } + + tests := []struct { + name string + artifacts []database.Artifact + want string + }{ + {"no artifacts", nil, ""}, + { + "sidecar only is not browsable", + []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata")}, + "", + }, + { + // '-' (0x2D) sorts before '.' (0x2E), so the sidecar precedes the + // sdist in the filename-ordered list the query returns. + "sidecar sorting ahead of the sdist is skipped", + []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0.tar.gz")}, + "foo-1.0.tar.gz", + }, + { + "sidecar skipped in favour of its own wheel", + []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0-py3-none-any.whl")}, + "foo-1.0-py3-none-any.whl", + }, + { + "uncached archive is still not selected", + []database.Artifact{cached("foo-1.0.tar.gz.metadata"), uncached("foo-1.0.tar.gz")}, + "", + }, + {"plain sdist", []database.Artifact{cached("foo-1.0.tar.gz")}, "foo-1.0.tar.gz"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := firstBrowsableArtifact(tt.artifacts) + + if tt.want == "" { + if got != nil { + t.Fatalf("firstBrowsableArtifact() = %q, want nil", got.Filename) + } + return + } + + if got == nil { + t.Fatalf("firstBrowsableArtifact() = nil, want %q", tt.want) + } + if got.Filename != tt.want { + t.Errorf("firstBrowsableArtifact() = %q, want %q", got.Filename, tt.want) + } + }) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 13c5997..153a94c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -791,13 +791,9 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, isOutdated := pkg.LatestVersion.Valid && pkg.LatestVersion.String != version - hasCached := false - for _, art := range artifacts { - if art.StoragePath.Valid { - hasCached = true - break - } - } + // A version whose only cached artifact is a metadata sidecar cannot be + // browsed, so it must not be advertised as cached. + hasCached := firstBrowsableArtifact(artifacts) != nil data := VersionShowData{ Layout: s.layoutFor(r), From 800ffc6b382d7f6d341b01b0b0d9465ed65ae2d1 Mon Sep 17 00:00:00 2001 From: Vincent Palatin Date: Wed, 5 Aug 2026 18:48:00 +0200 Subject: [PATCH 43/70] Make Debian upstream repository configurable. (#229) * Refactor LoadFromEnv to use helpers Avoid triggering the linter about the cyclomatic complexity of the LoadFromEnv function in later changes by refactoring it to use setEnvString/setEnvBool helpers. No functional change, just collapse ~29 repetitive if-blocks into single-line calls to two small helpers. * Make Debian upstream repository configurable Support overriding the Debian handler's upstream (e.g. Ubuntu archives) via PROXY_UPSTREAM_DEBIAN or upstream.debian in the config file. Tested with PROXY_UPSTREAM_DEBIAN=http://archive.ubuntu.com/ubuntu to get Ubuntu Resolute packages. --- internal/config/config.go | 134 +++++++------------ internal/config/config_test.go | 7 + internal/handler/debian.go | 7 +- internal/handler/debian_test.go | 2 +- internal/handler/download_test.go | 2 +- internal/handler/notfound_ecosystems_test.go | 2 +- internal/server/server.go | 2 +- 7 files changed, 66 insertions(+), 90 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index ec510d2..3dabc62 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -304,6 +304,11 @@ type UpstreamConfig struct { // Default: https://static.crates.io/crates CargoDownload string `json:"cargo_download" yaml:"cargo_download"` + // Debian is the upstream APT repository base URL. + // Example: http://archive.ubuntu.com/ubuntu would get Ubuntu. + // Default: http://deb.debian.org/debian + Debian string `json:"debian" yaml:"debian"` + // Auth configures authentication for upstream registries. // Keys are URL prefixes that are matched against request URLs. // Example: "https://npm.pkg.github.com" matches all requests to that host. @@ -378,6 +383,7 @@ func Default() *Config { GradlePluginPortal: "https://plugins.gradle.org/m2", Cargo: "https://index.crates.io", CargoDownload: "https://static.crates.io/crates", + Debian: "http://deb.debian.org/debian", }, Gradle: GradleConfig{ BuildCache: GradleBuildCacheConfig{ @@ -422,6 +428,21 @@ func Load(path string) (*Config, error) { return cfg, nil } +// setEnvString sets *dst from the named environment variable, leaving it +// untouched if the variable is unset or empty. +func setEnvString(dst *string, key string) { + if v := os.Getenv(key); v != "" { + *dst = v + } +} + +// setEnvBool is setEnvString for boolean fields, parsed via envBool. +func setEnvBool(dst *bool, key string) { + if v := os.Getenv(key); v != "" { + *dst = envBool(v) + } +} + // LoadFromEnv applies environment variable overrides to a Config. // Environment variables use the PROXY_ prefix: // - PROXY_LISTEN @@ -434,90 +455,35 @@ func Load(path string) (*Config, error) { // - PROXY_LOG_FORMAT // - PROXY_HEALTH_STORAGE_PROBE_INTERVAL func (c *Config) LoadFromEnv() { - if v := os.Getenv("PROXY_LISTEN"); v != "" { - c.Listen = v - } - if v := os.Getenv("PROXY_BASE_URL"); v != "" { - c.BaseURL = v - } - if v := os.Getenv("PROXY_UI_URL"); v != "" { - c.UIBaseURL = v - } - if v := os.Getenv("PROXY_STORAGE_URL"); v != "" { - c.Storage.URL = v - } - if v := os.Getenv("PROXY_STORAGE_PATH"); v != "" { - c.Storage.Path = v - } - if v := os.Getenv("PROXY_STORAGE_MAX_SIZE"); v != "" { - c.Storage.MaxSize = v - } - if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE"); v != "" { - c.Storage.DirectServe = envBool(v) - } - if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_TTL"); v != "" { - c.Storage.DirectServeTTL = v - } - if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL"); v != "" { - c.Storage.DirectServeBaseURL = v - } - if v := os.Getenv("PROXY_DATABASE_DRIVER"); v != "" { - c.Database.Driver = v - } - if v := os.Getenv("PROXY_DATABASE_PATH"); v != "" { - c.Database.Path = v - } - if v := os.Getenv("PROXY_DATABASE_URL"); v != "" { - c.Database.URL = v - } - if v := os.Getenv("PROXY_LOG_LEVEL"); v != "" { - c.Log.Level = v - } - if v := os.Getenv("PROXY_LOG_FORMAT"); v != "" { - c.Log.Format = v - } - if v := os.Getenv("PROXY_UPSTREAM_MAVEN"); v != "" { - c.Upstream.Maven = v - } - if v := os.Getenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL"); v != "" { - c.Upstream.GradlePluginPortal = v - } - if v := os.Getenv("PROXY_COOLDOWN_DEFAULT"); v != "" { - c.Cooldown.Default = v - } - if v := os.Getenv("PROXY_CACHE_METADATA"); v != "" { - c.CacheMetadata = envBool(v) - } - if v := os.Getenv("PROXY_MIRROR_API"); v != "" { - c.MirrorAPI = envBool(v) - } - if v := os.Getenv("PROXY_METADATA_TTL"); v != "" { - c.MetadataTTL = v - } - if v := os.Getenv("PROXY_METADATA_MAX_SIZE"); v != "" { - c.MetadataMaxSize = v - } - if v := os.Getenv("PROXY_HTTP_TIMEOUT"); v != "" { - c.HTTPTimeout = v - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY"); v != "" { - c.Gradle.BuildCache.ReadOnly = v == "true" || v == "1" - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE"); v != "" { - c.Gradle.BuildCache.MaxUploadSize = v - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE"); v != "" { - c.Gradle.BuildCache.MaxAge = v - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE"); v != "" { - c.Gradle.BuildCache.MaxSize = v - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL"); v != "" { - c.Gradle.BuildCache.SweepInterval = v - } - if v := os.Getenv("PROXY_HEALTH_STORAGE_PROBE_INTERVAL"); v != "" { - c.Health.StorageProbeInterval = v - } + setEnvString(&c.Listen, "PROXY_LISTEN") + setEnvString(&c.BaseURL, "PROXY_BASE_URL") + setEnvString(&c.UIBaseURL, "PROXY_UI_URL") + setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") + setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") + setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") + setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") + setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") + setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") + setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") + setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH") + setEnvString(&c.Database.URL, "PROXY_DATABASE_URL") + setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL") + setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") + setEnvString(&c.Upstream.Maven, "PROXY_UPSTREAM_MAVEN") + setEnvString(&c.Upstream.GradlePluginPortal, "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL") + setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN") + setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT") + setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") + setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") + setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") + setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") + setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY") + setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE") + setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE") + setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE") + setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL") + setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL") } // validateAbsoluteURL returns an error if value is not a parseable URL with diff --git a/internal/config/config_test.go b/internal/config/config_test.go index decc18f..ef6ac90 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -37,6 +37,9 @@ func TestDefault(t *testing.T) { if cfg.Upstream.GradlePluginPortal != "https://plugins.gradle.org/m2" { t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.gradle.org/m2") } + if cfg.Upstream.Debian != "http://deb.debian.org/debian" { + t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://deb.debian.org/debian") + } } func TestValidate(t *testing.T) { @@ -273,6 +276,7 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_LOG_LEVEL", testLevelDebug) t.Setenv("PROXY_UPSTREAM_MAVEN", "https://maven.example.com/repository/maven-public") t.Setenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", "https://plugins.example.com/m2") + t.Setenv("PROXY_UPSTREAM_DEBIAN", "http://archive.ubuntu.com/ubuntu") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h") @@ -302,6 +306,9 @@ func TestLoadFromEnv(t *testing.T) { if cfg.Upstream.GradlePluginPortal != "https://plugins.example.com/m2" { t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.example.com/m2") } + if cfg.Upstream.Debian != "http://archive.ubuntu.com/ubuntu" { + t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://archive.ubuntu.com/ubuntu") + } if !cfg.Gradle.BuildCache.ReadOnly { t.Error("Gradle.BuildCache.ReadOnly = false, want true") } diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 9a4aaab..b48f4fc 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -21,10 +21,13 @@ type DebianHandler struct { } // NewDebianHandler creates a new Debian/APT protocol handler. -func NewDebianHandler(proxy *Proxy, proxyURL string) *DebianHandler { +func NewDebianHandler(proxy *Proxy, proxyURL string, upstreamURL string) *DebianHandler { + if upstreamURL == "" { + upstreamURL = debianUpstream + } return &DebianHandler{ proxy: proxy, - upstreamURL: debianUpstream, + upstreamURL: strings.TrimSuffix(upstreamURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"), } } diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index dfdd326..60fa23a 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -18,6 +18,6 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { } func TestDebianHandler_Routes(t *testing.T) { - h := NewDebianHandler(nil, "http://localhost:8080") + h := NewDebianHandler(nil, "http://localhost:8080", "") assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd") } diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 980e234..8192eeb 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -1165,7 +1165,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) { ContentType: "application/vnd.debian.binary-package", } - h := NewDebianHandler(proxy, "http://localhost") + h := NewDebianHandler(proxy, "http://localhost", "") srv := httptest.NewServer(h.Routes()) defer srv.Close() diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index 3c2cecf..6f04113 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -17,7 +17,7 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { handler func(p *Proxy) http.Handler }{ {"debian", "/pool/main/n/nginx/nginx_1.18.0-6_amd64.deb", - func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost").Routes() }}, + func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost", "").Routes() }}, {"rpm", "/releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm", func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }}, {"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", diff --git a/internal/server/server.go b/internal/server/server.go index 153a94c..856fe2d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -223,7 +223,7 @@ func (s *Server) Start() error { cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL) juliaHandler := handler.NewJuliaHandler(proxy, s.cfg.BaseURL) containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL) - debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL) + debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Debian) rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) From 5f993e396175e842dfb63654af2675880c146932 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:59:52 +0100 Subject: [PATCH 44/70] Bump docker/login-action from 4.5.0 to 4.6.0 (#232) Bumps [docker/login-action](https://github.com/docker/login-action) from 4.5.0 to 4.6.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/06fb636fac595d6fb4b28a5dfcb21a6f5091859c...dbcb813823bdd20940b903addbd779551569679f) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c08cfa6..cba6f55 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: Log in to the Container registry - uses: docker/login-action@06fb636fac595d6fb4b28a5dfcb21a6f5091859c + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: registry: ghcr.io username: ${{ github.actor }} From bc1ea307f3550ec534966ad349dc0977124c5084 Mon Sep 17 00:00:00 2001 From: Ondrej Kokes Date: Thu, 6 Aug 2026 23:03:20 +0200 Subject: [PATCH 45/70] Multi-platform build (linux/amd64 and linux/arm64) (#230) * wip * pin to latest version, not commit * test run * Revert "test run" This reverts commit 355e8f867e1481db36327974ad94063d4be591cd. --- .github/workflows/publish.yml | 9 +++++++++ Dockerfile | 7 ++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cba6f55..e2b1084 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,6 +24,14 @@ jobs: with: persist-credentials: false + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + with: + platforms: linux/amd64,linux/arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - name: Log in to the Container registry uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: @@ -41,6 +49,7 @@ jobs: uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a with: context: . + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index c2e197f..9c51d30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.26.5-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.26.5-alpine AS builder WORKDIR /src @@ -12,8 +12,9 @@ RUN go mod download # Copy source code COPY . . -# Build the binary -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy +# Build the binary for the target platform +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy FROM alpine:3.24.1 From d0f93196a3c44352866c08d4b8ac673d5f5e0f06 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 6 Aug 2026 22:14:51 +0100 Subject: [PATCH 46/70] Bump dependencies ahead of v0.7.0 (#235) - github.com/git-pkgs/archives v0.4.0 -> v0.5.0 - github.com/git-pkgs/magic v0.1.0 -> v0.2.0 - github.com/git-pkgs/spdx v0.1.4 -> v0.3.0 - github.com/prometheus/client_golang v1.24.0 -> v1.24.1 - modernc.org/sqlite v1.55.0 -> v1.56.0 - golang.org/x/text v0.38.0 -> v0.40.0 (fixes GO-2026-5970) --- go.mod | 22 +++++++++++----------- go.sum | 57 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index e6e0a08..3449818 100644 --- a/go.mod +++ b/go.mod @@ -5,19 +5,19 @@ go 1.25.6 require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.11.0 - github.com/git-pkgs/archives v0.4.0 + github.com/git-pkgs/archives v0.5.0 github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/enrichment v0.6.4 - github.com/git-pkgs/magic v0.1.0 + github.com/git-pkgs/magic v0.2.0 github.com/git-pkgs/purl v0.1.15 github.com/git-pkgs/registries v0.6.4 - github.com/git-pkgs/spdx v0.1.4 + github.com/git-pkgs/spdx v0.3.0 github.com/git-pkgs/vers v0.3.0 github.com/git-pkgs/vulns v0.2.1 github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 - github.com/prometheus/client_golang v1.24.0 + github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 @@ -25,7 +25,7 @@ require ( golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.55.0 + modernc.org/sqlite v1.56.0 ) require ( @@ -205,7 +205,7 @@ require ( github.com/maratori/testpackage v1.1.2 // indirect github.com/matoous/godox v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mgechev/revive v1.14.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -226,7 +226,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect github.com/quasilyte/go-ruleguard v0.4.5 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect @@ -293,14 +293,14 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.53.0 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.37.0 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.38.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect @@ -309,7 +309,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect - modernc.org/libc v1.74.1 // indirect + modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect diff --git a/go.sum b/go.sum index c239240..7ae59f3 100644 --- a/go.sum +++ b/go.sum @@ -244,14 +244,14 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= -github.com/git-pkgs/archives v0.4.0 h1:KNmmIsLiSH27lUdT27EfUkQXFaLgXV5KezE81iyOIgo= -github.com/git-pkgs/archives v0.4.0/go.mod h1:tfio0OIuPKEBKHs/UCL5XBUvYmKpnvtnba2iDlfSd6g= +github.com/git-pkgs/archives v0.5.0 h1:QdowC1jTOSbEOKTYGqVt8ZjIr+yJzy7cmLNWcLPj9Y8= +github.com/git-pkgs/archives v0.5.0/go.mod h1:tfio0OIuPKEBKHs/UCL5XBUvYmKpnvtnba2iDlfSd6g= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= github.com/git-pkgs/enrichment v0.6.4 h1:mGrfenttwmcUfPXRkWpB0wBJiiGj55ltniUh66Pq4bU= github.com/git-pkgs/enrichment v0.6.4/go.mod h1:zz1vPUak/w8Jhajll0KDRN2MjKaEYeCzQTxumWnVhqY= -github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY= -github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= +github.com/git-pkgs/magic v0.2.0 h1:c7HqVxnP8c88EaVMH0/KraDFVTcmiXckRiSvNZEnvMQ= +github.com/git-pkgs/magic v0.2.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= @@ -260,8 +260,8 @@ github.com/git-pkgs/purl v0.1.15 h1:iQ3clh0Cw41rkM0rf24B7ShnN9Z+UtLMAFlNDUs+Qd4= github.com/git-pkgs/purl v0.1.15/go.mod h1:PqCLVBDeZrZgHysR803/AntMELgIr2LFZVNCcwLH2m0= github.com/git-pkgs/registries v0.6.4 h1:Kq/KlStjaQyE83UXT/tKuzCrIzc4keGeBjtroMqgoHA= github.com/git-pkgs/registries v0.6.4/go.mod h1:YkGHbxHIe2Ha/ROH6zNkS5PJUUoa9g0Ti/s2XhZnrak= -github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= -github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= +github.com/git-pkgs/spdx v0.3.0 h1:AN0guJE7vN5gbOMi9We4j1ziS4cwgFVhTvjVDGfaC7Q= +github.com/git-pkgs/spdx v0.3.0/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= github.com/git-pkgs/vulns v0.2.1 h1:tWGhOfPVDZwkM2Y9vRkMpMR+gjtlu2jhERS5JeNBoKQ= @@ -362,8 +362,8 @@ github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQE github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= -github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -424,8 +424,8 @@ github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3 github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -483,8 +483,8 @@ github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -543,12 +543,12 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= -github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= -github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= @@ -740,8 +740,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= @@ -773,8 +773,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -805,7 +805,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -825,8 +824,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -884,8 +883,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= -modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= -modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= @@ -896,8 +895,8 @@ modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= -modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -906,8 +905,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM= -modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 14f80ced3479999c86c6690e5005dfc00a398841 Mon Sep 17 00:00:00 2001 From: Philipp Garbe Date: Mon, 10 Aug 2026 10:21:49 +0200 Subject: [PATCH 47/70] fix(npm): use combined Accept header to support Artifactory upstreams (#241) When cooldown is disabled, send: Accept: application/vnd.npm.install-v1+json;q=1.0, application/json;q=0.8 This allows upstreams like JFrog Artifactory that return 406 for the abbreviated packument type to fall back to full JSON metadata, while letting the public npm registry continue to serve the smaller abbreviated format it prefers. When cooldown is enabled, keep sending only application/json because the abbreviated format omits the "time" map required for version age filtering. Fixes #228 Co-authored-by: Claude Sonnet 4.6 --- internal/handler/npm.go | 11 +++++++---- internal/handler/npm_test.go | 12 ++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index ee9b59c..78aa5e4 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -13,7 +13,7 @@ import ( const ( npmUpstream = "https://registry.npmjs.org" - npmAbbreviatedCT = "application/vnd.npm.install-v1+json" + npmAcceptDefault = "application/vnd.npm.install-v1+json;q=1.0, application/json;q=0.8" scopedParts = 2 // scope + name in scoped packages ) @@ -71,9 +71,12 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) - // Use abbreviated metadata when cooldown is disabled — it's much smaller - // (e.g. drizzle-orm: 4MB vs 92MB) but lacks the time map needed for cooldown. - accept := npmAbbreviatedCT + // Prefer the smaller abbreviated packument format but include application/json + // as a fallback so upstreams that reject the abbreviated type (e.g. JFrog + // Artifactory, which returns 406) can still respond with full metadata. + // When cooldown is enabled we must use full metadata exclusively because the + // abbreviated format omits the "time" map required for version age filtering. + accept := npmAcceptDefault if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { accept = contentTypeJSON } diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index e0257dd..fd6f080 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -396,7 +396,7 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { })) defer upstream.Close() - t.Run("no cooldown uses abbreviated metadata", func(t *testing.T) { + t.Run("no cooldown uses combined accept header", func(t *testing.T) { h := &NPMHandler{ proxy: testProxy(), upstreamURL: upstream.URL, @@ -407,12 +407,12 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { w := httptest.NewRecorder() h.handlePackageMetadata(w, req) - if gotAccept != npmAbbreviatedCT { - t.Errorf("Accept = %q, want abbreviated metadata header", gotAccept) + if gotAccept != npmAcceptDefault { + t.Errorf("Accept = %q, want %q", gotAccept, npmAcceptDefault) } }) - t.Run("cooldown enabled uses full metadata", func(t *testing.T) { + t.Run("cooldown enabled uses full metadata only", func(t *testing.T) { proxy := testProxy() proxy.Cooldown = &cooldown.Config{Default: "3d"} @@ -426,8 +426,8 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { w := httptest.NewRecorder() h.handlePackageMetadata(w, req) - if gotAccept == npmAbbreviatedCT { - t.Error("cooldown enabled should use full metadata, not abbreviated") + if gotAccept != contentTypeJSON { + t.Errorf("Accept = %q, want %q (cooldown requires full metadata)", gotAccept, contentTypeJSON) } }) } From 4fa903e01e26aecdfc0001a965c0a4598b015afd Mon Sep 17 00:00:00 2001 From: oscar-broman Date: Mon, 10 Aug 2026 12:27:03 +0400 Subject: [PATCH 48/70] Enforce cooldown on artifact downloads (#240) Cooldown filtering only ran when rewriting metadata, so a version could be missing from the npm packument and the PyPI simple index while its tarball stayed reachable. Lockfiles record artifact URLs verbatim, so npm ci and pinned pip requirements reach handleDownload without ever requesting metadata. The shared artifact path has no publish time to check against, since updateCacheDB upserts versions without PublishedAt and the column is only set by enrichment. Each handler now resolves the publish time from metadata it already fetches and returns 404 while a version is inside the window. Versions with no usable publish time are still served, as they are when filtering metadata. --- internal/handler/npm.go | 51 +++++++++++++++++++++++ internal/handler/npm_test.go | 78 +++++++++++++++++++++++++++++++++++ internal/handler/pypi.go | 22 ++++++++++ internal/handler/pypi_test.go | 56 +++++++++++++++++++++++++ 4 files changed, 207 insertions(+) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 78aa5e4..b7d96a3 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -268,6 +268,13 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("npm download request", "package", packageName, "version", version, "filename", filename) + if h.versionInCooldown(r, packageName, version) { + h.proxy.Logger.Info("cooldown: withholding npm tarball", + "package", packageName, "version", version) + JSONError(w, http.StatusNotFound, "version not found") + return + } + downloadURL := fmt.Sprintf( "%s/%s/-/%s", h.upstreamURL, @@ -290,6 +297,50 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifact(w, result) } +// versionInCooldown reports whether a version is still inside the cooldown +// window. Filtering the packument is not enough on its own: tarball URLs are +// predictable and lockfiles record them directly, so `npm ci` reaches the +// download path without ever requesting metadata. +// +// The packument is served from the metadata cache, so this normally costs no +// extra upstream request. A version with no usable publish time is allowed +// through, matching how applyCooldownFiltering treats it. +func (h *NPMHandler) versionInCooldown(r *http.Request, packageName, version string) bool { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return false + } + + upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) + + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON) + if err != nil { + h.proxy.Logger.Warn("cooldown: could not fetch npm metadata for download check", + "package", packageName, "version", version, "error", err) + return false + } + + var metadata struct { + Time map[string]string `json:"time"` + } + if err := json.Unmarshal(body, &metadata); err != nil { + h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check", + "package", packageName, "version", version, "error", err) + return false + } + + published, ok := metadata.Time[version] + if !ok { + return false + } + + publishedAt, err := time.Parse(time.RFC3339, published) + if err != nil { + return false + } + + return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt) +} + func escapeNPMDownloadPackage(packageName string) string { scope, name, scoped := strings.Cut(packageName, "/") if scoped && strings.HasPrefix(scope, "@") && len(scope) > 1 && name != "" && !strings.Contains(name, "/") { diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index fd6f080..07da9c3 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -454,3 +454,81 @@ func TestNPMHandlerMetadataNotFound(t *testing.T) { t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound) } } + +func TestNPMDownloadCooldown(t *testing.T) { + now := time.Now() + packument := `{ + "name": "leftpad", + "dist-tags": {"latest": "2.0.0"}, + "time": { + "1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `", + "2.0.0": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `" + }, + "versions": {"1.0.0": {}, "2.0.0": {}} + }` + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = io.WriteString(w, packument) + })) + defer upstream.Close() + + tests := []struct { + name string + version string + wantStatus int + }{ + {"published before the window serves the tarball", testVersion100, http.StatusOK}, + {"published inside the window is withheld", "2.0.0", http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("tarball data")), + ContentType: "application/octet-stream", + } + + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != tt.wantStatus { + t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { + t.Error("fetched a version that is still inside the cooldown window") + } + }) + } +} + +func TestNPMDownloadCooldownDisabled(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Error("metadata must not be fetched when cooldown is disabled") + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("tarball data")), + ContentType: "application/octet-stream", + } + + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + if h.versionInCooldown(httptest.NewRequest(http.MethodGet, "/", nil), "leftpad", testVersion100) { + t.Error("versionInCooldown = true, want false when cooldown is not configured") + } +} diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 713f6c8..7875cdd 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -311,6 +311,21 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) } +// versionInCooldown reports whether a version is still inside the cooldown +// window. Filtering the simple index is not enough on its own: file URLs are +// recorded in lockfiles and requirements pins, so pip can reach the download +// path without ever reading the index. +// +// A release whose upload time cannot be determined is allowed through, matching +// how fetchFilteredVersions treats it. +func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) bool { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return false + } + + return h.fetchFilteredVersions(r, name)[version] +} + // rewriteFileEntries rewrites URLs in a list of file entries. func (h *PyPIHandler) rewriteFileEntries(files any) { filesArr, ok := files.([]any) @@ -417,6 +432,13 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { filename := parts[len(parts)-1] name, version := h.parseFilename(filename) + if name != "" && h.versionInCooldown(r, name, version) { + h.proxy.Logger.Info("cooldown: withholding pypi file", + "name", name, "version", version, "filename", filename) + http.Error(w, "not found", http.StatusNotFound) + return + } + if name == "" { // Can't determine name/version, use hash as identifier name = fmt.Sprintf("_hash_%s", hashPath(path)) diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index 5ae76ca..6bbcf3e 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -236,3 +236,59 @@ func TestPyPIHandler_DownloadCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } } + +func TestPyPIDownloadCooldown(t *testing.T) { + now := time.Now() + releases := `{"releases": { + "1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}], + "2.0.0": [{"upload_time_iso_8601": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"}] + }}` + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = io.WriteString(w, releases) + })) + defer upstream.Close() + + tests := []struct { + name string + filename string + wantStatus int + }{ + {"published before the window serves the file", "newpkg-1.0.0.tar.gz", http.StatusOK}, + {"published inside the window is withheld", "newpkg-2.0.0.tar.gz", http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("sdist data")), + ContentType: "application/octet-stream", + } + + h := &PyPIHandler{ + proxy: proxy, + upstreamURL: upstream.URL, + proxyURL: "http://localhost", + } + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/packages/packages/ab/cd/ef0123456789/" + tt.filename) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != tt.wantStatus { + t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { + t.Error("fetched a version that is still inside the cooldown window") + } + }) + } +} From 30e40526154eb8fb23f8d3b4e810967b5155c89d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 10 Aug 2026 09:34:56 +0100 Subject: [PATCH 49/70] Document upstream.debian in README and config.example.yaml (#237) Follow-up to #229 which added the config key and env var but didn't touch docs. --- README.md | 7 +++++++ config.example.yaml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index 4609b66..01012e2 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,13 @@ Replace your existing sources.list entries, then: sudo apt update ``` +The upstream defaults to `http://deb.debian.org/debian`. To proxy a different APT repository (e.g. Ubuntu), set `upstream.debian` in the config file or `PROXY_UPSTREAM_DEBIAN` in the environment: + +```yaml +upstream: + debian: "http://archive.ubuntu.com/ubuntu" +``` + ### RPM / Yum / DNF Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`: diff --git a/config.example.yaml b/config.example.yaml index 1176f5b..7cf7bb2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -95,6 +95,9 @@ upstream: # Cargo crate download URL cargo_download: "https://static.crates.io/crates" + # Debian/APT repository URL (used by /debian endpoint) + debian: "http://deb.debian.org/debian" + # Authentication for upstream registries # Keys are URL prefixes matched against request URLs. # Values can reference environment variables using ${VAR_NAME} syntax. From a17bdc7c898c85c2f361b3b9aee15c705a30ef2b Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 10 Aug 2026 16:42:52 +0100 Subject: [PATCH 50/70] Sign published container images (#227) * Sign published container images * Attest per-platform SPDX SBOMs with cosign Extract each platform's SPDX document from the BuildKit SBOM attestation and sign it as a cosign spdxjson attestation against the manifest-list digest, so downstream consumers (e.g. Kyverno image-verification policies) can verify the predicate signature rather than relying on the unsigned BuildKit attachment. --- .github/workflows/publish.yml | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e2b1084..1c4eb1e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,6 +18,7 @@ jobs: permissions: packages: write contents: read + id-token: write steps: - name: Check out the repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -45,7 +46,10 @@ jobs: with: images: ghcr.io/${{ github.repository }} + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - name: Build and push Docker image + id: build uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a with: context: . @@ -53,3 +57,42 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + provenance: mode=max + sbom: true + + - name: Sign image by digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + cosign sign --yes "${IMAGE}@${DIGEST}" + + - name: Verify BuildKit attestations and extract SPDX predicates + env: + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + reference="${IMAGE}@${DIGEST}" + docker buildx imagetools inspect "$reference" --format '{{ json .Provenance }}' > provenance.json + docker buildx imagetools inspect "$reference" --format '{{ json .SBOM }}' > sbom.json + + for platform in linux/amd64 linux/arm64; do + jq -e --arg p "$platform" '.[$p].SLSA | type == "object" and length > 0' \ + provenance.json >/dev/null + jq -e --arg p "$platform" '.[$p].SPDX' sbom.json > "sbom-${platform//\//-}.spdx.json" + done + + - name: Attest platform SBOMs by digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + reference="${IMAGE}@${DIGEST}" + for predicate in sbom-linux-amd64.spdx.json sbom-linux-arm64.spdx.json; do + cosign attest --yes --type spdxjson --predicate "$predicate" "$reference" + done From bbea63f04640572a68bd531b932c367dfb3b89d1 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 13 Aug 2026 07:08:23 +0100 Subject: [PATCH 51/70] fix(upstream): apply authentication through shared transport (#198) * upstream: apply authentication through shared transport * Address upstream authentication review findings --- config.example.yaml | 3 +- docs/architecture.md | 2 + docs/configuration.md | 6 +- internal/config/config.go | 75 ++++- internal/config/config_test.go | 71 +++++ internal/httpclient/transport.go | 433 ++++++++++++++++++++++++++ internal/httpclient/transport_test.go | 251 +++++++++++++++ internal/server/server.go | 32 +- 8 files changed, 852 insertions(+), 21 deletions(-) create mode 100644 internal/httpclient/transport.go create mode 100644 internal/httpclient/transport_test.go diff --git a/config.example.yaml b/config.example.yaml index 7cf7bb2..7ada017 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -99,7 +99,8 @@ upstream: debian: "http://deb.debian.org/debian" # Authentication for upstream registries - # Keys are URL prefixes matched against request URLs. + # Keys are absolute URL scopes. Scheme, host, effective port, and path + # segment boundaries must match; the longest matching scope wins. # Values can reference environment variables using ${VAR_NAME} syntax. # # Supported auth types: diff --git a/docs/architecture.md b/docs/architecture.md index f04d548..cf8b0e2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -240,6 +240,8 @@ Fetches artifacts from upstream registries. - Exponential backoff retry on 429 (rate limit) and 5xx errors - Returns streaming reader (doesn't load into memory) - Configurable user-agent +- Shares an authentication-aware transport with metadata requests so URL-scoped credentials apply consistently +- Discovers and caches scoped OCI Bearer tokens from registry challenges **Resolver:** - Determines download URL for a package/version diff --git a/docs/configuration.md b/docs/configuration.md index 8998ac2..fa0f576 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -123,7 +123,9 @@ upstream: ## Authentication -Configure authentication for private upstream registries. Auth is matched by URL prefix, and credentials can reference environment variables using `${VAR_NAME}` syntax. +Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. + +OCI registries that return a Bearer challenge from a `/v2/{repository}/…` endpoint are handled automatically. The proxy discovers the token realm from `WWW-Authenticate`, applies any configured credentials for the token URL, and reuses the scoped token until shortly before it expires. ### Bearer Token @@ -172,7 +174,7 @@ upstream: ### URL Matching -Auth configs are matched by URL prefix. The longest matching prefix wins, so you can configure different credentials for different paths: +Auth keys must be absolute URLs. Matching compares the scheme, host, effective port, and path-segment prefix, preventing credentials for `registry.example.com` from being sent to a lookalike host such as `registry.example.com.evil.test`. The longest matching scope wins, so you can configure different credentials for different paths: ```yaml upstream: diff --git a/internal/config/config.go b/internal/config/config.go index 3dabc62..31f8c26 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -310,23 +310,29 @@ type UpstreamConfig struct { Debian string `json:"debian" yaml:"debian"` // Auth configures authentication for upstream registries. - // Keys are URL prefixes that are matched against request URLs. + // Keys are absolute URL scopes matched by scheme, host, effective port, + // and path-segment prefix. // Example: "https://npm.pkg.github.com" matches all requests to that host. Auth map[string]AuthConfig `json:"auth" yaml:"auth"` } // AuthForURL returns the auth config that matches the given URL. -// Matches are based on URL prefix - the longest matching prefix wins. +// The longest matching URL scope wins. func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig { if u.Auth == nil { return nil } + target, err := parseAuthURL(url) + if err != nil { + return nil + } var bestMatch *AuthConfig var bestLen int for pattern, auth := range u.Auth { - if strings.HasPrefix(url, pattern) && len(pattern) > bestLen { + configured, err := parseAuthURL(pattern) + if err == nil && authURLMatches(configured, target) && len(pattern) > bestLen { a := auth // copy to avoid loop variable capture bestMatch = &a bestLen = len(pattern) @@ -336,6 +342,55 @@ func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig { return bestMatch } +// Validate checks upstream authentication URL scopes. +func (u *UpstreamConfig) Validate() error { + for pattern := range u.Auth { + if _, err := parseAuthURL(pattern); err != nil { + return fmt.Errorf("invalid upstream.auth URL %q: %w", pattern, err) + } + } + return nil +} + +func parseAuthURL(value string) (*url.URL, error) { + parsed, err := url.Parse(value) + if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" { + return nil, fmt.Errorf("invalid authentication URL") + } + return parsed, nil +} + +func authURLMatches(configured, target *url.URL) bool { + if !strings.EqualFold(configured.Scheme, target.Scheme) || + !strings.EqualFold(configured.Hostname(), target.Hostname()) || + authURLPort(configured) != authURLPort(target) { + return false + } + if configured.RawQuery != "" && configured.RawQuery != target.RawQuery { + return false + } + + configuredPath := strings.TrimSuffix(configured.EscapedPath(), "/") + if configuredPath == "" { + return true + } + targetPath := strings.TrimSuffix(target.EscapedPath(), "/") + return targetPath == configuredPath || strings.HasPrefix(targetPath, configuredPath+"/") +} + +func authURLPort(value *url.URL) string { + if port := value.Port(); port != "" { + return port + } + if strings.EqualFold(value.Scheme, "https") { + return "443" + } + if strings.EqualFold(value.Scheme, "http") { + return "80" + } + return "" +} + // AuthConfig configures authentication for an upstream registry. type AuthConfig struct { // Type is the authentication type: "bearer", "basic", or "header". @@ -577,15 +632,19 @@ func (c *Config) Validate() error { return err } + return c.validateComponents() +} + +func (c *Config) validateComponents() error { + if err := c.Upstream.Validate(); err != nil { + return err + } + if err := c.Health.Validate(); err != nil { return err } - if err := c.Gradle.BuildCache.Validate(); err != nil { - return err - } - - return nil + return c.Gradle.BuildCache.Validate() } // Validate checks the /health configuration. An unset interval is allowed diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ef6ac90..e4677c3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strings" "testing" "time" ) @@ -795,3 +796,73 @@ func TestDatabaseConfigString(t *testing.T) { } } } + +func TestUpstreamAuthForURLMatchesURLComponents(t *testing.T) { + registryAuth := AuthConfig{Type: "bearer", Token: "registry-token"} + privateAuth := AuthConfig{Type: "bearer", Token: "private-token"} + config := UpstreamConfig{Auth: map[string]AuthConfig{ + "https://registry.example.com": registryAuth, + "https://registry.example.com/private": privateAuth, + }} + + tests := []struct { + name string + url string + wantToken string + }{ + {name: "registry root", url: "https://registry.example.com/package", wantToken: "registry-token"}, + {name: "host is case insensitive", url: "https://REGISTRY.EXAMPLE.COM/package", wantToken: "registry-token"}, + {name: "longest path match", url: "https://registry.example.com/private/package", wantToken: "private-token"}, + {name: "exact path match", url: "https://registry.example.com/private", wantToken: "private-token"}, + {name: "path segment boundary", url: "https://registry.example.com/private-other/package", wantToken: "registry-token"}, + {name: "lookalike host rejected", url: "https://registry.example.com.evil.test/package"}, + {name: "different scheme rejected", url: "http://registry.example.com/package"}, + {name: "different port rejected", url: "https://registry.example.com:8443/package"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + auth := config.AuthForURL(tt.url) + if tt.wantToken == "" { + if auth != nil { + t.Fatalf("AuthForURL() = %+v, want nil", auth) + } + return + } + if auth == nil { + t.Fatal("AuthForURL() = nil, want authentication") + } + if auth.Token != tt.wantToken { + t.Errorf("token = %q, want %q", auth.Token, tt.wantToken) + } + }) + } +} + +func TestValidateUpstreamAuthURLs(t *testing.T) { + t.Run("valid absolute URL", func(t *testing.T) { + cfg := Default() + cfg.Upstream.Auth = map[string]AuthConfig{ + "https://registry.example.com/private": {Type: "bearer", Token: "token"}, + } + + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error = %v", err) + } + }) + + t.Run("invalid URL", func(t *testing.T) { + cfg := Default() + cfg.Upstream.Auth = map[string]AuthConfig{ + "registry.example.com": {Type: "bearer", Token: "token"}, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("Validate() error = nil, want invalid upstream.auth URL error") + } + if !strings.Contains(err.Error(), "upstream.auth") || !strings.Contains(err.Error(), "registry.example.com") { + t.Errorf("Validate() error = %q, want field and URL", err) + } + }) +} diff --git a/internal/httpclient/transport.go b/internal/httpclient/transport.go new file mode 100644 index 0000000..d96c827 --- /dev/null +++ b/internal/httpclient/transport.go @@ -0,0 +1,433 @@ +// Package httpclient provides authentication-aware HTTP transports for upstream requests. +package httpclient + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +const ( + defaultTokenLifetime = 60 * time.Second + tokenExpirySkew = 5 * time.Second + maxTokenResponseSize = 1 << 20 + shortTokenSkewDivisor = 10 +) + +// AuthFunc returns a configured authentication header for a URL. +type AuthFunc func(url string) (headerName, headerValue string) + +// Transport adds configured authentication and follows OCI Bearer challenges. +type Transport struct { + base http.RoundTripper + authForURL AuthFunc + + mu sync.Mutex + tokens map[string]cachedToken + challenges map[string]bearerChallenge +} + +type cachedToken struct { + value string + expiresAt time.Time +} + +type bearerChallenge struct { + realm string + service string + scopes []string +} + +type tokenResponse struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + ExpiresIn int64 `json:"expires_in"` + IssuedAt string `json:"issued_at"` +} + +// NewTransport creates an authentication-aware transport around base. +func NewTransport(base http.RoundTripper, authForURL AuthFunc) *Transport { + if base == nil { + base = http.DefaultTransport + } + return &Transport{ + base: base, + authForURL: authForURL, + tokens: make(map[string]cachedToken), + challenges: make(map[string]bearerChallenge), + } +} + +// RoundTrip implements http.RoundTripper. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + hasExplicitAuthorization := req.Header.Get("Authorization") != "" + outbound := cloneRequest(req) + t.applyAuthentication(outbound, hasExplicitAuthorization) + + resp, err := t.base.RoundTrip(outbound) + if err != nil || resp.StatusCode != http.StatusUnauthorized { + return resp, err + } + if hasExplicitAuthorization { + return resp, nil + } + if registryProtectionSpace(req.URL) == "" { + return resp, nil + } + + challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate")) + if !ok || !canReplay(req) { + return resp, nil + } + + drainAndClose(resp.Body) + token, err := t.token(req.Context(), challenge) + if err != nil { + return nil, fmt.Errorf("registry authentication: %w", err) + } + t.rememberChallenge(req.URL, challenge) + + retry, err := cloneRequestForRetry(req) + if err != nil { + return nil, err + } + t.applyConfiguredAuthentication(retry) + retry.Header.Set("Authorization", "Bearer "+token) + return t.base.RoundTrip(retry) +} + +func (t *Transport) applyAuthentication(req *http.Request, hasExplicitAuthorization bool) { + t.applyConfiguredAuthentication(req) + if hasExplicitAuthorization { + return + } + if token := t.cachedTokenForRequest(req.URL); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } +} + +func (t *Transport) applyConfiguredAuthentication(req *http.Request) { + if t.authForURL == nil { + return + } + name, value := t.authForURL(req.URL.String()) + if name != "" && value != "" && req.Header.Get(name) == "" { + req.Header.Set(name, value) + } +} + +func (t *Transport) token(ctx context.Context, challenge bearerChallenge) (string, error) { + key := challenge.key() + if token := t.cachedToken(key); token != "" { + return token, nil + } + + token, expiresAt, err := t.fetchToken(ctx, challenge) + if err != nil { + return "", err + } + + t.cacheToken(key, cachedToken{value: token, expiresAt: expiresAt}) + return token, nil +} + +func (t *Transport) cacheToken(key string, token cachedToken) { + now := time.Now() + t.mu.Lock() + defer t.mu.Unlock() + + for cachedKey, cached := range t.tokens { + if !now.Before(cached.expiresAt) { + delete(t.tokens, cachedKey) + } + } + t.tokens[key] = token +} + +func (t *Transport) fetchToken(ctx context.Context, challenge bearerChallenge) (string, time.Time, error) { + tokenURL, err := url.Parse(challenge.realm) + if err != nil || !tokenURL.IsAbs() || (tokenURL.Scheme != "https" && tokenURL.Scheme != "http") { + return "", time.Time{}, fmt.Errorf("invalid token realm %q", challenge.realm) + } + + query := tokenURL.Query() + if challenge.service != "" { + query.Set("service", challenge.service) + } + for _, scope := range challenge.scopes { + query.Add("scope", scope) + } + query.Set("client_id", "git-pkgs-proxy") + tokenURL.RawQuery = query.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL.String(), nil) + if err != nil { + return "", time.Time{}, err + } + + client := &http.Client{Transport: configuredTransport{parent: t}} + resp, err := client.Do(req) + if err != nil { + return "", time.Time{}, fmt.Errorf("requesting token: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTokenResponseSize)) + return "", time.Time{}, fmt.Errorf("token service returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var payload tokenResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, maxTokenResponseSize)).Decode(&payload); err != nil { + return "", time.Time{}, fmt.Errorf("decoding token response: %w", err) + } + token := payload.Token + if token == "" { + token = payload.AccessToken + } + if token == "" { + return "", time.Time{}, fmt.Errorf("token response did not contain a token") + } + + issuedAt := time.Now() + if payload.IssuedAt != "" { + if parsed, parseErr := time.Parse(time.RFC3339, payload.IssuedAt); parseErr == nil { + issuedAt = parsed + } + } + lifetime := time.Duration(payload.ExpiresIn) * time.Second + if lifetime <= 0 { + lifetime = defaultTokenLifetime + } + expiresAt := issuedAt.Add(lifetime).Add(-expirySkew(lifetime)) + return token, expiresAt, nil +} + +type configuredTransport struct { + parent *Transport +} + +func (t configuredTransport) RoundTrip(req *http.Request) (*http.Response, error) { + outbound := cloneRequest(req) + t.parent.applyConfiguredAuthentication(outbound) + return t.parent.base.RoundTrip(outbound) +} + +func (t *Transport) cachedTokenForRequest(requestURL *url.URL) string { + space := registryProtectionSpace(requestURL) + if space == "" { + return "" + } + + t.mu.Lock() + challenge, ok := t.challenges[space] + t.mu.Unlock() + if !ok { + return "" + } + return t.cachedToken(challenge.key()) +} + +func (t *Transport) cachedToken(key string) string { + now := time.Now() + t.mu.Lock() + defer t.mu.Unlock() + + token, ok := t.tokens[key] + if !ok { + return "" + } + if !now.Before(token.expiresAt) { + delete(t.tokens, key) + return "" + } + return token.value +} + +func (t *Transport) rememberChallenge(requestURL *url.URL, challenge bearerChallenge) { + space := registryProtectionSpace(requestURL) + if space == "" { + return + } + t.mu.Lock() + t.challenges[space] = challenge + t.mu.Unlock() +} + +func (c bearerChallenge) key() string { + return c.realm + "\x00" + c.service + "\x00" + strings.Join(c.scopes, "\x00") +} + +func registryProtectionSpace(u *url.URL) string { + const registryPrefix = "/v2/" + if u == nil || !strings.HasPrefix(u.Path, registryPrefix) { + return "" + } + rest := strings.TrimPrefix(u.Path, registryPrefix) + end := len(rest) + for _, marker := range []string{"/blobs/", "/manifests/", "/tags/", "/referrers/"} { + if index := strings.Index(rest, marker); index >= 0 && index < end { + end = index + } + } + if end == len(rest) || end == 0 { + return "" + } + return u.Scheme + "://" + u.Host + registryPrefix + rest[:end] +} + +func parseBearerChallenge(values []string) (bearerChallenge, bool) { + for _, value := range values { + params, ok := bearerParameters(value) + if !ok || params["realm"] == "" { + continue + } + challenge := bearerChallenge{ + realm: params["realm"], + service: params["service"], + } + if scope := params["scope"]; scope != "" { + challenge.scopes = append(challenge.scopes, scope) + } + return challenge, true + } + return bearerChallenge{}, false +} + +func bearerParameters(value string) (map[string]string, bool) { + start := findAuthScheme(value, "Bearer") + if start < 0 { + return nil, false + } + rest := value[start+len("Bearer"):] + params := make(map[string]string) + for { + rest = strings.TrimLeft(rest, " \t,") + if rest == "" { + break + } + + keyEnd := strings.IndexAny(rest, "= \t,") + if keyEnd <= 0 { + break + } + key := strings.ToLower(rest[:keyEnd]) + rest = strings.TrimLeft(rest[keyEnd:], " \t") + if rest == "" || rest[0] != '=' { + break + } + rest = strings.TrimLeft(rest[1:], " \t") + + parsed, remaining, ok := parseAuthValue(rest) + if !ok { + return nil, false + } + params[key] = parsed + rest = remaining + } + return params, true +} + +func findAuthScheme(value, scheme string) int { + inQuote := false + escaped := false + for index := 0; index+len(scheme) <= len(value); index++ { + char := value[index] + if escaped { + escaped = false + continue + } + if char == '\\' && inQuote { + escaped = true + continue + } + if char == '"' { + inQuote = !inQuote + continue + } + if inQuote || !strings.EqualFold(value[index:index+len(scheme)], scheme) { + continue + } + beforeOK := index == 0 || value[index-1] == ',' || value[index-1] == ' ' || value[index-1] == '\t' + after := index + len(scheme) + afterOK := after < len(value) && (value[after] == ' ' || value[after] == '\t') + if beforeOK && afterOK { + return index + } + } + return -1 +} + +func parseAuthValue(value string) (parsed, remaining string, ok bool) { + if value == "" { + return "", "", false + } + if value[0] != '"' { + end := strings.IndexAny(value, " \t,") + if end < 0 { + return value, "", true + } + return value[:end], value[end:], end > 0 + } + + var builder strings.Builder + escaped := false + for index := 1; index < len(value); index++ { + char := value[index] + if escaped { + builder.WriteByte(char) + escaped = false + continue + } + if char == '\\' { + escaped = true + continue + } + if char == '"' { + return builder.String(), value[index+1:], true + } + builder.WriteByte(char) + } + return "", "", false +} + +func cloneRequest(req *http.Request) *http.Request { + clone := req.Clone(req.Context()) + clone.Header = req.Header.Clone() + return clone +} + +func canReplay(req *http.Request) bool { + return req.Body == nil || req.GetBody != nil +} + +func cloneRequestForRetry(req *http.Request) (*http.Request, error) { + clone := cloneRequest(req) + if req.Body == nil { + return clone, nil + } + body, err := req.GetBody() + if err != nil { + return nil, fmt.Errorf("replaying authenticated request: %w", err) + } + clone.Body = body + return clone, nil +} + +func expirySkew(lifetime time.Duration) time.Duration { + if lifetime < tokenExpirySkew*2 { + return lifetime / shortTokenSkewDivisor + } + return tokenExpirySkew +} + +func drainAndClose(body io.ReadCloser) { + _, _ = io.Copy(io.Discard, io.LimitReader(body, maxTokenResponseSize)) + _ = body.Close() +} diff --git a/internal/httpclient/transport_test.go b/internal/httpclient/transport_test.go new file mode 100644 index 0000000..bd1f266 --- /dev/null +++ b/internal/httpclient/transport_test.go @@ -0,0 +1,251 @@ +package httpclient + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestTransportFollowsBearerChallengeAndCachesToken(t *testing.T) { + var registryRequests int + var tokenRequests int + var server *httptest.Server + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + tokenRequests++ + if got := r.URL.Query().Get("service"); got != "registry.test" { + t.Errorf("service = %q, want %q", got, "registry.test") + } + if got := r.URL.Query().Get("scope"); got != "repository:library/test:pull" { + t.Errorf("scope = %q, want %q", got, "repository:library/test:pull") + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"token":"registry-token","expires_in":3600}`) + case "/v2/library/test/blobs/sha256:first", "/v2/library/test/blobs/sha256:second": + registryRequests++ + if r.Header.Get("Authorization") != "Bearer registry-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token",service="registry.test",scope="repository:library/test:pull"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + return + } + _, _ = io.WriteString(w, "blob") + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} + for _, digest := range []string{"sha256:first", "sha256:second"} { + resp, err := client.Get(server.URL + "/v2/library/test/blobs/" + digest) + if err != nil { + t.Fatalf("GET %s: %v", digest, err) + } + body, readErr := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if readErr != nil { + t.Fatalf("read %s response: %v", digest, readErr) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s status = %d, want %d", digest, resp.StatusCode, http.StatusOK) + } + if string(body) != "blob" { + t.Errorf("GET %s body = %q, want %q", digest, body, "blob") + } + } + + if tokenRequests != 1 { + t.Errorf("token requests = %d, want 1", tokenRequests) + } + if registryRequests != 3 { + t.Errorf("registry requests = %d, want 3", registryRequests) + } +} + +func TestTransportAddsConfiguredAuthentication(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Registry-Token"); got != "configured-token" { + t.Errorf("X-Registry-Token = %q, want %q", got, "configured-token") + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + authForURL := func(url string) (string, string) { + if strings.HasPrefix(url, server.URL) { + return "X-Registry-Token", "configured-token" + } + return "", "" + } + client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)} + + resp, err := client.Get(server.URL + "/metadata") + if err != nil { + t.Fatalf("GET metadata: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent) + } +} + +func TestTransportPreservesExplicitAuthentication(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" { + t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token") + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + authForURL := func(string) (string, string) { + return "Authorization", "Bearer configured-token" + } + client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)} + req, err := http.NewRequest(http.MethodGet, server.URL+"/artifact", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer explicit-token") + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("GET artifact: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent) + } +} + +func TestTransportDoesNotReplaceExplicitAuthenticationAfterBearerChallenge(t *testing.T) { + var registryRequests int + var tokenRequests int + var server *httptest.Server + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + tokenRequests++ + _, _ = io.WriteString(w, `{"token":"registry-token"}`) + case "/v2/library/test/blobs/sha256:test": + registryRequests++ + if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" { + t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token") + } + w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} + req, err := http.NewRequest(http.MethodGet, server.URL+"/v2/library/test/blobs/sha256:test", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer explicit-token") + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("GET blob: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized) + } + if registryRequests != 1 { + t.Errorf("registry requests = %d, want 1", registryRequests) + } + if tokenRequests != 0 { + t.Errorf("token requests = %d, want 0", tokenRequests) + } +} + +func TestTransportDoesNotForwardConfiguredAuthenticationOnTokenRedirect(t *testing.T) { + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Registry-Token"); got != "" { + t.Errorf("redirected X-Registry-Token = %q, want empty", got) + } + _, _ = io.WriteString(w, `{"token":"registry-token"}`) + })) + defer destination.Close() + + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Registry-Token"); got != "configured-token" { + t.Errorf("source X-Registry-Token = %q, want %q", got, "configured-token") + } + http.Redirect(w, r, destination.URL+"/token", http.StatusFound) + })) + defer source.Close() + + authForURL := func(rawURL string) (string, string) { + if strings.HasPrefix(rawURL, source.URL) { + return "X-Registry-Token", "configured-token" + } + return "", "" + } + transport := NewTransport(http.DefaultTransport, authForURL) + token, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: source.URL + "/token"}) + if err != nil { + t.Fatalf("fetchToken: %v", err) + } + if token != "registry-token" { + t.Errorf("token = %q, want %q", token, "registry-token") + } +} + +func TestTransportPrunesExpiredTokens(t *testing.T) { + transport := NewTransport(http.DefaultTransport, nil) + transport.tokens["expired-unused"] = cachedToken{ + value: "expired-token", + expiresAt: time.Now().Add(-time.Minute), + } + transport.cacheToken("current", cachedToken{ + value: "current-token", + expiresAt: time.Now().Add(time.Minute), + }) + + if got := transport.cachedToken("current"); got != "current-token" { + t.Errorf("cachedToken(current) = %q, want %q", got, "current-token") + } + if _, ok := transport.tokens["expired-unused"]; ok { + t.Error("expired unused token was not pruned") + } +} + +func TestTransportDoesNotFollowBearerChallengeOutsideOCIRegistry(t *testing.T) { + tokenRequests := 0 + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + tokenRequests++ + _, _ = io.WriteString(w, `{"token":"unexpected"}`) + return + } + w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + })) + defer server.Close() + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} + resp, err := client.Get(server.URL + "/api/packages") + if err != nil { + t.Fatalf("GET API: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized) + } + if tokenRequests != 0 { + t.Errorf("token requests = %d, want 0", tokenRequests) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 856fe2d..c98d1cb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -58,17 +58,19 @@ import ( "strings" "time" + "github.com/git-pkgs/cooldown" swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/enrichment" "github.com/git-pkgs/proxy/internal/handler" + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" + "github.com/git-pkgs/registries/safehttp" "github.com/git-pkgs/spdx" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -84,12 +86,12 @@ const ( // Server is the main proxy server. type Server struct { - cfg *config.Config - db *database.DB - storage storage.Storage - logger *slog.Logger - http *http.Server - templates *Templates + cfg *config.Config + db *database.DB + storage storage.Storage + logger *slog.Logger + http *http.Server + templates *Templates cancel context.CancelFunc healthCache *healthCache } @@ -156,8 +158,18 @@ func New(cfg *config.Config, logger *slog.Logger) (*Server, error) { // Start starts the HTTP server. func (s *Server) Start() error { - // Create shared components with circuit breaker - baseFetcher := fetch.NewFetcher(fetch.WithAuthFunc(s.authForURL)) + // Use one authentication-aware transport for metadata and artifacts so + // configured credentials and cached OCI challenges apply consistently. + safeClient := safehttp.New(nil, safehttp.Options{}) + authTransport := upstreamhttp.NewTransport(safeClient.Transport, upstreamhttp.AuthFunc(s.authForURL)) + metadataClient := *safeClient + metadataClient.Timeout = s.cfg.ParseHTTPTimeout() + metadataClient.Transport = authTransport + artifactClient := metadataClient + artifactClient.Timeout = serverWriteTimeout + + // Create shared components with circuit breaker. + baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(&artifactClient)) fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher) resolver := fetch.NewResolver() cd := &cooldown.Config{ @@ -166,7 +178,7 @@ func (s *Server) Start() error { Packages: s.cfg.Cooldown.NormalizedPackages(), } proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) - proxy.HTTPClient.Timeout = s.cfg.ParseHTTPTimeout() + proxy.HTTPClient = &metadataClient proxy.AuthForURL = s.authForURL proxy.Cooldown = cd proxy.CacheMetadata = s.cfg.CacheMetadata From 538a15d9f8950e401e7de09479b31a36ef141c52 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 13 Aug 2026 07:35:07 +0100 Subject: [PATCH 52/70] fix(container): serve cached images when upstream is unavailable (#199) * container: cache manifests for offline pulls * Preserve direct-serve redirects for blob HEAD requests --- docs/architecture.md | 1 + docs/configuration.md | 2 + internal/database/metadata_cache_test.go | 55 +- internal/database/queries.go | 14 +- internal/database/schema.go | 19 + internal/database/types.go | 23 +- internal/handler/container.go | 131 +---- internal/handler/container_manifest.go | 251 +++++++++ internal/handler/container_test.go | 560 ++++++++++++++++--- internal/handler/handler.go | 39 +- internal/handler/handler_test.go | 6 +- internal/handler/notfound_ecosystems_test.go | 16 +- 12 files changed, 888 insertions(+), 229 deletions(-) create mode 100644 internal/handler/container_manifest.go diff --git a/docs/architecture.md b/docs/architecture.md index cf8b0e2..6d9bfda 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -353,6 +353,7 @@ Eviction can be implemented as: - Fresh data - new versions visible immediately - Metadata is small, upstream fetch is fast - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table +- OCI manifests are the exception: they are cached automatically so previously fetched images remain pullable when the registry or token service is unavailable **Why stream artifacts?** - Memory efficient - don't load large files into RAM diff --git a/docs/configuration.md b/docs/configuration.md index fa0f576..1e5a1c7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -248,6 +248,8 @@ Note: Hex cooldown requires disabling registry signature verification since the By default the proxy fetches metadata fresh from upstream on every request. Enable `cache_metadata` to store metadata responses in the database and storage backend for offline fallback. When upstream is unreachable, the proxy serves the last cached copy. ETag-based revalidation avoids re-downloading unchanged metadata. +OCI manifests are always cached because cached image blobs cannot be pulled without their manifests. Digest-addressed manifests are immutable and served directly from cache. Tag-addressed manifests follow `metadata_ttl`, revalidate when stale, and fall back to the last cached response when the registry is unavailable. + ```yaml cache_metadata: true ``` diff --git a/internal/database/metadata_cache_test.go b/internal/database/metadata_cache_test.go index 5701816..09dcba3 100644 --- a/internal/database/metadata_cache_test.go +++ b/internal/database/metadata_cache_test.go @@ -30,8 +30,12 @@ func TestUpsertAndGetMetadataCache(t *testing.T) { StoragePath: "_metadata/npm/lodash/metadata", ETag: sql.NullString{String: `"abc123"`, Valid: true}, ContentType: sql.NullString{String: "application/json", Valid: true}, - Size: sql.NullInt64{Int64: 1024, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + ContentDigest: sql.NullString{ + String: "sha256:0123456789abcdef", + Valid: true, + }, + Size: sql.NullInt64{Int64: 1024, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, } err := db.UpsertMetadataCache(entry) @@ -62,6 +66,9 @@ func TestUpsertAndGetMetadataCache(t *testing.T) { if !got.ContentType.Valid || got.ContentType.String != "application/json" { t.Errorf("content_type = %v, want %q", got.ContentType, "application/json") } + if !got.ContentDigest.Valid || got.ContentDigest.String != "sha256:0123456789abcdef" { + t.Errorf("content_digest = %v, want %q", got.ContentDigest, "sha256:0123456789abcdef") + } if !got.Size.Valid || got.Size.Int64 != 1024 { t.Errorf("size = %v, want 1024", got.Size) } @@ -178,3 +185,47 @@ func TestMetadataCacheTableCreatedByMigration(t *testing.T) { t.Error("metadata_cache table should exist after migration") } } + +func TestMetadataCacheContentDigestMigrationPreservesExistingRows(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + db, err := Create(dbPath) + if err != nil { + t.Fatalf("Create failed: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.Exec("ALTER TABLE metadata_cache DROP COLUMN content_digest"); err != nil { + t.Fatalf("dropping content_digest: %v", err) + } + if _, err := db.Exec("DELETE FROM migrations WHERE name = ?", "006_add_metadata_content_digest"); err != nil { + t.Fatalf("resetting digest migration: %v", err) + } + if _, err := db.Exec(` + INSERT INTO metadata_cache (ecosystem, name, storage_path, content_type, size, fetched_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "oci-manifest", "cache-key", "_metadata/oci-manifest/cache-key/metadata", "application/json", 2, time.Now(), time.Now(), time.Now()); err != nil { + t.Fatalf("inserting legacy cache row: %v", err) + } + + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema() error = %v", err) + } + hasDigest, err := db.HasColumn("metadata_cache", "content_digest") + if err != nil { + t.Fatalf("HasColumn() error = %v", err) + } + if !hasDigest { + t.Fatal("metadata_cache.content_digest was not added") + } + + entry, err := db.GetMetadataCache("oci-manifest", "cache-key") + if err != nil { + t.Fatalf("GetMetadataCache() error = %v", err) + } + if entry == nil || entry.StoragePath != "_metadata/oci-manifest/cache-key/metadata" { + t.Fatalf("existing metadata cache row was not preserved: %#v", entry) + } + if entry.ContentDigest.Valid { + t.Errorf("legacy content digest = %q, want NULL", entry.ContentDigest.String) + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 5d95596..f8b76fe 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -894,7 +894,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err var entry MetadataCacheEntry query := db.Rebind(` SELECT id, ecosystem, name, storage_path, etag, content_type, - size, last_modified, fetched_at, created_at, updated_at + content_digest, size, last_modified, fetched_at, created_at, updated_at FROM metadata_cache WHERE ecosystem = ? AND name = ? `) err := db.Get(&entry, query, ecosystem, name) @@ -914,12 +914,13 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { if db.dialect == DialectPostgres { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, - size, last_modified, fetched_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + content_digest, size, last_modified, fetched_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = EXCLUDED.storage_path, etag = EXCLUDED.etag, content_type = EXCLUDED.content_type, + content_digest = EXCLUDED.content_digest, size = EXCLUDED.size, last_modified = EXCLUDED.last_modified, fetched_at = EXCLUDED.fetched_at, @@ -928,12 +929,13 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { } else { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, - size, last_modified, fetched_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + content_digest, size, last_modified, fetched_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = excluded.storage_path, etag = excluded.etag, content_type = excluded.content_type, + content_digest = excluded.content_digest, size = excluded.size, last_modified = excluded.last_modified, fetched_at = excluded.fetched_at, @@ -943,7 +945,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { _, err := db.Exec(query, entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, - entry.ContentType, entry.Size, entry.LastModified, entry.FetchedAt, now, now, + entry.ContentType, entry.ContentDigest, entry.Size, entry.LastModified, entry.FetchedAt, now, now, ) if err != nil { return fmt.Errorf("upserting metadata cache: %w", err) diff --git a/internal/database/schema.go b/internal/database/schema.go index c8d8d1e..c73877d 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -102,6 +102,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, + content_digest TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, @@ -202,6 +203,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, + content_digest TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, @@ -359,6 +361,7 @@ var migrations = []migration{ {"003_ensure_artifacts_table", migrateEnsureArtifactsTable}, {"004_ensure_vulnerabilities_table", migrateEnsureVulnerabilitiesTable}, {"005_ensure_metadata_cache_table", migrateEnsureMetadataCacheTable}, + {"006_add_metadata_content_digest", migrateAddMetadataContentDigest}, } // isTableNotFound returns true if the error indicates a missing table. @@ -581,6 +584,20 @@ func migrateEnsureMetadataCacheTable(db *DB) error { return db.EnsureMetadataCacheTable() } +func migrateAddMetadataContentDigest(db *DB) error { + hasColumn, err := db.HasColumn("metadata_cache", "content_digest") + if err != nil { + return fmt.Errorf("checking metadata_cache content_digest column: %w", err) + } + if hasColumn { + return nil + } + if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN content_digest TEXT"); err != nil { + return fmt.Errorf("adding metadata_cache content_digest column: %w", err) + } + return nil +} + // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. func (db *DB) EnsureMetadataCacheTable() error { has, err := db.HasTable("metadata_cache") @@ -601,6 +618,7 @@ func (db *DB) EnsureMetadataCacheTable() error { storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, + content_digest TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, @@ -618,6 +636,7 @@ func (db *DB) EnsureMetadataCacheTable() error { storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, + content_digest TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, diff --git a/internal/database/types.go b/internal/database/types.go index 47dc47e..7128f12 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -78,17 +78,18 @@ func (a *Artifact) IsCached() bool { // MetadataCacheEntry represents a cached metadata blob for offline serving. type MetadataCacheEntry struct { - ID int64 `db:"id" json:"id"` - Ecosystem string `db:"ecosystem" json:"ecosystem"` - Name string `db:"name" json:"name"` - StoragePath string `db:"storage_path" json:"storage_path"` - ETag sql.NullString `db:"etag" json:"etag,omitempty"` - ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` - Size sql.NullInt64 `db:"size" json:"size,omitempty"` - LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` - FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + ID int64 `db:"id" json:"id"` + Ecosystem string `db:"ecosystem" json:"ecosystem"` + Name string `db:"name" json:"name"` + StoragePath string `db:"storage_path" json:"storage_path"` + ETag sql.NullString `db:"etag" json:"etag,omitempty"` + ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` + ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"` + Size sql.NullInt64 `db:"size" json:"size,omitempty"` + LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` + FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // Vulnerability represents a cached vulnerability record. diff --git a/internal/handler/container.go b/internal/handler/container.go index 0710ed1..3a3b267 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -12,7 +12,6 @@ import ( const ( dockerHubRegistry = "https://registry-1.docker.io" - dockerHubAuth = "https://auth.docker.io" blobMatchCount = 3 // full match + name + digest manifestMatchCount = 3 // full match + name + reference tagsListMatchCount = 2 // full match + name @@ -24,7 +23,6 @@ const ( type ContainerHandler struct { proxy *Proxy registryURL string - authURL string proxyURL string } @@ -33,7 +31,6 @@ func NewContainerHandler(proxy *Proxy, proxyURL string) *ContainerHandler { return &ContainerHandler{ proxy: proxy, registryURL: dockerHubRegistry, - authURL: dockerHubAuth, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } @@ -90,31 +87,34 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req h.proxy.Logger.Info("container blob request", "name", name, "digest", digest) - // Get auth token for upstream - token, err := h.getAuthToken(r.Context(), name, "pull") + filename := digest + cached, err := h.proxy.GetCachedArtifact(r.Context(), "oci", name, digest, filename) if err != nil { - h.proxy.Logger.Error("failed to get auth token", "error", err) - h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") + h.proxy.Logger.Error("failed to check blob cache", "error", err) + h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to check blob cache") + return + } + if cached != nil { + w.Header().Set("Docker-Content-Digest", digest) + w.Header().Set("Content-Type", "application/octet-stream") + serveArtifact(w, r.Method, cached) return } // For HEAD requests, just proxy to upstream if r.Method == http.MethodHead { - h.proxyBlobHead(w, r, name, digest, token) + h.proxyBlobHead(w, r, name, digest) return } - // Try to get from cache, or fetch from upstream with auth - filename := digest - headers := http.Header{"Authorization": {"Bearer " + token}} - result, err := h.proxy.GetOrFetchArtifactFromURLWithHeaders( + // Try to get from cache, or fetch from the authentication-aware upstream client. + result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "oci", name, digest, // use digest as version filename, fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest), - headers, ) if err != nil { @@ -132,8 +132,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req ServeArtifact(w, result) } -// handleManifest proxies manifest requests to upstream. -// Manifests change when tags are updated, so we proxy these directly. +// handleManifest serves immutable manifests from cache and revalidates mutable tags. // Path format: {name}/manifests/{reference} func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request, path string) { if r.Method != http.MethodGet && r.Method != http.MethodHead { @@ -148,57 +147,7 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request } h.proxy.Logger.Info("container manifest request", "name", name, "reference", reference) - - // Get auth token - token, err := h.getAuthToken(r.Context(), name, "pull") - if err != nil { - h.proxy.Logger.Error("failed to get auth token", "error", err) - h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") - return - } - - // Proxy to upstream - upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", h.registryURL, name, reference) - - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) - if err != nil { - h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") - return - } - - req.Header.Set("Authorization", "Bearer "+token) - - // Forward Accept header for content negotiation - if accept := r.Header.Get("Accept"); accept != "" { - req.Header.Set("Accept", accept) - } else { - // Default accept headers for manifests - req.Header.Set("Accept", strings.Join([]string{ - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.oci.image.index.v1+json", - "application/vnd.docker.distribution.manifest.v2+json", - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.docker.distribution.manifest.v1+prettyjws", - }, ", ")) - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - h.proxy.Logger.Error("failed to fetch manifest", "error", err) - h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") - return - } - defer func() { _ = resp.Body.Close() }() - - // Copy relevant headers - for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag"} { - if v := resp.Header.Get(header); v != "" { - w.Header().Set(header, v) - } - } - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.serveManifest(w, r, name, reference) } // handleTagsList proxies tag list requests to upstream. @@ -214,13 +163,6 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request return } - // Get auth token - token, err := h.getAuthToken(r.Context(), name, "pull") - if err != nil { - h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") - return - } - upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", h.registryURL, name) if r.URL.RawQuery != "" { upstreamURL += "?" + r.URL.RawQuery @@ -232,8 +174,6 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request return } - req.Header.Set("Authorization", "Bearer "+token) - resp, err := h.proxy.HTTPClient.Do(req) if err != nil { h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") @@ -246,45 +186,8 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request _, _ = io.Copy(w, resp.Body) } -// getAuthToken gets a bearer token for the specified repository. -// Docker Hub requires auth even for public images. -func (h *ContainerHandler) getAuthToken(_ interface{ Done() <-chan struct{} }, repository, action string) (string, error) { - // For Docker Hub: https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull - authURL := fmt.Sprintf("%s/token?service=registry.docker.io&scope=repository:%s:%s", - h.authURL, repository, action) - - req, err := http.NewRequest(http.MethodGet, authURL, nil) - if err != nil { - return "", err - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return "", err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("auth failed with status %d", resp.StatusCode) - } - - var tokenResp struct { - Token string `json:"token"` - AccessToken string `json:"access_token"` - } - - if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { - return "", err - } - - if tokenResp.Token != "" { - return tokenResp.Token, nil - } - return tokenResp.AccessToken, nil -} - // proxyBlobHead handles HEAD requests for blobs. -func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, name, digest, token string) { +func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, name, digest string) { upstreamURL := fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest) req, err := http.NewRequestWithContext(r.Context(), http.MethodHead, upstreamURL, nil) @@ -293,8 +196,6 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, return } - req.Header.Set("Authorization", "Bearer "+token) - resp, err := h.proxy.HTTPClient.Do(req) if err != nil { h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go new file mode 100644 index 0000000..245ced4 --- /dev/null +++ b/internal/handler/container_manifest.go @@ -0,0 +1,251 @@ +package handler + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +const ( + containerManifestCacheEcosystem = "oci-manifest" + containerStaleWarning = `110 - "Response is Stale"` +) + +var manifestDigestReferencePattern = regexp.MustCompile(`^[a-z0-9]+:[a-f0-9]+$`) + +type cachedContainerManifest struct { + body []byte + contentType string + contentDigest string + etag string + size int64 + fetchedAt time.Time +} + +func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, name, reference string) { + accept := containerManifestAccept(r) + cacheKey := h.containerManifestCacheKey(name, reference, accept) + cached, err := h.loadContainerManifest(r.Context(), cacheKey) + if err != nil { + h.proxy.Logger.Warn("failed to read cached container manifest", "error", err) + cached = nil + } + + immutable := manifestDigestReferencePattern.MatchString(reference) + if cached != nil && (immutable || h.containerManifestFresh(cached)) { + writeContainerManifest(w, r.Method, cached, false) + return + } + + upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", h.registryURL, name, reference) + req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) + if err != nil { + h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") + return + } + req.Header.Set("Accept", accept) + if cached != nil && cached.etag != "" { + req.Header.Set("If-None-Match", cached.etag) + } + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.serveStaleManifestOrError(w, r, cached, err) + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified && cached != nil { + cached.fetchedAt = time.Now() + if err := h.storeContainerManifest(r.Context(), cacheKey, cached); err != nil { + h.proxy.Logger.Warn("failed to refresh cached container manifest", "error", err) + } + writeContainerManifest(w, r.Method, cached, false) + return + } + if resp.StatusCode != http.StatusOK { + if cached != nil && shouldServeStaleManifest(resp.StatusCode) { + writeContainerManifest(w, r.Method, cached, true) + return + } + copyContainerManifestHeaders(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + return + } + + if r.Method == http.MethodHead { + copyContainerManifestHeaders(w.Header(), resp.Header) + w.WriteHeader(http.StatusOK) + return + } + + body, err := h.proxy.ReadMetadata(resp.Body) + if err != nil { + h.serveStaleManifestOrError(w, r, cached, fmt.Errorf("reading manifest: %w", err)) + return + } + manifest := &cachedContainerManifest{ + body: body, + contentType: resp.Header.Get("Content-Type"), + contentDigest: resp.Header.Get("Docker-Content-Digest"), + etag: resp.Header.Get("ETag"), + size: int64(len(body)), + fetchedAt: time.Now(), + } + if manifest.contentDigest == "" { + manifest.contentDigest = sha256Digest(body) + } + if err := h.storeContainerManifest(r.Context(), cacheKey, manifest); err != nil { + h.proxy.Logger.Warn("failed to cache container manifest", "error", err) + } + if manifest.contentDigest != reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) { + digestKey := h.containerManifestCacheKey(name, manifest.contentDigest, accept) + if err := h.storeContainerManifest(r.Context(), digestKey, manifest); err != nil { + h.proxy.Logger.Warn("failed to cache container manifest by digest", "error", err) + } + } + writeContainerManifest(w, r.Method, manifest, false) +} + +func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, cached *cachedContainerManifest, err error) { + if cached != nil { + h.proxy.Logger.Warn("upstream manifest fetch failed, serving stale cache", "error", err) + writeContainerManifest(w, r.Method, cached, true) + return + } + h.proxy.Logger.Error("failed to fetch manifest", "error", err) + h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") +} + +func (h *ContainerHandler) containerManifestFresh(manifest *cachedContainerManifest) bool { + return h.proxy.MetadataTTL > 0 && !manifest.fetchedAt.IsZero() && time.Since(manifest.fetchedAt) < h.proxy.MetadataTTL +} + +func (h *ContainerHandler) containerManifestCacheKey(name, reference, accept string) string { + identity := strings.Join([]string{h.registryURL, name, reference, accept}, "\x00") + sum := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(sum[:]) +} + +func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, error) { + if h.proxy.DB == nil || h.proxy.Storage == nil { + return nil, nil + } + entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) + if err != nil || entry == nil { + return nil, err + } + reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil, nil + } + defer func() { _ = reader.Close() }() + body, err := h.proxy.ReadMetadata(reader) + if err != nil { + return nil, err + } + + manifest := &cachedContainerManifest{body: body, size: int64(len(body))} + if entry.ContentType.Valid { + manifest.contentType = entry.ContentType.String + } + if entry.ContentDigest.Valid { + manifest.contentDigest = entry.ContentDigest.String + } else { + manifest.contentDigest = sha256Digest(body) + } + if entry.ETag.Valid { + manifest.etag = entry.ETag.String + } + if entry.Size.Valid { + manifest.size = entry.Size.Int64 + } + if entry.FetchedAt.Valid { + manifest.fetchedAt = entry.FetchedAt.Time + } + return manifest, nil +} + +func (h *ContainerHandler) storeContainerManifest(ctx context.Context, cacheKey string, manifest *cachedContainerManifest) error { + if h.proxy.DB == nil || h.proxy.Storage == nil { + return nil + } + storagePath := metadataStoragePath(containerManifestCacheEcosystem, cacheKey) + size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(manifest.body)) + if err != nil { + return fmt.Errorf("storing manifest: %w", err) + } + manifest.size = size + return h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: containerManifestCacheEcosystem, + Name: cacheKey, + StoragePath: storagePath, + ETag: sql.NullString{String: manifest.etag, Valid: manifest.etag != ""}, + ContentType: sql.NullString{String: manifest.contentType, Valid: manifest.contentType != ""}, + ContentDigest: sql.NullString{String: manifest.contentDigest, Valid: manifest.contentDigest != ""}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: manifest.fetchedAt, Valid: !manifest.fetchedAt.IsZero()}, + }) +} + +func writeContainerManifest(w http.ResponseWriter, method string, manifest *cachedContainerManifest, stale bool) { + if manifest.contentType != "" { + w.Header().Set("Content-Type", manifest.contentType) + } + w.Header().Set("Content-Length", strconv.FormatInt(manifest.size, 10)) + if manifest.contentDigest != "" { + w.Header().Set("Docker-Content-Digest", manifest.contentDigest) + } + if manifest.etag != "" { + w.Header().Set("ETag", manifest.etag) + } + if stale { + w.Header().Set("Warning", containerStaleWarning) + } + w.WriteHeader(http.StatusOK) + if method != http.MethodHead { + _, _ = w.Write(manifest.body) + } +} + +func containerManifestAccept(r *http.Request) string { + if accept := r.Header.Get("Accept"); accept != "" { + return accept + } + return strings.Join([]string{ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v1+prettyjws", + }, ", ") +} + +func copyContainerManifestHeaders(destination, source http.Header) { + for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag", "WWW-Authenticate"} { + if value := source.Get(header); value != "" { + destination.Set(header, value) + } + } +} + +func shouldServeStaleManifest(status int) bool { + return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError +} + +func sha256Digest(body []byte) string { + digest := sha256.Sum256(body) + return "sha256:" + hex.EncodeToString(digest[:]) +} diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index 853059e..6e7322c 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -1,16 +1,15 @@ package handler import ( - "bytes" - "context" "encoding/json" "io" - "log/slog" "net/http" "net/http/httptest" + "strconv" "testing" + "time" - "github.com/git-pkgs/proxy/internal/database" + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/registries/fetch" ) @@ -135,90 +134,521 @@ func TestContainerHandler_parseTagsListPath(t *testing.T) { } } -func TestContainerHandler_BlobDownload_CachesWithAuth(t *testing.T) { - // Set up a mock auth server that returns a token - authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]string{"token": "test-token-123"}) +func TestContainerHandler_BlobDownload_DiscoversBearerChallenge(t *testing.T) { + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + registryRequests := 0 + tokenRequests := 0 + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + tokenRequests++ + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": "discovered-token", + "expires_in": 3600, + }) + case "/v2/library/nginx/blobs/" + digest: + registryRequests++ + if r.Header.Get("Authorization") != "Bearer discovered-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="registry.test",scope="repository:library/nginx:pull"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = io.WriteString(w, "upstream blob") + default: + http.NotFound(w, r) + } })) - defer authServer.Close() + defer upstream.Close() - // Set up mock fetcher that captures headers - var capturedHeaders http.Header - mf := &mockFetcherWithHeaders{ - fetchFn: func(_ context.Context, _ string, headers http.Header) (*fetch.Artifact, error) { - capturedHeaders = headers - return &fetch.Artifact{ - Body: io.NopCloser(bytes.NewReader([]byte("blob-content"))), - Size: 12, - ContentType: "application/octet-stream", - }, nil - }, - } - - dir := t.TempDir() - db, err := database.Create(dir + "/test.db") - if err != nil { - t.Fatalf("failed to create test database: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - - store := newMockStorage() - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - proxy := &Proxy{ - DB: db, - Storage: store, - Fetcher: mf, - Logger: logger, - HTTPClient: &http.Client{}, - } + proxy, _, _, _ := setupTestProxy(t) + authTransport := upstreamhttp.NewTransport(http.DefaultTransport, nil) + client := &http.Client{Transport: authTransport} + artifactFetcher := fetch.NewFetcher( + fetch.WithHTTPClient(client), + fetch.WithMaxRetries(0), + ) + t.Cleanup(func() { _ = artifactFetcher.Close() }) + proxy.Fetcher = artifactFetcher + proxy.HTTPClient = client h := &ContainerHandler{ proxy: proxy, - registryURL: "https://registry-1.docker.io", - authURL: authServer.URL, + registryURL: upstream.URL, proxyURL: "http://localhost:8080", } - handler := h.Routes() - req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil) + for range 2 { + req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := w.Body.String(); got != "upstream blob" { + t.Errorf("body = %q, want %q", got, "upstream blob") + } + } + + if tokenRequests != 1 { + t.Errorf("token requests = %d, want 1", tokenRequests) + } + if registryRequests != 2 { + t.Errorf("registry requests = %d, want 2", registryRequests) + } +} + +func TestContainerHandler_CachedImagePullSurvivesRegistryAndTokenOutages(t *testing.T) { + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` + blob := "cached image blob" + registryAvailable := true + tokenAvailable := true + registryRequests := 0 + tokenRequests := 0 + + tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + tokenRequests++ + if !tokenAvailable { + http.Error(w, "token service unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": "discovered-token", + "expires_in": 3600, + }) + })) + defer tokenServer.Close() + + registryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + registryRequests++ + if !registryAvailable { + http.Error(w, "registry unavailable", http.StatusServiceUnavailable) + return + } + if r.Header.Get("Authorization") != "Bearer discovered-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenServer.URL+`",service="registry.test",scope="repository:library/nginx:pull"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + return + } + + switch r.URL.Path { + case "/v2/library/nginx/manifests/latest": + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + _, _ = io.WriteString(w, manifest) + case "/v2/library/nginx/blobs/" + digest: + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = io.WriteString(w, blob) + default: + http.NotFound(w, r) + } + })) + defer registryServer.Close() + + warmProxy, db, store, _ := setupTestProxy(t) + warmClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)} + warmFetcher := fetch.NewFetcher( + fetch.WithHTTPClient(warmClient), + fetch.WithMaxRetries(0), + ) + t.Cleanup(func() { _ = warmFetcher.Close() }) + warmProxy.Fetcher = warmFetcher + warmProxy.HTTPClient = warmClient + warmProxy.MetadataTTL = time.Hour + warmHandler := (&ContainerHandler{ + proxy: warmProxy, + registryURL: registryServer.URL, + proxyURL: "http://localhost:8080", + }).Routes() + + for _, request := range []struct { + path string + body string + }{ + {path: "/library/nginx/manifests/latest", body: manifest}, + {path: "/library/nginx/blobs/" + digest, body: blob}, + } { + response := httptest.NewRecorder() + warmHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil)) + if response.Code != http.StatusOK { + t.Fatalf("warming %s: status = %d, want %d; body: %s", request.path, response.Code, http.StatusOK, response.Body.String()) + } + if got := response.Body.String(); got != request.body { + t.Fatalf("warming %s: body = %q, want %q", request.path, got, request.body) + } + } + + warmRegistryRequests := registryRequests + warmTokenRequests := tokenRequests + registryAvailable = false + tokenAvailable = false + + offlineClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)} + offlineFetcher := fetch.NewFetcher( + fetch.WithHTTPClient(offlineClient), + fetch.WithMaxRetries(0), + ) + t.Cleanup(func() { _ = offlineFetcher.Close() }) + offlineProxy := NewProxy(db, store, offlineFetcher, fetch.NewResolver(), warmProxy.Logger) + offlineProxy.HTTPClient = offlineClient + offlineProxy.MetadataTTL = time.Hour + offlineHandler := (&ContainerHandler{ + proxy: offlineProxy, + registryURL: registryServer.URL, + proxyURL: "http://localhost:8080", + }).Routes() + + for _, request := range []struct { + name string + path string + body string + }{ + {name: "tag manifest", path: "/library/nginx/manifests/latest", body: manifest}, + {name: "digest manifest", path: "/library/nginx/manifests/" + digest, body: manifest}, + {name: "blob", path: "/library/nginx/blobs/" + digest, body: blob}, + } { + t.Run(request.name, func(t *testing.T) { + response := httptest.NewRecorder() + offlineHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil)) + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", response.Code, http.StatusOK, response.Body.String()) + } + if got := response.Body.String(); got != request.body { + t.Errorf("body = %q, want %q", got, request.body) + } + if got := response.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } + }) + } + + if registryRequests != warmRegistryRequests { + t.Errorf("offline registry requests = %d, want 0", registryRequests-warmRegistryRequests) + } + if tokenRequests != warmTokenRequests { + t.Errorf("offline token requests = %d, want 0", tokenRequests-warmTokenRequests) + } +} + +func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + })) + defer upstream.Close() + + h := &ContainerHandler{ + proxy: proxy, + registryURL: upstream.URL, + proxyURL: "http://localhost:8080", + } + + req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) w := httptest.NewRecorder() - handler.ServeHTTP(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusOK { - t.Errorf("got status %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) } - - // Verify auth header was passed to the fetcher - if capturedHeaders == nil { - t.Fatal("expected headers to be passed to fetcher, got nil") + if got := w.Body.String(); got != "cached blob" { + t.Errorf("body = %q, want %q", got, "cached blob") } - auth := capturedHeaders.Get("Authorization") - if auth != "Bearer test-token-123" { - t.Errorf("Authorization = %q, want %q", auth, "Bearer test-token-123") + if upstreamRequests != 0 { + t.Errorf("upstream requests = %d, want 0", upstreamRequests) } - - // Verify response headers - if got := w.Header().Get("Docker-Content-Digest"); got != "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" { - t.Errorf("Docker-Content-Digest = %q, want digest", got) + if fetcher.fetchCalled { + t.Error("fetcher should not be called on cache hit") } } -// mockFetcherWithHeaders captures headers passed to FetchWithHeaders. -type mockFetcherWithHeaders struct { - fetchFn func(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) +func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + })) + defer upstream.Close() + proxy.HTTPClient = upstream.Client() + + h := &ContainerHandler{ + proxy: proxy, + registryURL: upstream.URL, + proxyURL: "http://localhost:8080", + } + + req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := w.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } + if got := w.Header().Get("Content-Length"); got != "11" { + t.Errorf("Content-Length = %q, want %q", got, "11") + } + if w.Body.Len() != 0 { + t.Errorf("HEAD response body length = %d, want 0", w.Body.Len()) + } + if upstreamRequests != 0 { + t.Errorf("upstream requests = %d, want 0", upstreamRequests) + } + if fetcher.fetchCalled { + t.Error("fetcher should not be called on cache hit") + } } -func (f *mockFetcherWithHeaders) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { - return f.FetchWithHeaders(ctx, url, nil) +func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + store.signedURL = "https://storage.example.test/cached-blob?signature=test" + proxy.DirectServe = true + + h := &ContainerHandler{ + proxy: proxy, + registryURL: "https://registry.example.test", + proxyURL: "http://localhost:8080", + } + + req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusFound { + t.Fatalf("status = %d, want %d", w.Code, http.StatusFound) + } + if got := w.Header().Get("Location"); got != store.signedURL { + t.Errorf("Location = %q, want %q", got, store.signedURL) + } + if got := w.Header().Get("ETag"); got != `"abc123"` { + t.Errorf("ETag = %q, want %q", got, `"abc123"`) + } + if w.Body.Len() != 0 { + t.Errorf("HEAD response body length = %d, want 0", w.Body.Len()) + } + if fetcher.fetchCalled { + t.Error("fetcher should not be called on cache hit") + } } -func (f *mockFetcherWithHeaders) FetchWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) { - return f.fetchFn(ctx, url, headers) +func TestContainerHandler_ManifestByDigest_CacheHitSkipsUpstream(t *testing.T) { + digest := "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` + upstreamAvailable := true + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + if !upstreamAvailable { + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + return + } + if r.URL.Path != "/v2/library/nginx/manifests/"+digest { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + w.Header().Set("ETag", `"manifest-etag"`) + if r.Method != http.MethodHead { + _, _ = io.WriteString(w, manifest) + } + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + first := httptest.NewRecorder() + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) + if first.Code != http.StatusOK { + t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) + } + if first.Body.String() != manifest { + t.Fatalf("initial body = %q, want %q", first.Body.String(), manifest) + } + + upstreamAvailable = false + second := httptest.NewRecorder() + h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) + if second.Code != http.StatusOK { + t.Fatalf("cached status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String()) + } + if second.Body.String() != manifest { + t.Errorf("cached body = %q, want %q", second.Body.String(), manifest) + } + if got := second.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("cached Docker-Content-Digest = %q, want %q", got, digest) + } + + head := httptest.NewRecorder() + h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/"+digest, nil)) + if head.Code != http.StatusOK { + t.Fatalf("cached HEAD status = %d, want %d", head.Code, http.StatusOK) + } + wantLength := strconv.Itoa(len(manifest)) + if got := head.Header().Get("Content-Length"); got != wantLength { + t.Errorf("cached HEAD Content-Length = %q, want %q", got, wantLength) + } + if head.Body.Len() != 0 { + t.Errorf("cached HEAD body length = %d, want 0", head.Body.Len()) + } + if upstreamRequests != 1 { + t.Errorf("upstream requests = %d, want 1", upstreamRequests) + } } -func (f *mockFetcherWithHeaders) Head(_ context.Context, _ string) (int64, string, error) { - return 0, "", nil +func TestContainerHandler_ManifestByTag_UsesStaleCacheOnUpstreamFailure(t *testing.T) { + digest := "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json"}` + upstreamAvailable := true + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + if !upstreamAvailable { + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + _, _ = io.WriteString(w, manifest) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.MetadataTTL = 0 + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + first := httptest.NewRecorder() + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) + if first.Code != http.StatusOK { + t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) + } + + upstreamAvailable = false + second := httptest.NewRecorder() + h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) + if second.Code != http.StatusOK { + t.Fatalf("stale status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String()) + } + if second.Body.String() != manifest { + t.Errorf("stale body = %q, want %q", second.Body.String(), manifest) + } + if got := second.Header().Get("Warning"); got != `110 - "Response is Stale"` { + t.Errorf("Warning = %q, want stale warning", got) + } + if got := second.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("stale Docker-Content-Digest = %q, want %q", got, digest) + } + if upstreamRequests != 2 { + t.Errorf("upstream requests = %d, want 2", upstreamRequests) + } +} + +func TestContainerHandler_ManifestByTag_CachesDigestAlias(t *testing.T) { + digest := "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` + upstreamAvailable := true + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + if !upstreamAvailable { + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + return + } + if r.URL.Path != "/v2/library/nginx/manifests/latest" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + _, _ = io.WriteString(w, manifest) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + first := httptest.NewRecorder() + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) + if first.Code != http.StatusOK { + t.Fatalf("tag status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) + } + + upstreamAvailable = false + byDigest := httptest.NewRecorder() + h.Routes().ServeHTTP(byDigest, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) + if byDigest.Code != http.StatusOK { + t.Fatalf("digest status = %d, want %d; body: %s", byDigest.Code, http.StatusOK, byDigest.Body.String()) + } + if byDigest.Body.String() != manifest { + t.Errorf("digest body = %q, want %q", byDigest.Body.String(), manifest) + } + if got := byDigest.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } + if upstreamRequests != 1 { + t.Errorf("upstream requests = %d, want 1", upstreamRequests) + } +} + +func TestContainerHandler_ManifestByTag_StaleHeadChecksUpstream(t *testing.T) { + oldDigest := "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + newDigest := "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + currentDigest := oldDigest + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", currentDigest) + w.Header().Set("ETag", `"`+currentDigest+`"`) + if r.Method != http.MethodHead { + _, _ = io.WriteString(w, `{"schemaVersion":2}`) + } + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.MetadataTTL = 0 + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + first := httptest.NewRecorder() + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) + if first.Code != http.StatusOK { + t.Fatalf("initial status = %d, want %d", first.Code, http.StatusOK) + } + + currentDigest = newDigest + head := httptest.NewRecorder() + h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/latest", nil)) + if head.Code != http.StatusOK { + t.Fatalf("HEAD status = %d, want %d", head.Code, http.StatusOK) + } + if got := head.Header().Get("Docker-Content-Digest"); got != newDigest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, newDigest) + } + if upstreamRequests != 2 { + t.Errorf("upstream requests = %d, want 2", upstreamRequests) + } } func TestContainerHandler_Routes_VersionCheck(t *testing.T) { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index fc78dbf..72b9c28 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -136,18 +136,25 @@ type CacheResult struct { // GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream. func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { - pkgPURL := purl.MakePURLString(ecosystem, name, "") - versionPURL := purl.MakePURLString(ecosystem, name, version) - - if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { + if cached, err := p.GetCachedArtifact(ctx, ecosystem, name, version, filename); err != nil { return nil, err } else if cached != nil { return cached, nil } + pkgPURL := purl.MakePURLString(ecosystem, name, "") + versionPURL := purl.MakePURLString(ecosystem, name, version) return p.fetchAndCache(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL) } +// GetCachedArtifact retrieves an artifact from cache without contacting an upstream. +// It returns nil when no usable cache entry exists. +func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { + pkgPURL := purl.MakePURLString(ecosystem, name, "") + versionPURL := purl.MakePURLString(ecosystem, name, version) + return p.checkCache(ctx, pkgPURL, versionPURL, filename) +} + // checkCache looks up an artifact in the cache. Returns nil if not cached. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { pkg, err := p.DB.GetPackageByPURL(pkgPURL) @@ -363,6 +370,10 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u // ServeArtifact writes a CacheResult to an HTTP response. func ServeArtifact(w http.ResponseWriter, result *CacheResult) { + serveArtifact(w, http.MethodGet, result) +} + +func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { if result.RedirectURL != "" { if result.Hash != "" { w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) @@ -372,12 +383,14 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { return } - defer func() { _ = result.Reader.Close() }() + if result.Reader != nil { + defer func() { _ = result.Reader.Close() }() + } if result.ContentType != "" { w.Header().Set("Content-Type", result.ContentType) } - if result.Size > 0 { + if result.Size > 0 || (method == http.MethodHead && result.Size == 0) { w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size)) } if result.Hash != "" { @@ -385,7 +398,9 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { } w.WriteHeader(http.StatusOK) - _, _ = io.Copy(w, result.Reader) + if method != http.MethodHead && result.Reader != nil { + _, _ = io.Copy(w, result.Reader) + } } // ProxyUpstream forwards a request to an upstream URL without caching. @@ -807,18 +822,16 @@ func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, } // GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL -// with additional HTTP headers. This is needed for registries that require authentication -// (e.g. Docker Hub requires a Bearer token even for public images). +// with additional request-specific HTTP headers. func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) { - pkgPURL := purl.MakePURLString(ecosystem, name, "") - versionPURL := purl.MakePURLString(ecosystem, name, version) - - if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { + if cached, err := p.GetCachedArtifact(ctx, ecosystem, name, version, filename); err != nil { return nil, err } else if cached != nil { return cached, nil } + pkgPURL := purl.MakePURLString(ecosystem, name, "") + versionPURL := purl.MakePURLString(ecosystem, name, version) return p.fetchAndCacheFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 9a2b329..a3bd9b3 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -5,7 +5,6 @@ import ( "context" "database/sql" "errors" - "fmt" "io" "log/slog" "net/http" @@ -17,6 +16,7 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -152,7 +152,7 @@ func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, n t.Helper() pkg := &database.Package{ - PURL: fmt.Sprintf("pkg:%s/%s", ecosystem, name), + PURL: purl.MakePURLString(ecosystem, name, ""), Ecosystem: ecosystem, Name: name, } @@ -160,7 +160,7 @@ func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, n t.Fatalf("failed to upsert package: %v", err) } - versionPURL := fmt.Sprintf("pkg:%s/%s@%s", ecosystem, name, version) + versionPURL := purl.MakePURLString(ecosystem, name, version) ver := &database.Version{ PURL: versionPURL, PackagePURL: pkg.PURL, diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index 6f04113..44c4486 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -1,7 +1,6 @@ package handler import ( - "context" "net/http" "net/http/httptest" "strings" @@ -117,23 +116,12 @@ func TestComposerDownloadUpstreamNotFoundReturns404(t *testing.T) { } func TestContainerBlobUpstreamNotFoundReturns404(t *testing.T) { - authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"token": "test-token-123"}`)) - })) - defer authServer.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.Fetcher = &mockFetcherWithHeaders{ - fetchFn: func(_ context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { - return nil, fetch.ErrNotFound - }, - } + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound h := &ContainerHandler{ proxy: proxy, registryURL: "https://registry-1.docker.io", - authURL: authServer.URL, proxyURL: "http://localhost:8080", } From 6fcc57c994f5d3b3ca51ac51a3a56bd045c07e2a Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 13 Aug 2026 08:06:41 +0100 Subject: [PATCH 53/70] Optimize cached artifact serving (#245) --- internal/database/database_test.go | 80 +++++++ internal/database/queries.go | 22 ++ internal/database/types.go | 10 + internal/handler/handler.go | 65 +++--- internal/handler/handler_bench_test.go | 300 +++++++++++++++++++++++++ internal/handler/handler_test.go | 4 +- 6 files changed, 444 insertions(+), 37 deletions(-) create mode 100644 internal/handler/handler_bench_test.go diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 6fca4ea..3e92b91 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -239,6 +239,86 @@ func TestArtifactCRUD(t *testing.T) { }) } +func TestGetCachedArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const ( + packagePURL = "pkg:npm/lodash" + versionPURL = "pkg:npm/lodash@4.17.21" + filename = "lodash-4.17.21.tgz" + ) + seedCachedArtifactTestData(t, db, packagePURL, versionPURL, filename) + + cached, err := db.GetCachedArtifact(packagePURL, versionPURL, filename) + if err != nil { + t.Fatalf("GetCachedArtifact before cache failed: %v", err) + } + if cached != nil { + t.Fatalf("expected no cached artifact, got %+v", cached) + } + + if err := db.MarkArtifactCached(versionPURL, filename, "/cache/npm/"+filename, + "sha256-abc", 12345, "application/gzip"); err != nil { + t.Fatalf("MarkArtifactCached failed: %v", err) + } + + cached, err = db.GetCachedArtifact(packagePURL, versionPURL, filename) + if err != nil { + t.Fatalf("GetCachedArtifact failed: %v", err) + } + if cached == nil { + t.Fatal("expected cached artifact, got nil") + } + if cached.Ecosystem != "npm" { + t.Errorf("expected npm ecosystem, got %q", cached.Ecosystem) + } + if cached.StoragePath != "/cache/npm/"+filename { + t.Errorf("expected cached storage path, got %q", cached.StoragePath) + } + if cached.ContentHash.String != "sha256-abc" { + t.Errorf("expected cached content hash, got %q", cached.ContentHash.String) + } + if cached.Size.Int64 != 12345 { + t.Errorf("expected cached size 12345, got %d", cached.Size.Int64) + } + if cached.ContentType.String != "application/gzip" { + t.Errorf("expected cached content type, got %q", cached.ContentType.String) + } + if cached.Integrity.String != "sha512-abc123" { + t.Errorf("expected cached integrity, got %q", cached.Integrity.String) + } + + cached, err = db.GetCachedArtifact("pkg:npm/other", versionPURL, filename) + if err != nil { + t.Fatalf("GetCachedArtifact with wrong package failed: %v", err) + } + if cached != nil { + t.Fatalf("expected package mismatch to miss cache, got %+v", cached) + } + }) +} + +func seedCachedArtifactTestData(t *testing.T, db *DB, packagePURL, versionPURL, filename string) { + t.Helper() + + if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage failed: %v", err) + } + if err := db.UpsertVersion(&Version{ + PURL: versionPURL, + PackagePURL: packagePURL, + Integrity: sql.NullString{String: "sha512-abc123", Valid: true}, + }); err != nil { + t.Fatalf("UpsertVersion failed: %v", err) + } + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://registry.npmjs.org/lodash/-/" + filename, + }); err != nil { + t.Fatalf("UpsertArtifact failed: %v", err) + } +} + func TestCacheManagement(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { pkg := &Package{ diff --git a/internal/database/queries.go b/internal/database/queries.go index f8b76fe..b01abe4 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -191,6 +191,28 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { return &a, nil } +// GetCachedArtifact returns the fields needed to serve a cached artifact. +func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*CachedArtifact, error) { + var artifact CachedArtifact + query := db.Rebind(` + SELECT packages.ecosystem, artifacts.storage_path, artifacts.content_hash, artifacts.size, + artifacts.content_type, versions.integrity + FROM artifacts + JOIN versions ON versions.purl = artifacts.version_purl + JOIN packages ON packages.purl = versions.package_purl + WHERE packages.purl = ? AND artifacts.version_purl = ? AND artifacts.filename = ? + AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL + `) + err := db.Get(&artifact, query, packagePURL, versionPURL, filename) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + return &artifact, nil +} + func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) { var a Artifact query := db.Rebind(` diff --git a/internal/database/types.go b/internal/database/types.go index 7128f12..3826c7a 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -76,6 +76,16 @@ func (a *Artifact) IsCached() bool { return a.StoragePath.Valid && a.FetchedAt.Valid } +// CachedArtifact contains the fields needed to serve a cached artifact. +type CachedArtifact struct { + Ecosystem string `db:"ecosystem"` + StoragePath string `db:"storage_path"` + ContentHash sql.NullString `db:"content_hash"` + Size sql.NullInt64 `db:"size"` + ContentType sql.NullString `db:"content_type"` + Integrity sql.NullString `db:"integrity"` +} + // MetadataCacheEntry represents a cached metadata blob for offline serving. type MetadataCacheEntry struct { ID int64 `db:"id" json:"id"` diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 72b9c28..6a7aab4 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -13,6 +13,7 @@ import ( "net/url" "strconv" "strings" + "sync" "time" "github.com/git-pkgs/cooldown" @@ -48,6 +49,15 @@ func hasDotDotSegment(path string) bool { const defaultHTTPTimeout = 30 * time.Second +const artifactCopyBufferSize = 32 << 10 + +var artifactCopyBufferPool = sync.Pool{ //nolint:gochecknoglobals // shared across artifact responses + New: func() any { + buffer := make([]byte, artifactCopyBufferSize) + return &buffer + }, +} + // canonicalPackagePURL returns a versionless PURL in canonical form so cooldown // lookups match keys produced by config.CooldownConfig.NormalizedPackages. func canonicalPackagePURL(ecosystem, name string) string { @@ -157,27 +167,11 @@ func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, // checkCache looks up an artifact in the cache. Returns nil if not cached. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { - pkg, err := p.DB.GetPackageByPURL(pkgPURL) - if err != nil { - return nil, fmt.Errorf("checking package cache: %w", err) - } - if pkg == nil { - return nil, nil - } - - ver, err := p.DB.GetVersionByPURL(versionPURL) - if err != nil { - return nil, fmt.Errorf("checking version cache: %w", err) - } - if ver == nil { - return nil, nil - } - - artifact, err := p.DB.GetArtifact(versionPURL, filename) + artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) } - if artifact == nil || !artifact.IsCached() { + if artifact == nil { return nil, nil } @@ -189,39 +183,39 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s } if p.DirectServe { - signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL) + signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) if err == nil { result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL) - p.recordCacheHit(pkgPURL, versionPURL, filename) + p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) return result, nil } if !errors.Is(err, storage.ErrSignedURLUnsupported) { p.Logger.Warn("failed to sign storage URL, falling back to streaming", - "path", artifact.StoragePath.String, "error", err) + "path", artifact.StoragePath, "error", err) } } start := time.Now() - reader, err := p.Storage.Open(ctx, artifact.StoragePath.String) + reader, err := p.Storage.Open(ctx, artifact.StoragePath) metrics.RecordStorageOperation("read", time.Since(start)) if err != nil { metrics.RecordStorageError("read") p.Logger.Warn("cached artifact missing from storage, will refetch", - "path", artifact.StoragePath.String, "error", err) + "path", artifact.StoragePath, "error", err) return nil, nil } - result.Reader = newVerifyingReader(reader, artifact.ContentHash.String, ver.Integrity.String, + result.Reader = newVerifyingReader(reader, artifact.ContentHash.String, artifact.Integrity.String, func(reason string) { p.Logger.Error("cached artifact failed integrity check", "purl", versionPURL, "filename", filename, - "path", artifact.StoragePath.String, "reason", reason) - metrics.RecordIntegrityFailure(pkg.Ecosystem) + "path", artifact.StoragePath, "reason", reason) + metrics.RecordIntegrityFailure(artifact.Ecosystem) if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil { p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err) } }) - p.recordCacheHit(pkgPURL, versionPURL, filename) + p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) return result, nil } @@ -245,11 +239,9 @@ func rewriteSignedURLHost(signed, baseURL string) string { return s.String() } -func (p *Proxy) recordCacheHit(pkgPURL, versionPURL, filename string) { +func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { _ = p.DB.RecordArtifactHit(versionPURL, filename) - if parsed, err := purl.Parse(pkgPURL); err == nil { - metrics.RecordCacheHit(purl.PURLTypeToEcosystem(parsed.Type)) - } + metrics.RecordCacheHit(purl.NormalizeEcosystem(ecosystem)) } func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) { @@ -376,7 +368,7 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { if result.RedirectURL != "" { if result.Hash != "" { - w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) + w.Header().Set("ETag", `"`+result.Hash+`"`) } w.Header().Set("Location", result.RedirectURL) w.WriteHeader(http.StatusFound) @@ -391,15 +383,18 @@ func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { w.Header().Set("Content-Type", result.ContentType) } if result.Size > 0 || (method == http.MethodHead && result.Size == 0) { - w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size)) + w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10)) } if result.Hash != "" { - w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) + w.Header().Set("ETag", `"`+result.Hash+`"`) } w.WriteHeader(http.StatusOK) if method != http.MethodHead && result.Reader != nil { - _, _ = io.Copy(w, result.Reader) + buffer := artifactCopyBufferPool.Get().(*[]byte) + defer artifactCopyBufferPool.Put(buffer) + // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. + _, _ = io.CopyBuffer(struct{ io.Writer }{w}, result.Reader, *buffer) } } diff --git a/internal/handler/handler_bench_test.go b/internal/handler/handler_bench_test.go new file mode 100644 index 0000000..cdbb524 --- /dev/null +++ b/internal/handler/handler_bench_test.go @@ -0,0 +1,300 @@ +package handler + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/fetch" +) + +const benchmarkArtifactSize = 64 << 10 + +const benchmarkMetadataSize = 1 << 20 + +type benchmarkResponseWriter struct { + header http.Header +} + +func (w *benchmarkResponseWriter) Header() http.Header { + return w.header +} + +func (w *benchmarkResponseWriter) Write(p []byte) (int, error) { + return len(p), nil +} + +func (w *benchmarkResponseWriter) WriteHeader(_ int) {} + +func benchmarkCachedProxy(b *testing.B) (*Proxy, *mockStorage) { + b.Helper() + + proxy, db, store, _ := setupTestProxy(b) + content := strings.Repeat("x", benchmarkArtifactSize) + seedPackage(b, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", content) + + artifact, err := db.GetArtifact("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz") + if err != nil { + b.Fatalf("get seeded artifact: %v", err) + } + sum := sha256.Sum256([]byte(content)) + artifact.ContentHash.String = hex.EncodeToString(sum[:]) + if err := db.UpsertArtifact(artifact); err != nil { + b.Fatalf("update seeded artifact hash: %v", err) + } + + return proxy, store +} + +func BenchmarkArtifactCacheHit(b *testing.B) { + ctx := context.Background() + + b.Run("stream-64KiB", func(b *testing.B) { + proxy, _ := benchmarkCachedProxy(b) + w := &benchmarkResponseWriter{header: make(http.Header)} + b.SetBytes(benchmarkArtifactSize) + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + b.Fatal(err) + } + ServeArtifact(w, result) + } + }) + + b.Run("direct-serve", func(b *testing.B) { + proxy, store := benchmarkCachedProxy(b) + proxy.DirectServe = true + store.signedURL = "https://storage.example/npm/lodash-4.17.21.tgz?signature=abc" + w := &benchmarkResponseWriter{header: make(http.Header)} + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + b.Fatal(err) + } + ServeArtifact(w, result) + } + }) +} + +func BenchmarkArtifactCacheHitParallel(b *testing.B) { + proxy, _ := benchmarkCachedProxy(b) + ctx := context.Background() + b.SetBytes(benchmarkArtifactSize) + b.ReportAllocs() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + w := &benchmarkResponseWriter{header: make(http.Header)} + for pb.Next() { + result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + b.Error(err) + return + } + ServeArtifact(w, result) + } + }) +} + +func BenchmarkReadMetadata(b *testing.B) { + payload := bytes.Repeat([]byte("x"), benchmarkMetadataSize) + proxy := &Proxy{MetadataMaxSize: benchmarkMetadataSize} + b.SetBytes(benchmarkMetadataSize) + b.ReportAllocs() + + var data []byte + for b.Loop() { + var err error + data, err = proxy.ReadMetadata(bytes.NewReader(payload)) + if err != nil { + b.Fatal(err) + } + } + if len(data) != len(payload) { + b.Fatalf("metadata size = %d, want %d", len(data), len(payload)) + } +} + +func BenchmarkArtifactPURLConstruction(b *testing.B) { + for _, tc := range []struct { + name string + ecosystem string + packageID string + }{ + {"npm", "npm", "lodash"}, + {"scoped-npm", "npm", "@scope/package"}, + {"go", "golang", "github.com/git-pkgs/proxy"}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var packagePURL, versionPURL string + for b.Loop() { + packagePURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "") + versionPURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "1.2.3") + } + if packagePURL == "" || versionPURL == "" { + b.Fatal("empty PURL") + } + }) + } +} + +type benchmarkNPMServer struct { + client *http.Client + requestURL string + db *database.DB + versionPURL string + filename string +} + +func newBenchmarkNPMServer(b *testing.B) *benchmarkNPMServer { + b.Helper() + + ctx := context.Background() + dir := b.TempDir() + db, err := database.Create(filepath.Join(dir, "benchmark.db")) + if err != nil { + b.Fatalf("create database: %v", err) + } + b.Cleanup(func() { _ = db.Close() }) + + store, err := storage.OpenBucket(ctx, "file://"+filepath.Join(dir, "cache")) + if err != nil { + b.Fatalf("open storage: %v", err) + } + b.Cleanup(func() { _ = store.Close() }) + + content := bytes.Repeat([]byte("x"), benchmarkArtifactSize) + storagePath := storage.ArtifactPath("npm", "", "lodash", "4.17.21", "lodash-4.17.21.tgz") + size, hash, err := store.Store(ctx, storagePath, bytes.NewReader(content)) + if err != nil { + b.Fatalf("store artifact: %v", err) + } + + pkg := &database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"} + if err := db.UpsertPackage(pkg); err != nil { + b.Fatalf("seed package: %v", err) + } + version := &database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: pkg.PURL} + if err := db.UpsertVersion(version); err != nil { + b.Fatalf("seed version: %v", err) + } + artifact := &database.Artifact{ + VersionPURL: version.PURL, + Filename: "lodash-4.17.21.tgz", + UpstreamURL: "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + StoragePath: sql.NullString{String: storagePath, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + ContentType: sql.NullString{String: "application/gzip", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + } + if err := db.UpsertArtifact(artifact); err != nil { + b.Fatalf("seed artifact: %v", err) + } + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger) + handler := NewNPMHandler(proxy, "http://proxy.example", "https://registry.npmjs.org") + server := httptest.NewServer(handler.Routes()) + b.Cleanup(server.Close) + client := server.Client() + return &benchmarkNPMServer{ + client: client, + requestURL: server.URL + "/lodash/-/lodash-4.17.21.tgz", + db: db, + versionPURL: version.PURL, + filename: artifact.Filename, + } +} + +func (s *benchmarkNPMServer) request() error { + resp, err := s.client.Get(s.requestURL) + if err != nil { + return fmt.Errorf("GET cached artifact: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET cached artifact status = %d, want %d", resp.StatusCode, http.StatusOK) + } + n, err := io.Copy(io.Discard, resp.Body) + if err != nil { + return fmt.Errorf("read cached artifact: %w", err) + } + if n != benchmarkArtifactSize { + return fmt.Errorf("cached artifact size = %d, want %d", n, benchmarkArtifactSize) + } + return nil +} + +func (s *benchmarkNPMServer) hitCount(b *testing.B) int64 { + b.Helper() + artifact, err := s.db.GetArtifact(s.versionPURL, s.filename) + if err != nil { + b.Fatalf("get artifact hit count: %v", err) + } + return artifact.HitCount +} + +func benchmarkNPMArtifactCacheHitHTTP(b *testing.B, parallel bool) { + server := newBenchmarkNPMServer(b) + if err := server.request(); err != nil { + b.Fatal(err) + } + startHits := server.hitCount(b) + + b.SetBytes(benchmarkArtifactSize) + b.ReportAllocs() + b.ResetTimer() + if parallel { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := server.request(); err != nil { + b.Error(err) + return + } + } + }) + } else { + for b.Loop() { + if err := server.request(); err != nil { + b.Fatal(err) + } + } + } + b.StopTimer() + + if hitCount := server.hitCount(b) - startHits; hitCount != int64(b.N) { + b.Fatalf("new artifact hits = %d, want %d", hitCount, b.N) + } + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "requests/s") +} + +func BenchmarkNPMArtifactCacheHitHTTP(b *testing.B) { + benchmarkNPMArtifactCacheHitHTTP(b, false) +} + +func BenchmarkNPMArtifactCacheHitHTTPParallel(b *testing.B) { + benchmarkNPMArtifactCacheHitHTTP(b, true) +} diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index a3bd9b3..d52e7b6 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -128,7 +128,7 @@ func (f *mockFetcher) Head(_ context.Context, _ string) (int64, string, error) { } // setupTestProxy creates a Proxy with a real DB (SQLite in temp dir) and mock storage/fetcher. -func setupTestProxy(t *testing.T) (*Proxy, *database.DB, *mockStorage, *mockFetcher) { +func setupTestProxy(t testing.TB) (*Proxy, *database.DB, *mockStorage, *mockFetcher) { t.Helper() dir := t.TempDir() @@ -148,7 +148,7 @@ func setupTestProxy(t *testing.T) (*Proxy, *database.DB, *mockStorage, *mockFetc } // seedPackage creates a package, version, and cached artifact in the test DB and storage. -func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) { +func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) { t.Helper() pkg := &database.Package{ From ed540053fadc13f1f6f520b5bca333b64d40ac9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:15:33 +0100 Subject: [PATCH 54/70] Bump zizmorcore/zizmor-action from 0.6.1 to 0.6.2 (#251) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.1 to 0.6.2. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/6fc4b006235f201fdab3722e17240ab420d580e5...3dc1ecc9bcb9e94e9b2c709687979e1298497054) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 4df0e18..ac587dd 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1 + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 From 849500de1ec35442ef675361e4fe3769c86c9d9b Mon Sep 17 00:00:00 2001 From: wickedOne Date: Fri, 14 Aug 2026 11:38:08 +0200 Subject: [PATCH 55/70] fix: decode PURL percent-encoding in versions and package paths (#244) * fix: decode PURL percent-encoding in versions and package paths * review fix --- internal/database/queries.go | 37 ++- internal/database/types.go | 76 +++++- internal/database/version_purl_test.go | 159 ++++++++++++ internal/handler/debian_test.go | 5 + internal/server/api.go | 10 +- internal/server/browse.go | 34 ++- internal/server/browse_test.go | 17 +- internal/server/resolve.go | 95 ++++++- internal/server/resolve_test.go | 115 +++++++-- internal/server/server.go | 27 +- internal/server/server_test.go | 235 ++++++++++++++++++ .../server/templates/pages/browse_source.html | 11 +- .../templates/pages/compare_versions.html | 8 +- .../server/templates/pages/package_show.html | 12 +- .../server/templates/pages/version_show.html | 6 +- 15 files changed, 745 insertions(+), 102 deletions(-) create mode 100644 internal/database/version_purl_test.go diff --git a/internal/database/queries.go b/internal/database/queries.go index b01abe4..9fa5381 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -465,11 +465,14 @@ func (db *DB) GetMostPopularPackages(limit int) ([]PopularPackage, error) { } type RecentPackage struct { - Ecosystem string `db:"ecosystem"` - Name string `db:"name"` - Version string `db:"version"` - CachedAt time.Time `db:"fetched_at"` - Size int64 `db:"size"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + VersionPURL string `db:"version_purl"` + CachedAt time.Time `db:"fetched_at"` + Size int64 `db:"size"` + // Version is derived from VersionPURL rather than selected, so that the + // PURL percent-encoding is decoded (e.g. "%2B" back to "+"). + Version string `db:"-"` } func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { @@ -483,10 +486,10 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { } var packages []RecentPackage - // We need to extract version from the purl since there's no separate version column + // There is no separate version column, so the full version PURL is selected + // and the version is decoded from it in Go. query := db.Rebind(` - SELECT p.ecosystem, p.name, - SUBSTR(v.purl, INSTR(v.purl, '@') + 1) as version, + SELECT p.ecosystem, p.name, v.purl as version_purl, a.fetched_at, COALESCE(a.size, 0) as size FROM artifacts a JOIN versions v ON v.purl = a.version_purl @@ -496,25 +499,13 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { LIMIT ? `) - // For postgres, use different string function - if db.dialect == DialectPostgres { - query = db.Rebind(` - SELECT p.ecosystem, p.name, - SUBSTRING(v.purl FROM POSITION('@' IN v.purl) + 1) as version, - a.fetched_at, COALESCE(a.size, 0) as size - FROM artifacts a - JOIN versions v ON v.purl = a.version_purl - JOIN packages p ON p.purl = v.package_purl - WHERE a.storage_path IS NOT NULL AND a.fetched_at IS NOT NULL - ORDER BY a.fetched_at DESC - LIMIT ? - `) - } - err = db.Select(&packages, query, limit) if err != nil { return nil, err } + for i := range packages { + packages[i].Version = VersionFromPURL(packages[i].VersionPURL) + } return packages, nil } diff --git a/internal/database/types.go b/internal/database/types.go index 3826c7a..5ddb9f3 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "net/url" "strings" "time" ) @@ -47,10 +48,79 @@ type Version struct { // Version extracts the version string from the PURL. // e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21" func (v *Version) Version() string { - if idx := strings.LastIndex(v.PURL, "@"); idx >= 0 { - return v.PURL[idx+1:] + return VersionFromPURL(v.PURL) +} + +// EscapedVersion returns the version escaped for use as a single URL path +// segment. +// +// Version returns decoded text, which is what should be shown to a user but is +// not safe to drop into a link: html/template preserves reserved characters and +// existing escapes in a URL, so "release/1" would split into two path segments, +// "v1?build" would start a query string, and a literal "%2B" would be read back +// as "+". Escaping here and decoding in splitWildcardPath round-trips the value, +// so the link resolves to the version that was stored. +func (v *Version) EscapedVersion() string { + return url.PathEscape(v.Version()) +} + +// DisplayPURL returns the PURL with its path components percent-decoded, for +// showing in the UI. The stored PURL keeps the canonical encoding (which is +// what the API and all lookups use); this is only a readable rendering, so that +// a version like "7.91+dfsg1-2ubuntu0.1" is not shown as "7.91%2Bdfsg1-2ubuntu0.1" +// and an npm scope is shown as "@babel" rather than "%40babel". Qualifiers and +// subpath keep their encoding, since decoding those would be ambiguous. +func (v *Version) DisplayPURL() string { + base, suffix := v.PURL, "" + if i := strings.IndexAny(base, "?#"); i >= 0 { + base, suffix = base[:i], base[i:] } - return "" + + name, version := base, "" + if idx := strings.LastIndex(base, "@"); idx >= 0 { + name, version = base[:idx], "@"+decodePURLComponent(base[idx+1:]) + } + + parts := strings.Split(name, "/") + for i, part := range parts { + parts[i] = decodePURLComponent(part) + } + return strings.Join(parts, "/") + version + suffix +} + +// VersionFromPURL extracts the decoded version string from a PURL. +// +// PURL percent-encodes characters that are not safe in a path component, so a +// Debian version like "7.91+dfsg1-2ubuntu0.1" is stored as +// "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1". The raw substring after "@" is +// therefore not the version: it must be percent-decoded before being displayed +// or used to build a URL, otherwise "%2B" leaks into the UI and round-tripping +// the value back into a PURL double-encodes it. +// +// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21" +func VersionFromPURL(p string) string { + // Qualifiers ("?key=value") and subpath ("#path") follow the version. + if i := strings.IndexAny(p, "?#"); i >= 0 { + p = p[:i] + } + idx := strings.LastIndex(p, "@") + if idx < 0 { + return "" + } + return decodePURLComponent(p[idx+1:]) +} + +// decodePURLComponent percent-decodes a single PURL path component, returning +// the input unchanged if it is not valid percent-encoding. +func decodePURLComponent(s string) string { + if !strings.Contains(s, "%") { + return s + } + decoded, err := url.PathUnescape(s) + if err != nil { + return s + } + return decoded } // Artifact represents a cached artifact in the database. diff --git a/internal/database/version_purl_test.go b/internal/database/version_purl_test.go new file mode 100644 index 0000000..517022b --- /dev/null +++ b/internal/database/version_purl_test.go @@ -0,0 +1,159 @@ +package database + +import ( + "database/sql" + "net/url" + "testing" + "time" +) + +func TestVersionFromPURL(t *testing.T) { + tests := []struct { + name string + purl string + want string + }{ + {"simple", "pkg:npm/lodash@4.17.21", "4.17.21"}, + {"namespaced", "pkg:composer/symfony/console@6.0.0", "6.0.0"}, + // Debian/Ubuntu versions routinely contain "+", which PURL encodes. + {"encoded plus", "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"}, + {"encoded epoch", "pkg:deb/curl@1%3A7.81.0-1", "1:7.81.0-1"}, + {"encoded plus with qualifier", "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", "7.91+dfsg1"}, + {"tilde is not encoded", "pkg:deb/foo@1.0~rc1", "1.0~rc1"}, + {"no version", "pkg:npm/lodash", ""}, + {"invalid escape passed through", "pkg:npm/lodash@1.0%zz", "1.0%zz"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := VersionFromPURL(tt.purl); got != tt.want { + t.Errorf("VersionFromPURL(%q) = %q, want %q", tt.purl, got, tt.want) + } + v := &Version{PURL: tt.purl} + if got := v.Version(); got != tt.want { + t.Errorf("Version.Version() for %q = %q, want %q", tt.purl, got, tt.want) + } + }) + } +} + +// TestVersionEscapedVersion checks the value the templates put in a URL. It +// must survive the round trip back through the router: escaping here and +// decoding per path segment on the way in has to yield the original version. +func TestVersionEscapedVersion(t *testing.T) { + tests := []struct { + name string + purl string + want string + }{ + {"simple", "pkg:npm/lodash@4.17.21", "4.17.21"}, + // "+" is legal in a path segment, so it stays literal and the UI keeps + // showing the version the way Debian writes it. + {"plus stays literal", "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1-2ubuntu0.1"}, + // A slash would otherwise split the version into two path segments. + {"slash", "pkg:golang/example@release%2F1", "release%2F1"}, + // A question mark would otherwise start the query string. + {"question mark", "pkg:npm/example@v1%3Fbuild", "v1%3Fbuild"}, + // A version containing a literal "%2B" is stored double-encoded; the + // link must re-encode it or it decodes back to "+" instead. + {"literal percent escape", "pkg:npm/example@1.0%252B", "1.0%252B"}, + {"space", "pkg:npm/example@1.0%20beta", "1.0%20beta"}, + {"no version", "pkg:npm/lodash", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := &Version{PURL: tt.purl} + got := v.EscapedVersion() + if got != tt.want { + t.Errorf("EscapedVersion() for %q = %q, want %q", tt.purl, got, tt.want) + } + // The router decodes each path segment, which must give back the + // version the page displays. + decoded, err := url.PathUnescape(got) + if err != nil { + t.Fatalf("PathUnescape(%q) failed: %v", got, err) + } + if decoded != v.Version() { + t.Errorf("round trip for %q = %q, want %q", tt.purl, decoded, v.Version()) + } + }) + } +} + +func TestVersionDisplayPURL(t *testing.T) { + tests := []struct { + name string + purl string + want string + }{ + {"simple", "pkg:npm/lodash@4.17.21", "pkg:npm/lodash@4.17.21"}, + { + "encoded plus", + "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", + "pkg:deb/nmap@7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", + }, + { + "qualifier preserved", + "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", + "pkg:deb/nmap@7.91+dfsg1?repository_url=http%3A%2F%2Fexample.com", + }, + // The namespace is encoded too: MakePURLString("npm", "@babel/core", …) + // produces "pkg:npm/%40babel/core@…". + {"encoded npm scope", "pkg:npm/%40babel/core@7.0.0", "pkg:npm/@babel/core@7.0.0"}, + {"encoded scope without version", "pkg:npm/%40babel/core", "pkg:npm/@babel/core"}, + {"no version", "pkg:npm/lodash", "pkg:npm/lodash"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := &Version{PURL: tt.purl} + if got := v.DisplayPURL(); got != tt.want { + t.Errorf("DisplayPURL() for %q = %q, want %q", tt.purl, got, tt.want) + } + }) + } +} + +// TestGetRecentlyCachedPackagesDecodesVersion guards the dashboard's "recently +// cached" list, which derives the version from the version PURL. +func TestGetRecentlyCachedPackagesDecodesVersion(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1" + + if err := db.UpsertPackage(&Package{ + PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap", + }); err != nil { + t.Fatalf("UpsertPackage failed: %v", err) + } + if err := db.UpsertVersion(&Version{ + PURL: versionPURL, PackagePURL: "pkg:deb/nmap", + }); err != nil { + t.Fatalf("UpsertVersion failed: %v", err) + } + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: versionPURL, + Filename: "nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb", + UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb", + StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact failed: %v", err) + } + + recent, err := db.GetRecentlyCachedPackages(10) + if err != nil { + t.Fatalf("GetRecentlyCachedPackages failed: %v", err) + } + if len(recent) != 1 { + t.Fatalf("expected 1 recent package, got %d", len(recent)) + } + const want = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1" + if recent[0].Version != want { + t.Errorf("Version = %q, want %q", recent[0].Version, want) + } + if recent[0].VersionPURL != versionPURL { + t.Errorf("VersionPURL = %q, want %q", recent[0].VersionPURL, versionPURL) + } + }) +} diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index 60fa23a..b086fdf 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -12,6 +12,11 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { {"pool/main/libn/libncurses/libncurses6_6.2-1_amd64.deb", "libncurses6", "6.2-1", "amd64"}, {"pool/contrib/v/virtualbox/virtualbox_6.1.38-1_amd64.deb", "virtualbox", "6.1.38-1", "amd64"}, {"pool/main/g/git/git_2.39.2-1_arm64.deb", "git", "2.39.2-1", "arm64"}, + { + "pool/universe/n/nmap/nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb", + "nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64", + }, + {"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"}, {"invalid/path", "", "", ""}, {"pool/main/n/nginx/nginx.deb", "", "", ""}, }) diff --git a/internal/server/api.go b/internal/server/api.go index ddb9ca7..992d736 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -139,12 +139,11 @@ type BulkResponse struct { // Resolves namespaced package names (Composer vendor/name, npm @scope/name) from the path. func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { badRequest(w, err.Error()) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) == 0 { badRequest(w, "ecosystem and name are required") @@ -277,12 +276,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste // Supports both {name} and {name}/{version} paths with namespaced package names. func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { badRequest(w, err.Error()) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) == 0 { badRequest(w, "ecosystem and name are required") diff --git a/internal/server/browse.go b/internal/server/browse.go index c60cba3..43ad9ae 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -147,12 +147,11 @@ type BrowseFileInfo struct { // {name}/{version}/file/{path} -> browse file func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { badRequest(w, err.Error()) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) < 2 { badRequest(w, "ecosystem, name, and version required") @@ -203,12 +202,11 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) { // Supported paths: {name}/{fromVersion}/{toVersion} func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { badRequest(w, err.Error()) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) < 3 { badRequest(w, "ecosystem, name, fromVersion, and toVersion required") @@ -506,11 +504,16 @@ func isLikelyText(filename string) bool { } // BrowseSourceData contains data for the browse source page. +// +// Version is the decoded version, for display. EscapedVersion is the same value +// escaped as a single URL path segment and is what the links and the browse API +// calls must use; see database.Version.EscapedVersion. type BrowseSourceData struct { Layout - Ecosystem string - PackageName string - Version string + Ecosystem string + PackageName string + Version string + EscapedVersion string } // handleBrowseSource is now showBrowseSource in server.go, dispatched via handlePackagePath. @@ -601,12 +604,17 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, } // ComparePageData contains data for the version comparison page. +// +// FromVersion and ToVersion are decoded, for display; the Escaped variants are +// the path-segment form used to build the compare API URL. type ComparePageData struct { Layout - Ecosystem string - PackageName string - FromVersion string - ToVersion string + Ecosystem string + PackageName string + FromVersion string + ToVersion string + EscapedFromVersion string + EscapedToVersion string } // handleComparePage is now showComparePage in server.go, dispatched via handlePackagePath. diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index 6ea0f7e..3cc37c8 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -450,8 +450,10 @@ func TestHandleBrowseSourcePage(t *testing.T) { if !strings.Contains(body, "const packageName = 'test-browse'") { t.Error("browse source page missing packageName variable") } - if !strings.Contains(body, "const version = '1.0.0'") { - t.Error("browse source page missing version variable") + // The version reaches the browse API as one path segment, so the page holds + // its escaped form. + if !strings.Contains(body, "const versionPath = '1.0.0'") { + t.Error("browse source page missing versionPath variable") } // Verify content type @@ -617,12 +619,13 @@ func TestHandleComparePage(t *testing.T) { body := w.Body.String() - // Check that versions are set correctly in JavaScript - if !strings.Contains(body, "const fromVersion = '1.0.0'") { - t.Error("page should set fromVersion") + // Check that versions are set correctly in JavaScript. The compare API takes + // each version as a path segment, so the page holds their escaped forms. + if !strings.Contains(body, "const fromVersionPath = '1.0.0'") { + t.Error("page should set fromVersionPath") } - if !strings.Contains(body, "const toVersion = '2.0.0'") { - t.Error("page should set toVersion") + if !strings.Contains(body, "const toVersionPath = '2.0.0'") { + t.Error("page should set toVersionPath") } // Test invalid format (missing separator) diff --git a/internal/server/resolve.go b/internal/server/resolve.go index 51f203d..f7a1b23 100644 --- a/internal/server/resolve.go +++ b/internal/server/resolve.go @@ -2,10 +2,13 @@ package server import ( "fmt" + "net/http" + "net/url" "strings" "unicode" "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" ) // maxPackagePathLen bounds the wildcard portion of package routes (name plus @@ -13,22 +16,69 @@ import ( // longer, so 512 leaves room without admitting pathological inputs. const maxPackagePathLen = 512 +// packagePathSegments validates the wildcard portion of a package route and +// splits it into decoded path segments. +func packagePathSegments(r *http.Request) ([]string, error) { + wildcard := chi.URLParam(r, "*") + encoded := wildcardIsEncoded(r) + if err := validatePackagePath(wildcard, encoded); err != nil { + return nil, err + } + + return splitWildcardPath(wildcard, encoded), nil +} + +// wildcardIsEncoded reports whether the chi wildcard for this request is still +// percent-encoded. +// +// chi routes on r.URL.RawPath when it is set and on r.URL.Path otherwise, and +// net/url only sets RawPath when the request's escaping differs from the +// canonical encoding of the decoded path. A version such as "release%2F1" is +// therefore routed raw, while "1.0%252B" (a version whose text contains a +// literal "%2B") encodes canonically and arrives already decoded once. The +// distinction decides whether the segments still need decoding: decoding the +// second case again would turn it into "1.0+" and resolve a different version. +func wildcardIsEncoded(r *http.Request) bool { + return r.URL.RawPath != "" +} + // validatePackagePath rejects wildcard package paths that cannot be valid in // any supported ecosystem. It is a coarse filter applied before database or // enrichment lookups; ecosystem-specific name rules are layered on top. -func validatePackagePath(path string) error { +// +// encoded has the meaning described on wildcardIsEncoded. +func validatePackagePath(path string, encoded bool) error { if path == "" { return fmt.Errorf("package name required") } if len(path) > maxPackagePathLen { return fmt.Errorf("package path exceeds %d bytes", maxPackagePathLen) } - for _, r := range path { - if r == 0 { - return fmt.Errorf("package path contains null byte") - } - if unicode.IsControl(r) { - return fmt.Errorf("package path contains control character %#U", r) + // Validate the decoded segments: the handlers work with decoded values, so + // an escape such as "%00" or "%2E%2E" must not slip past these checks. + for _, seg := range splitWildcardPath(path, encoded) { + // Each segment is checked both as the handlers see it and decoded once + // more: a segment can reach a handler with escapes intact, and the + // upstream registry is then the one that decodes them. + for _, value := range []string{seg, decodePathSegment(seg)} { + // A decoded segment can itself contain slashes (from "%2F"), and + // the segments are later rejoined into a package name that + // registries interpolate straight into an upstream URL. Check every + // path element, not just the segment as a whole, or + // "a%2F..%2F..%2Fb" traverses. + for _, elem := range strings.Split(value, "/") { + if elem == ".." { + return fmt.Errorf("package path contains parent directory segment") + } + } + for _, r := range value { + if r == 0 { + return fmt.Errorf("package path contains null byte") + } + if unicode.IsControl(r) { + return fmt.Errorf("package path contains control character %#U", r) + } + } } } return nil @@ -60,10 +110,37 @@ func resolvePackageName(db *database.DB, ecosystem string, segments []string) (n // splitWildcardPath splits a chi wildcard path value into segments, // trimming any leading/trailing slashes. -func splitWildcardPath(path string) []string { +// +// When encoded is set the value is still percent-encoded (see +// wildcardIsEncoded), so each segment is decoded after splitting. Splitting +// first keeps an encoded "%2F" inside a name from being mistaken for a +// separator. Decoding matters for versions such as "1.0%2Bbuild1", which must +// reach the handlers as "1.0+build1" so that rebuilding the PURL yields the +// value that was stored rather than a double-encoded one. +func splitWildcardPath(path string, encoded bool) []string { path = strings.Trim(path, "/") if path == "" { return nil } - return strings.Split(path, "/") + segments := strings.Split(path, "/") + if !encoded { + return segments + } + for i, seg := range segments { + segments[i] = decodePathSegment(seg) + } + return segments +} + +// decodePathSegment percent-decodes a single URL path segment, returning it +// unchanged if it is not valid percent-encoding. +func decodePathSegment(seg string) string { + if !strings.Contains(seg, "%") { + return seg + } + decoded, err := url.PathUnescape(seg) + if err != nil { + return seg + } + return decoded } diff --git a/internal/server/resolve_test.go b/internal/server/resolve_test.go index dd7d2dc..1867f46 100644 --- a/internal/server/resolve_test.go +++ b/internal/server/resolve_test.go @@ -1,12 +1,15 @@ package server import ( + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" "testing" "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" ) func newTestDB(t *testing.T) (*database.DB, func()) { @@ -95,26 +98,44 @@ func TestResolvePackageName(t *testing.T) { func TestSplitWildcardPath(t *testing.T) { tests := []struct { - input string - want []string + input string + encoded bool + want []string }{ - {"lodash", []string{"lodash"}}, - {"lodash/4.17.21", []string{"lodash", "4.17.21"}}, - {"monolog/monolog", []string{"monolog", "monolog"}}, - {"symfony/console/6.0.0/browse", []string{"symfony", "console", "6.0.0", "browse"}}, - {"", nil}, - {"/", nil}, + {"lodash", false, []string{"lodash"}}, + {"lodash/4.17.21", false, []string{"lodash", "4.17.21"}}, + {"monolog/monolog", false, []string{"monolog", "monolog"}}, + {"symfony/console/6.0.0/browse", false, []string{"symfony", "console", "6.0.0", "browse"}}, + {"", false, nil}, + {"/", false, nil}, + // chi routes on the raw path when it differs from the canonical + // encoding of the decoded path, so segments arrive percent-encoded and + // must be decoded. + { + "nmap/7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", true, + []string{"nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"}, + }, + {"%40babel/core/7.0.0", true, []string{"@babel", "core", "7.0.0"}}, + // An encoded separator stays inside its segment rather than splitting. + {"vendor%2Fname/1.0.0", true, []string{"vendor/name", "1.0.0"}}, + // Invalid escapes are passed through untouched. + {"lodash/1.0%zz", true, []string{"lodash", "1.0%zz"}}, + // When chi routed on the already-decoded path, an escape that survived + // is part of the value: a version whose text is "1.0%2B" reaches here + // as "1.0%2B" and decoding it again would yield "1.0+". + {"nmap/1.0%2B", false, []string{"nmap", "1.0%2B"}}, } for _, tt := range tests { - got := splitWildcardPath(tt.input) + got := splitWildcardPath(tt.input, tt.encoded) if len(got) != len(tt.want) { - t.Errorf("splitWildcardPath(%q) = %v, want %v", tt.input, got, tt.want) + t.Errorf("splitWildcardPath(%q, %v) = %v, want %v", tt.input, tt.encoded, got, tt.want) continue } for i := range got { if got[i] != tt.want[i] { - t.Errorf("splitWildcardPath(%q)[%d] = %q, want %q", tt.input, i, got[i], tt.want[i]) + t.Errorf("splitWildcardPath(%q, %v)[%d] = %q, want %q", + tt.input, tt.encoded, i, got[i], tt.want[i]) } } } @@ -132,8 +153,19 @@ func TestValidatePackagePath(t *testing.T) { {"composer namespaced", "symfony/console/6.0.0", false}, {"maven coordinates", "org.apache.commons/commons-lang3/3.12.0", false}, {"unicode", "café/1.0.0", false}, + {"encoded plus in version", "nmap/7.91%2Bdfsg1-2ubuntu0.1", false}, {"empty", "", true}, {"null byte", "lodash\x00/4.17.21", true}, + {"encoded null byte", "lodash/%00", true}, + {"encoded newline", "lodash/1.0%0A", true}, + {"parent segment", "lodash/../4.17.21", true}, + {"encoded parent segment", "lodash/%2E%2E/4.17.21", true}, + // A decoded segment can contain slashes, so traversal can hide inside + // one segment. Registries interpolate the resolved name straight into + // an upstream URL, and Go sends dot-segments verbatim. + {"traversal inside one segment", "pkg%2F..%2F..%2Fadmin", true}, + {"traversal via encoded dots and slash", "pkg%2f%2e%2e%2fadmin", true}, + {"encoded slash alone is allowed", "vendor%2Fname/1.0.0", false}, {"null byte suffix", "lodash\x00", true}, {"newline", "lodash\n4.17.21", true}, {"carriage return", "lodash\r", true}, @@ -145,9 +177,64 @@ func TestValidatePackagePath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := validatePackagePath(tt.path) - if (err != nil) != tt.wantErr { - t.Errorf("validatePackagePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr) + // The verdict must not depend on whether chi routed on the raw or + // on the already-decoded path: an escape that reaches a handler + // undecoded is decoded by the upstream registry instead, so it is + // rejected either way. + for _, encoded := range []bool{false, true} { + err := validatePackagePath(tt.path, encoded) + if (err != nil) != tt.wantErr { + t.Errorf("validatePackagePath(%q, %v) error = %v, wantErr %v", + tt.path, encoded, err, tt.wantErr) + } + } + }) + } +} + +// TestPackagePathSegments drives the real router, which is what decides whether +// the wildcard still carries percent-encoding. Go decodes the request path +// itself unless the escaping is non-canonical, so the same version can arrive +// either way and only one of the two forms may be decoded again. +func TestPackagePathSegments(t *testing.T) { + tests := []struct { + name string + target string + want []string + }{ + {"plain", "/pkg/npm/lodash/4.17.21", []string{"lodash", "4.17.21"}}, + {"encoded plus", "/pkg/deb/nmap/7.91%2Bdfsg1-2ubuntu0.1", []string{"nmap", "7.91+dfsg1-2ubuntu0.1"}}, + {"decoded plus", "/pkg/deb/nmap/7.91+dfsg1-2ubuntu0.1", []string{"nmap", "7.91+dfsg1-2ubuntu0.1"}}, + // An encoded slash is one segment, not a separator. + {"encoded slash", "/pkg/composer/vendor%2Fname/1.0.0", []string{"vendor/name", "1.0.0"}}, + {"question mark", "/pkg/npm/example/v1%3Fbuild", []string{"example", "v1?build"}}, + // "1.0%252B" is the escaped form of the version "1.0%2B"; net/url + // already decoded it once, so it must not be decoded again. + {"literal percent escape", "/pkg/npm/example/1.0%252B", []string{"example", "1.0%2B"}}, + {"browse suffix", "/pkg/deb/nmap/7.91%2Bdfsg1/browse", []string{"nmap", "7.91+dfsg1", "browse"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got []string + var gotErr error + + router := chi.NewRouter() + router.Get("/pkg/{ecosystem}/*", func(_ http.ResponseWriter, r *http.Request) { + got, gotErr = packagePathSegments(r) + }) + router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", tt.target, nil)) + + if gotErr != nil { + t.Fatalf("packagePathSegments(%q) failed: %v", tt.target, gotErr) + } + if len(got) != len(tt.want) { + t.Fatalf("segments for %q = %v, want %v", tt.target, got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("segments for %q [%d] = %q, want %q", tt.target, i, got[i], tt.want[i]) + } } }) } diff --git a/internal/server/server.go b/internal/server/server.go index c98d1cb..e677bc9 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -54,6 +54,7 @@ import ( "fmt" "log/slog" "net/http" + "net/url" "strconv" "strings" "time" @@ -680,12 +681,11 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) { // {name}/compare/{v1}...{v2} -> compare versions func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) == 0 { http.Error(w, "ecosystem and package name required", http.StatusBadRequest) @@ -825,10 +825,11 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, func (s *Server) showBrowseSource(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { data := BrowseSourceData{ - Layout: s.layoutFor(r), - Ecosystem: ecosystem, - PackageName: name, - Version: version, + Layout: s.layoutFor(r), + Ecosystem: ecosystem, + PackageName: name, + Version: version, + EscapedVersion: url.PathEscape(version), } if err := s.templates.Render(w, "browse_source", data); err != nil { @@ -846,11 +847,13 @@ func (s *Server) showComparePage(w http.ResponseWriter, r *http.Request, ecosyst } data := ComparePageData{ - Layout: s.layoutFor(r), - Ecosystem: ecosystem, - PackageName: name, - FromVersion: parts[0], - ToVersion: parts[1], + Layout: s.layoutFor(r), + Ecosystem: ecosystem, + PackageName: name, + FromVersion: parts[0], + ToVersion: parts[1], + EscapedFromVersion: url.PathEscape(parts[0]), + EscapedToVersion: url.PathEscape(parts[1]), } if err := s.templates.Render(w, "compare_versions", data); err != nil { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 2d27147..98b58cc 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -4,12 +4,16 @@ import ( "database/sql" "encoding/json" "fmt" + "html" "io" "log/slog" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" + "regexp" + "strconv" "strings" "testing" "time" @@ -18,6 +22,7 @@ import ( "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" "github.com/go-chi/chi/v5" ) @@ -764,6 +769,236 @@ func TestVersionShowPage_NotFoundServer(t *testing.T) { } } +// TestVersionShowPage_PlusInVersion covers Debian/Ubuntu style versions such as +// nmap's "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1". PURL percent-encodes "+" as +// "%2B", so the UI must show the decoded version and resolve both the decoded +// and the still-encoded form of the URL back to the same version. +func TestVersionShowPage_PlusInVersion(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + const version = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1" + const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1" + + pkg := &database.Package{PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap"} + if err := ts.db.UpsertPackage(pkg); err != nil { + t.Fatalf("failed to upsert package: %v", err) + } + if err := ts.db.UpsertVersion(&database.Version{ + PURL: versionPURL, PackagePURL: pkg.PURL, + }); err != nil { + t.Fatalf("failed to upsert version: %v", err) + } + + // The package page must link to and display the decoded version. + req := httptest.NewRequest("GET", "/ui/package/deb/nmap", nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("package page: expected status 200, got %d", w.Code) + } + body := w.Body.String() + if strings.Contains(body, "%2B") { + t.Error("package page leaks PURL percent-encoding into the UI") + } + // html/template renders "+" as the "+" entity inside attributes and text. + if !strings.Contains(body, "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1") { + t.Error("expected package page to show the decoded version") + } + + // Both the decoded and the encoded URL must reach the version page. + for _, path := range []string{ + "/ui/package/deb/nmap/" + version, + "/ui/package/deb/nmap/7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", + } { + req := httptest.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("GET %s: expected status 200, got %d", path, w.Code) + } + } +} + +// TestVersionURLEscaping covers versions whose characters are significant in a +// URL path: "/" splits off another path segment, "?" starts a query string, and +// a literal "%xx" is read back as the character it encodes. The pages show the +// decoded version but must build every link from a separately escaped value, +// and those links have to resolve back to the same version. +func TestVersionURLEscaping(t *testing.T) { + // A second version is needed for the compare controls to be rendered. + const otherVersion = "1.0.0" + + tests := []struct { + name string + version string + }{ + {"slash", "release/1"}, + {"question mark", "v1?build"}, + {"literal percent escape", "1.0%2B"}, + {"plus", "7.91+dfsg1-2ubuntu0.1"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedEscapingVersions(t, ts.db, tt.version, otherVersion) + + // The package page links to the escaped version. + escaped := url.PathEscape(tt.version) + versionPath := "/ui/package/deb/nmap/" + escaped + packagePage := ts.getOK(t, "/ui/package/deb/nmap") + if !containsValue(attrValues(packagePage, "href"), versionPath) { + t.Fatalf("package page has no link to %q; hrefs: %v", + versionPath, attrValues(packagePage, "href")) + } + + ts.checkVersionAndBrowsePages(t, versionPath, tt.version, escaped) + ts.checkComparePage(t, packagePage, tt.version, escaped, otherVersion) + }) + } +} + +// seedEscapingVersions stores a Debian package with the given versions, each +// with a cached artifact so that the version page offers its browse link. +func seedEscapingVersions(t *testing.T, db *database.DB, versions ...string) { + t.Helper() + + pkg := &database.Package{PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap"} + if err := db.UpsertPackage(pkg); err != nil { + t.Fatalf("failed to upsert package: %v", err) + } + for _, v := range versions { + versionPURL := purl.MakePURLString("deb", "nmap", v) + if err := db.UpsertVersion(&database.Version{ + PURL: versionPURL, PackagePURL: pkg.PURL, + }); err != nil { + t.Fatalf("failed to upsert version %q: %v", v, err) + } + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: "nmap.deb", + UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb", + StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }); err != nil { + t.Fatalf("failed to upsert artifact for %q: %v", v, err) + } + } +} + +// checkVersionAndBrowsePages follows a version link from the package page and +// then the browse link from the version page, checking that both resolve to the +// stored version and display it decoded. +func (ts *testServer) checkVersionAndBrowsePages(t *testing.T, versionPath, version, escaped string) { + t.Helper() + + versionPage := ts.getOK(t, versionPath) + wantPURL := "pkg:deb/nmap@" + version + if !strings.Contains(html.UnescapeString(versionPage), wantPURL) { + t.Errorf("version page does not show %q", wantPURL) + } + + browsePath := versionPath + "/browse" + if !containsValue(attrValues(versionPage, "href"), browsePath) { + t.Fatalf("version page has no browse link to %q; hrefs: %v", + browsePath, attrValues(versionPage, "href")) + } + + browsePage := ts.getOK(t, browsePath) + if !strings.Contains(html.UnescapeString(browsePage), "nmap@"+version) { + t.Errorf("browse page does not show the decoded version %q", version) + } + // The browse API is called with the escaped version, not with the text shown + // in the heading. + if got := jsConstant(t, browsePage, "versionPath"); got != escaped { + t.Errorf("browse page passes %q to the browse API, want %q", got, escaped) + } +} + +// checkComparePage builds the compare URL the way the package page's script +// does, from the values its checkboxes carry, and checks the page it reaches. +func (ts *testServer) checkComparePage(t *testing.T, packagePage, version, escaped, otherVersion string) { + t.Helper() + + selectable := attrValues(packagePage, "data-version-path") + if !containsValue(selectable, escaped) { + t.Fatalf("package page compare data holds %v, want %q", selectable, escaped) + } + + comparePage := ts.getOK(t, "/ui/package/deb/nmap/compare/"+escaped+"..."+otherVersion) + decoded := html.UnescapeString(comparePage) + for _, want := range []string{version, otherVersion} { + if !strings.Contains(decoded, want) { + t.Errorf("compare page does not show version %q", want) + } + } + if got := jsConstant(t, comparePage, "fromVersionPath"); got != escaped { + t.Errorf("compare page passes %q to the compare API, want %q", got, escaped) + } +} + +// getOK performs a GET against the server and fails the test unless it returns +// 200, returning the response body. +func (ts *testServer) getOK(t *testing.T, path string) string { + t.Helper() + + req := httptest.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("GET %s: expected status 200, got %d", path, w.Code) + } + + return w.Body.String() +} + +// attrValues returns the value of every occurrence of an HTML attribute in a +// rendered page, with HTML entities resolved so that values can be compared +// against the raw strings they were built from. +func attrValues(body, attr string) []string { + re := regexp.MustCompile(regexp.QuoteMeta(attr) + `="([^"]*)"`) + + var values []string + for _, match := range re.FindAllStringSubmatch(body, -1) { + values = append(values, html.UnescapeString(match[1])) + } + + return values +} + +// jsConstant returns the value of a single-quoted JavaScript string constant in +// a rendered page. html/template escapes characters that are significant in +// JavaScript, rendering "+" as "\\u002b" for instance, so the escapes are +// resolved to recover the value the page actually uses. +func jsConstant(t *testing.T, body, name string) string { + t.Helper() + + re := regexp.MustCompile(`const ` + regexp.QuoteMeta(name) + ` = '([^']*)'`) + match := re.FindStringSubmatch(body) + if match == nil { + t.Fatalf("page does not declare the constant %q", name) + } + + unescaped, err := strconv.Unquote(`"` + match[1] + `"`) + if err != nil { + t.Fatalf("cannot unescape %q: %v", match[1], err) + } + + return unescaped +} + +func containsValue(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + + return false +} + func TestPackageShowPage_WithLicense(t *testing.T) { ts := newTestServer(t) defer ts.close() diff --git a/internal/server/templates/pages/browse_source.html b/internal/server/templates/pages/browse_source.html index ca06652..a949111 100644 --- a/internal/server/templates/pages/browse_source.html +++ b/internal/server/templates/pages/browse_source.html @@ -7,7 +7,7 @@ / {{.PackageName}} / - {{.Version}} + {{.Version}} / Browse Source @@ -51,7 +51,10 @@