About
- git-pkgs proxy is a caching proxy for package registries supporting 17+ ecosystems. + git-pkgs proxy is a caching proxy for package registries supporting 16+ ecosystems.
-- - github.com/git-pkgs/proxy - -
- {{if .BuildInfo.Version}} -- proxy {{.BuildInfo.Version}}{{if .BuildInfo.Commit}} ({{.BuildInfo.Commit}}){{end}} -
- {{end}}Resources
Supported Ecosystems
- {{range supportedEcosystems}}
- {{template "ecosystem_badge" .}}
- {{end}}
+ npm
+ cargo
+ gem
+ go
+ pypi
+ maven
+
{{end}}
diff --git a/internal/server/templates/layout/header.html b/internal/server/templates/layout/header.html
index 8b4d94f..d893565 100644
--- a/internal/server/templates/layout/header.html
+++ b/internal/server/templates/layout/header.html
@@ -1,21 +1,31 @@
{{define "header"}}
+ Powered by git-pkgs +
+
-
-
+
@@ -28,7 +38,7 @@
+
-
{{end}}
-
-{{define "search_form"}}
-
-{{end}}
-
-{{define "nav_links"}}
-{{if .Projects}}Projects{{end}}
-Analytics
-Install
-Health
-API
-{{end}}
diff --git a/internal/server/templates/layout/styles.html b/internal/server/templates/layout/styles.html
index 76e9d9e..1d21184 100644
--- a/internal/server/templates/layout/styles.html
+++ b/internal/server/templates/layout/styles.html
@@ -1,4 +1,25 @@
{{define "styles"}}
+
+
{{end}}
diff --git a/internal/server/templates/pages/analytics.html b/internal/server/templates/pages/analytics.html
deleted file mode 100644
index cda81bb..0000000
--- a/internal/server/templates/pages/analytics.html
+++ /dev/null
@@ -1,354 +0,0 @@
-{{define "title"}}Analytics · git-pkgs proxy{{end}}
-
-{{define "head"}}
-
-{{end}}
-
-{{define "content"}}
-
-
-
-{{if .StatsStale}}
-Analytics
-- Accumulated download volume and cache composition across - {{.Totals.Ecosystems}} ecosystem{{if ne .Totals.Ecosystems 1}}s{{end}}. -
-
- Figures may be out of date.
- The database query is failing, so the download and cache numbers below are
- the last snapshot that could be read{{if .StatsAge}}, from {{.StatsAge}}{{end}}.
- Check the proxy logs.
-
-{{end}}
-
-
-
-
-
-
-
- {{if .Donut.HasSlices}}
- Download size by ecosystem
-Bytes served from cache
-
-
- {{else if .StatsFailed}}
-
-
-
-
-
-
-
- {{.Donut.CenterValue}}
- {{.Donut.CenterLabel}}
- -
- {{range .Donut.Slices}}
-
- - - - {{.Label}} - {{- if .IsOther}}({{.Members}}){{end}} - - {{.Value}} - {{.SharePct}}% - - {{end}} -
- Download figures are unavailable — the database query failed. Check the proxy logs.
-
- {{else}}
- Nothing has been served from cache yet
- {{end}}
-
-
-
-{{if .Totals.Amplification}}
-
-
- Downloads
- {{.Totals.Downloads}}
-
-
- Cache size
- {{.Totals.CacheSize}}
-
-
- Cached artifacts
- {{.Totals.CachedArtifacts}}
-
-
- Packages
- {{.Totals.Packages}}
-
-
- Versions
- {{.Totals.Versions}}
-
-
-Ecosystems in use
- {{.Totals.ActiveEcosystems}} / {{.Totals.Ecosystems}}
- - The cache has served {{.Totals.Amplification}} the bytes it currently stores. -
-{{end}} - - -
-
-
-
-
-
- {{if .Ecosystems}}
- Per-ecosystem breakdown
-Every ecosystem, including any folded into the ring's "Other" slice
-
-
-
-
-
- {{else if .StatsFailed}}
- | Ecosystem | -Downloaded | -Share | -Downloads | -Cache size | -Artifacts | -Avg size | -Packages | -Versions | -
|---|---|---|---|---|---|---|---|---|
| - {{template "ecosystem_badge" .Ecosystem}} - | -{{.Downloaded}} | -{{.SharePct}}% | -{{.Downloads}} | -{{.CacheSize}} | -{{.Artifacts}} | -{{.AvgArtifactSize}} | -{{.Packages}} | -{{.Versions}} | -
- The per-ecosystem query failed, so this table is unavailable. Check the proxy logs.
-
- {{else}}
- No packages cached yet
- {{end}}
-
-
-
-{{if .EnrichmentStats.HasVulns}}
-{{template "security_overview" .EnrichmentStats}}
-{{end}}
-
-{{if .Runtime.Available}}
-{{template "runtime_metrics" .Runtime}}
-{{end}}
-
-
-
- {{if .Metadata.Rows}}
- Metadata cache
-
- Cached registry metadata, kept apart from artifacts: the download, cache size and
- artifact figures above do not include it.
- {{if not .Metadata.Enabled}}Metadata caching is off; set cache_metadata to enable it.{{end}}
-
-
-
-
- Cached documents
- {{.Metadata.Entries}}
-
-
- Metadata size
- {{.Metadata.Size}}
-
-
-
-
-
- {{else if .Metadata.Failed}}
- | Ecosystem | -Size | -Share | -Documents | -
|---|---|---|---|
| {{template "ecosystem_badge" .Ecosystem}} | -{{.Size}} | -{{.SharePct}}% | -{{.Entries}} | -
- The metadata cache query failed, so these figures are unavailable. Check the proxy logs.
-
- {{else}}
- No metadata cached yet
- {{end}}
-
- Every figure on this page is also exported for Prometheus at
- /metrics.
- This page shows current state only — nothing here is a time series. For history,
- trends and alerting, scrape /metrics and use the
- Grafana dashboard shipped in deploy/grafana/.
-
@@ -51,24 +51,13 @@
diff --git a/internal/server/templates/pages/compare_versions.html b/internal/server/templates/pages/compare_versions.html
index 79cd89b..a929bb0 100644
--- a/internal/server/templates/pages/compare_versions.html
+++ b/internal/server/templates/pages/compare_versions.html
@@ -3,9 +3,9 @@
{{define "content"}}
@@ -66,22 +66,20 @@
-
+
-
+
+
{{end}}
diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html
deleted file mode 100644
index 2d1b56a..0000000
--- a/internal/server/templates/pages/project_show.html
+++ /dev/null
@@ -1,59 +0,0 @@
-{{define "title"}}{{.Project.Name}} - git-pkgs proxy{{end}}
-
-{{define "content"}}
-
- ← Projects
-
-
-
-{{if .Components}}
-{{.Project.Name}}
-- {{.Project.Components}} packages, uploaded {{.Project.UploadedAt}}{{if .Project.Branch}} from {{.Project.Branch}}{{end}}{{if .Project.Ref}} at {{.Project.Ref}}{{end}} -
- {{if .Outdated}} -
- {{.Outdated}} outdated
-
- {{end}}
-
-
-
-
-
-
-
-{{else}}
-
- {{range .Components}}
-
-
-
- {{end}}
-
- {{template "ecosystem_badge" .Ecosystem}}
- {{if .Cached}}
- {{.Name}}
- {{else}}
- {{.Name}}
- {{end}}
- {{.Version}}
-
-
- {{if .Outdated}}
- outdated, latest {{.LatestVersion}}
- {{end}}
- {{if not .Cached}}
- not cached
- {{end}}
-
-
-
-{{end}}
-{{end}}
diff --git a/internal/server/templates/pages/projects_list.html b/internal/server/templates/pages/projects_list.html
deleted file mode 100644
index d7256da..0000000
--- a/internal/server/templates/pages/projects_list.html
+++ /dev/null
@@ -1,33 +0,0 @@
-{{define "title"}}Projects - git-pkgs proxy{{end}}
-
-{{define "content"}}
-The latest SBOM lists no packages the proxy serves
-
-
-
-{{if .Rows}}
-Projects
-{{len .Rows}} projects, as of their latest SBOM upload
-
-
-{{else}}
-
- {{range .Rows}}
-
-
-
- {{end}}
-
- {{.Name}}
- {{if or .Branch .Ref}}
-
- {{.Branch}}{{if and .Branch .Ref}} @ {{end}}{{.Ref}}
- {{end}}
-
- {{.Components}} packages
- {{.UploadedAt}}
-
-
-
-{{end}}
-{{end}}
diff --git a/internal/server/templates/pages/search.html b/internal/server/templates/pages/search.html
index 42e82c5..2f3051a 100644
--- a/internal/server/templates/pages/search.html
+++ b/internal/server/templates/pages/search.html
@@ -13,9 +13,19 @@
No project has uploaded an SBOM yet
-
{{template "ecosystem_badge" .Ecosystem}}
- {{.Name}}
+ {{.Name}}
{{if .LatestVersion}}
@{{.LatestVersion}}
{{end}}
@@ -50,7 +60,7 @@
{{else}}
{{end}}
diff --git a/internal/server/templates/pages/version_show.html b/internal/server/templates/pages/version_show.html
index 468f24c..dede127 100644
--- a/internal/server/templates/pages/version_show.html
+++ b/internal/server/templates/pages/version_show.html
@@ -1,15 +1,15 @@
-{{define "title"}}{{.Version.DisplayPURL}} - git-pkgs proxy{{end}}
+{{define "title"}}{{.Package.Name}}@{{.Version.PURL}} - git-pkgs proxy{{end}}
{{define "content"}}
{{template "ecosystem_badge" .Package.Ecosystem}}
-
{{.Version.DisplayPURL}}
+{{.Version.PURL}}
{{if .IsOutdated}} outdated {{end}} @@ -22,7 +22,7 @@ {{end}} {{if .HasCachedArtifact}} {{if .Artifacts}} -
-
+
+
-
-{{if .CacheDelete}}
-
-{{end}}
{{else}}
Artifacts ({{len .Artifacts}})
- {{if and .CacheDelete .HasDeletableArtifact}} - - {{end}}
{{range .Artifacts}}
@@ -85,9 +81,6 @@
{{if .ContentHash.Valid}}
@@ -107,9 +100,6 @@
{{end}}
{{if .Size.Valid}}{{.Size.Int64}} bytes{{end}}
{{if gt .HitCount 0}}{{.HitCount}} cache hits{{end}}
- {{if and $.CacheDelete .StoragePath.Valid (ne .Source "local")}}
-
- {{end}}
No artifacts cached yet
diff --git a/internal/server/templates_handlers_test.go b/internal/server/templates_handlers_test.go
deleted file mode 100644
index 1e3f1f2..0000000
--- a/internal/server/templates_handlers_test.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package server
-
-import (
- "io/fs"
- "regexp"
- "testing"
-)
-
-// inlineHandlerWithData matches an inline event handler attribute whose value
-// interpolates a JavaScript template expression. The browser decodes HTML
-// entities in the attribute before running the handler, so escapeHTML cannot
-// keep a quote in the interpolated value from ending the handler's string.
-var inlineHandlerWithData = regexp.MustCompile(`\son[a-z]+\s*=\s*"[^"]*\$\{`)
-
-func TestTemplatesInterpolateNoDataIntoInlineHandlers(t *testing.T) {
- err := fs.WalkDir(templatesFS, "templates", func(path string, d fs.DirEntry, err error) error {
- if err != nil || d.IsDir() {
- return err
- }
- source, err := fs.ReadFile(templatesFS, path)
- if err != nil {
- return err
- }
- for _, match := range inlineHandlerWithData.FindAll(source, -1) {
- t.Errorf("%s: inline handler interpolates data: %s", path, match)
- }
- return nil
- })
- if err != nil {
- t.Fatal(err)
- }
-}
diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go
deleted file mode 100644
index c6fa492..0000000
--- a/internal/server/templates_test.go
+++ /dev/null
@@ -1,742 +0,0 @@
-package server
-
-import (
- "database/sql"
- "io/fs"
- "net/http"
- "net/http/httptest"
- "regexp"
- "strings"
- "testing"
- "time"
-
- "github.com/git-pkgs/proxy/internal/database"
-)
-
-func TestTemplatesRenderAllPages(t *testing.T) {
- templates := &Templates{}
-
- tests := []struct {
- page string
- data any
- }{
- {"dashboard", DashboardData{
- Stats: DashboardStats{
- CachedArtifacts: 42,
- TotalSize: "1.5 GB",
- TotalPackages: 10,
- TotalVersions: 25,
- },
- EnrichmentStats: EnrichmentStatsView{
- EnrichedPackages: 5,
- TotalVulnerabilities: 3,
- CriticalVulns: 1,
- HasVulns: true,
- },
- PopularPackages: []PackageInfo{
- {Ecosystem: "npm", Name: "lodash", Hits: 100, Size: "500 KB", License: "MIT", LicenseCategory: "permissive"},
- },
- RecentPackages: []PackageInfo{
- {Ecosystem: "cargo", Name: "serde", Version: "1.0.0", Size: "200 KB", CachedAt: "1 hour ago"},
- },
- }},
- {"analytics", AnalyticsData{
- Totals: AnalyticsTotals{
- DownloadedBytes: 1_500_000_000,
- Downloaded: "1.4 GB",
- Downloads: "12,004",
- CacheSize: "420.0 MB",
- CachedArtifacts: "1,204",
- Packages: "310",
- Versions: "902",
- Ecosystems: 3,
- ActiveEcosystems: 2,
- Amplification: "3.4x",
- },
- EnrichmentStats: EnrichmentStatsView{TotalVulnerabilities: 3, CriticalVulns: 1, HasVulns: true},
- Ecosystems: []EcosystemRow{
- {Ecosystem: "npm", DownloadedBytes: 1_000_000_000, Downloaded: "953.7 MB", Downloads: "9,000", CacheSize: "300.0 MB", AvgArtifactSize: "120 KB", Artifacts: "900", Packages: "200", Versions: "700", SharePct: "66.7"},
- {Ecosystem: "cargo", DownloadedBytes: 500_000_000, Downloaded: "476.8 MB", Downloads: "3,004", CacheSize: "120.0 MB", AvgArtifactSize: "400 KB", Artifacts: "300", Packages: "100", Versions: "200", SharePct: "33.3"},
- {Ecosystem: "rpm", Downloaded: "0 B", Downloads: "0", CacheSize: "0 B", AvgArtifactSize: "0 B", Artifacts: "0", Packages: "10", Versions: "2", SharePct: "0.0"},
- },
- Donut: donutView([]database.EcosystemStats{
- {Ecosystem: "npm", CacheSize: 300, Downloads: 9000, DownloadedBytes: 1_000_000_000},
- {Ecosystem: "cargo", CacheSize: 120, Downloads: 3004, DownloadedBytes: 500_000_000},
- }, 1_500_000_000, "1.4 GB"),
- Runtime: RuntimeView{
- Available: true,
- Requests: "12,004",
- ActiveRequests: "2",
- RequestMean: "8.1 ms",
- ArtifactCache: CacheLookups{Hits: "9,000", Misses: "1,000", HitRatio: "90.0", HasTraffic: true},
- MetadataCache: CacheLookups{Hits: "4,000", Misses: "500", HitRatio: "88.9", HasTraffic: true},
- MetadataServed: "12.0 MB",
- StatusClasses: []LabelledCount{{Label: "2xx", Count: "11,900"}, {Label: "5xx", Count: "104", Bad: true}},
- StorageOps: []LabelledStat{{Label: "get", Count: "9,000", Mean: "412 µs"}},
- ScanErrors: []LabelledCount{{Label: "npm · clamav · timeout", Count: "3", Bad: true}},
- ScanningOn: true,
- Breakers: []BreakerRow{{Registry: "registry.npmjs.org", State: "closed"}},
- },
- }},
- {"analytics", AnalyticsData{}},
- {"install", struct {
- Layout
- BaseURL string
- Registries []RegistryConfig
- }{
- BaseURL: "http://localhost:8080",
- Registries: getRegistryConfigs("http://localhost:8080"),
- }},
- {"search", SearchPageData{
- Query: "lodash",
- Ecosystem: "npm",
- Results: []SearchResultItem{{Ecosystem: "npm", Name: "lodash", LatestVersion: "4.17.21", Hits: 50, SizeFormatted: "1 MB"}},
- Count: 1,
- Page: 1,
- PerPage: 50,
- TotalPages: 1,
- }},
- {"search", SearchPageData{
- Query: "nothing",
- Results: []SearchResultItem{},
- Count: 0,
- Page: 1,
- PerPage: 50,
- TotalPages: 0,
- }},
- {"packages_list", PackagesListPageData{
- Ecosystem: "",
- SortBy: defaultSortBy,
- Results: []SearchResultItem{{Ecosystem: "npm", Name: "express", Hits: 200, SizeFormatted: "2 MB"}},
- Count: 1,
- TotalPackages: 1,
- EcosystemFilters: []EcosystemFilter{{Ecosystem: "npm", Count: 1}},
- Page: 1,
- PerPage: 50,
- TotalPages: 1,
- }},
- {"package_show", PackageShowData{
- Package: &database.Package{
- PURL: "pkg:npm/lodash",
- Ecosystem: "npm",
- Name: "lodash",
- LatestVersion: sql.NullString{String: "4.17.21", Valid: true},
- License: sql.NullString{String: "MIT", Valid: true},
- },
- Versions: []PackageVersion{
- {Version: database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}},
- {
- Version: database.Version{PURL: "pkg:npm/lodash@4.17.22", PackagePURL: "pkg:npm/lodash"},
- AvailableAt: time.Date(2026, 1, 2, 3, 4, 0, 0, time.UTC),
- },
- },
- Vulnerabilities: []database.Vulnerability{},
- LicenseCategory: "permissive",
- }},
- {"package_show", PackageShowData{
- Package: &database.Package{
- PURL: "pkg:npm/minimal",
- Ecosystem: "npm",
- Name: "minimal",
- },
- Versions: []PackageVersion{},
- Vulnerabilities: []database.Vulnerability{},
- LicenseCategory: "unknown",
- }},
- {"version_show", VersionShowData{
- Package: &database.Package{
- PURL: "pkg:npm/lodash",
- Ecosystem: "npm",
- Name: "lodash",
- LatestVersion: sql.NullString{String: "4.17.21", Valid: true},
- License: sql.NullString{String: "MIT", Valid: true},
- },
- Version: &database.Version{
- PURL: "pkg:npm/lodash@4.17.20",
- PackagePURL: "pkg:npm/lodash",
- },
- Artifacts: []database.Artifact{
- {
- VersionPURL: "pkg:npm/lodash@4.17.20",
- Filename: "lodash-4.17.20.tgz",
- StoragePath: sql.NullString{String: "npm/lodash/4.17.20/lodash-4.17.20.tgz", Valid: true},
- Size: sql.NullInt64{Int64: 1024, Valid: true},
- HitCount: 42,
- },
- },
- Vulnerabilities: []database.Vulnerability{},
- IsOutdated: true,
- LicenseCategory: "permissive",
- HasCachedArtifact: true,
- }},
- {"browse_source", BrowseSourceData{
- Ecosystem: "npm",
- PackageName: "lodash",
- Version: "4.17.21",
- }},
- {"compare_versions", ComparePageData{
- Ecosystem: "npm",
- PackageName: "lodash",
- FromVersion: "4.17.20",
- ToVersion: "4.17.21",
- }},
- }
-
- for _, tt := range tests {
- t.Run(tt.page, func(t *testing.T) {
- w := httptest.NewRecorder()
- err := templates.Render(w, tt.page, tt.data)
- if err != nil {
- t.Fatalf("Render(%q) failed: %v", tt.page, err)
- }
- if w.Code != http.StatusOK {
- t.Errorf("status = %d, want %d", w.Code, http.StatusOK)
- }
- if w.Header().Get("Content-Type") != "text/html; charset=utf-8" {
- t.Errorf("Content-Type = %q, want text/html", w.Header().Get("Content-Type"))
- }
- body := w.Body.String()
- if body == "" {
- t.Error("rendered page is empty")
- }
- if !strings.Contains(body, "") && !strings.Contains(body, "`,
- ``,
- ``,
- }
- for _, s := range want {
- if !strings.Contains(body, s) {
- t.Errorf("rendered body missing %q", s)
- }
- }
-}
-
-func TestFooterUsesBuildInfoWhenPageDefinesVersion(t *testing.T) {
- templates := &Templates{}
- buildInfo := BuildInfo{Version: "proxy-build-1.2.3", Commit: "abc123def"}
- wantFooter := "proxy proxy-build-1.2.3 (abc123def)"
-
- tests := []struct {
- name string
- page string
- data any
- shadow string
- }{
- {
- name: "version show page",
- page: "version_show",
- data: VersionShowData{
- Layout: Layout{BuildInfo: buildInfo},
- Package: &database.Package{
- PURL: "pkg:npm/lodash",
- Ecosystem: "npm",
- Name: "lodash",
- },
- Version: &database.Version{
- PURL: "pkg:npm/lodash@9.9.9",
- PackagePURL: "pkg:npm/lodash",
- },
- },
- shadow: "9.9.9",
- },
- {
- name: "browse source page",
- page: "browse_source",
- data: BrowseSourceData{
- Layout: Layout{BuildInfo: buildInfo},
- Ecosystem: "npm",
- PackageName: "lodash",
- Version: "9.9.9",
- },
- shadow: "9.9.9",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- w := httptest.NewRecorder()
- if err := templates.Render(w, tt.page, tt.data); err != nil {
- t.Fatalf("Render(%q) failed: %v", tt.page, err)
- }
- body := w.Body.String()
- if !strings.Contains(body, wantFooter) {
- t.Errorf("footer missing build info %q", wantFooter)
- }
- if strings.Contains(body, "proxy "+tt.shadow) {
- t.Errorf("footer used page Version %q instead of BuildInfo", tt.shadow)
- }
- })
- }
-}
-
-func TestRenderOmitsCanonicalWhenUIBaseURLUnset(t *testing.T) {
- templates := &Templates{}
-
- w := httptest.NewRecorder()
- if err := templates.Render(w, "dashboard", DashboardData{}); err != nil {
- t.Fatalf("Render failed: %v", err)
- }
-
- body := w.Body.String()
- if strings.Contains(body, `rel="canonical"`) {
- t.Error("canonical tag should be omitted when UIBaseURL is empty")
- }
- if strings.Contains(body, `property="og:url"`) {
- t.Error("og:url tag should be omitted when UIBaseURL is empty")
- }
-}
-
-func TestInstallPageBannerWhenUIDiffersFromBaseURL(t *testing.T) {
- templates := &Templates{}
-
- data := struct {
- Layout
- BaseURL string
- Registries []RegistryConfig
- }{
- Layout: Layout{
- UIBaseURL: "https://ui.example.com/ui",
- CanonicalPath: "/ui/install",
- },
- BaseURL: "http://pkg-proxy:8080",
- Registries: getRegistryConfigs("http://pkg-proxy:8080"),
- }
-
- w := httptest.NewRecorder()
- if err := templates.Render(w, "install", data); err != nil {
- t.Fatalf("Render failed: %v", err)
- }
-
- body := w.Body.String()
- if !strings.Contains(body, "https://ui.example.com/ui") || !strings.Contains(body, "http://pkg-proxy:8080") {
- t.Error("install banner should mention both UIBaseURL and BaseURL when they differ")
- }
- if !strings.Contains(body, "package managers should be configured") {
- t.Error("install banner copy missing")
- }
-}
-
-func TestInstallPageNoBannerWhenURLsMatch(t *testing.T) {
- templates := &Templates{}
-
- data := struct {
- Layout
- BaseURL string
- Registries []RegistryConfig
- }{
- Layout: Layout{
- UIBaseURL: "http://localhost:8080",
- CanonicalPath: "/ui/install",
- },
- BaseURL: "http://localhost:8080",
- Registries: getRegistryConfigs("http://localhost:8080"),
- }
-
- w := httptest.NewRecorder()
- if err := templates.Render(w, "install", data); err != nil {
- t.Fatalf("Render failed: %v", err)
- }
-
- body := w.Body.String()
- if strings.Contains(body, "package managers should be configured") {
- t.Error("install banner should be hidden when UIBaseURL == BaseURL")
- }
-}
-
-func TestTemplatesLazyLoading(t *testing.T) {
- templates := &Templates{}
-
- if templates.pages != nil {
- t.Fatal("expected pages to be nil before first Render call")
- }
-
- w := httptest.NewRecorder()
- _ = templates.Render(w, "dashboard", DashboardData{})
-
- if templates.pages == nil {
- t.Fatal("expected pages to be populated after first Render call")
- }
-}
-
-func TestTemplatesRenderUnknownPage(t *testing.T) {
- templates := &Templates{}
-
- w := httptest.NewRecorder()
- err := templates.Render(w, "nonexistent_page", nil)
- if err == nil {
- t.Error("expected error for unknown page")
- }
-}
-
-func TestInstallPage(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- req := httptest.NewRequest("GET", "/ui/install", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusOK {
- t.Fatalf("expected status 200, got %d", w.Code)
- }
-
- body := w.Body.String()
-
- // Should contain instructions for all registries
- registries := []string{"npm", "Cargo", "RubyGems", "Go Modules", "PyPI", "Maven", "Gradle Build Cache", "NuGet", "Composer", "Conan", "Conda", "CRAN"}
- for _, reg := range registries {
- if !strings.Contains(body, reg) {
- t.Errorf("install page should contain %s instructions", reg)
- }
- }
-}
-
-func TestPackageShowPage(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- // Create a package with license and version
- pkg := &database.Package{
- PURL: "pkg:npm/test-show",
- Ecosystem: "npm",
- Name: "test-show",
- LatestVersion: sql.NullString{String: "2.0.0", Valid: true},
- License: sql.NullString{String: "MIT", Valid: true},
- }
- if err := ts.db.UpsertPackage(pkg); err != nil {
- t.Fatalf("failed to upsert package: %v", err)
- }
- ver := &database.Version{PURL: "pkg:npm/test-show@2.0.0", PackagePURL: pkg.PURL}
- if err := ts.db.UpsertVersion(ver); err != nil {
- t.Fatalf("failed to upsert version: %v", err)
- }
-
- req := httptest.NewRequest("GET", "/ui/package/npm/test-show", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusOK {
- t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String())
- }
-
- body := w.Body.String()
- if !strings.Contains(body, "test-show") {
- t.Error("page should contain package name")
- }
- if !strings.Contains(body, "2.0.0") {
- t.Error("page should contain version")
- }
- if !strings.Contains(body, "MIT") {
- t.Error("page should contain license")
- }
-}
-
-func TestPackageShowPage_NotFound(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- req := httptest.NewRequest("GET", "/ui/package/npm/nonexistent", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusNotFound {
- t.Errorf("expected status 404, got %d", w.Code)
- }
-}
-
-func TestVersionShowPage_NotFound(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- req := httptest.NewRequest("GET", "/ui/package/npm/nonexistent/1.0.0", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusNotFound {
- t.Errorf("expected status 404, got %d", w.Code)
- }
-}
-
-func TestSearchPage_EmptyQuery(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- req := httptest.NewRequest("GET", "/ui/search", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- // Empty query should redirect to homepage
- if w.Code != http.StatusSeeOther {
- t.Errorf("expected redirect (303), got %d", w.Code)
- }
-}
-
-func TestSearchPage_WithQuery(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- req := httptest.NewRequest("GET", "/ui/search?q=test", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusOK {
- t.Fatalf("expected status 200, got %d", w.Code)
- }
-
- body := w.Body.String()
- if !strings.Contains(body, "test") {
- t.Error("search page should contain the query")
- }
-}
-
-func TestSearchPage_Pagination(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- // Page 0 or negative should default to page 1
- req := httptest.NewRequest("GET", "/ui/search?q=test&page=0", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusOK {
- t.Errorf("expected status 200, got %d", w.Code)
- }
-
- // Non-numeric page should default to page 1
- req = httptest.NewRequest("GET", "/ui/search?q=test&page=abc", nil)
- w = httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusOK {
- t.Errorf("expected status 200, got %d", w.Code)
- }
-}
-
-func TestSearchPage_EcosystemFilter(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- req := httptest.NewRequest("GET", "/ui/search?q=test&ecosystem=npm", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusOK {
- t.Errorf("expected status 200, got %d", w.Code)
- }
-}
-
-func TestEcosystemBadgeLabel(t *testing.T) {
- tests := []struct {
- ecosystem string
- want string
- }{
- {"oci", "container"},
- {"deb", "debian"},
- {"npm", "npm"},
- {"cargo", "cargo"},
- }
-
- for _, tt := range tests {
- got := ecosystemBadgeLabel(tt.ecosystem)
- if got != tt.want {
- t.Errorf("ecosystemBadgeLabel(%q) = %q, want %q", tt.ecosystem, got, tt.want)
- }
- }
-}
-
-func TestOCIRegistryInstructionsDockerPull(t *testing.T) {
- registries := getRegistryConfigs("http://package-proxy:8080")
- for _, registry := range registries {
- if registry.ID != "oci" {
- continue
- }
- want := "docker pull package-proxy:8080/library/nginx:latest"
- if !strings.Contains(string(registry.Instructions), want) {
- t.Errorf("OCI instructions = %q, want substring %q", registry.Instructions, want)
- }
- return
- }
- t.Fatal("OCI registry instructions not found")
-}
-
-func TestSwiftRegistryInstructionsAllowLocalHTTP(t *testing.T) {
- registries := getRegistryConfigs("http://localhost:8080")
- for _, registry := range registries {
- if registry.ID != "swift" {
- continue
- }
- if !strings.Contains(string(registry.Instructions), "--allow-insecure-http") {
- t.Error("Swift HTTP instructions do not allow the insecure local registry")
- }
- return
- }
- t.Fatal("Swift registry instructions not found")
-}
-
-func TestEcosystemBadgeClasses(t *testing.T) {
- // Every supported ecosystem should return a non-empty class string
- ecosystems := supportedEcosystems()
- for _, eco := range ecosystems {
- classes := ecosystemBadgeClasses(eco)
- if classes == "" {
- t.Errorf("ecosystemBadgeClasses(%q) returned empty string", eco)
- }
- if !strings.Contains(classes, "inline-flex") {
- t.Errorf("ecosystemBadgeClasses(%q) missing base classes", eco)
- }
- }
-
- // Unknown ecosystem should still return classes
- classes := ecosystemBadgeClasses("unknown")
- if !strings.Contains(classes, "gray") {
- t.Error("unknown ecosystem should use gray classes")
- }
-}
-
-func TestBuildEcosystemFilters(t *testing.T) {
- filters := buildEcosystemFilters(map[string]int64{
- "npm": 3,
- "cargo": 2,
- "empty": 0,
- })
-
- if len(filters) != 2 {
- t.Fatalf("expected 2 filters, got %d", len(filters))
- }
- if filters[0].Ecosystem != "cargo" || filters[0].Count != 2 {
- t.Errorf("first filter = %#v, want cargo with count 2", filters[0])
- }
- if filters[1].Ecosystem != "npm" || filters[1].Count != 3 {
- t.Errorf("second filter = %#v, want npm with count 3", filters[1])
- }
-
- extra := buildEcosystemFilters(map[string]int64{"custom": 1})
- if len(extra) != 1 || extra[0].Ecosystem != "custom" {
- t.Errorf("unexpected extra filters: %#v", extra)
- }
-}
-
-func TestEcosystemPillClasses(t *testing.T) {
- classes := ecosystemPillClasses("npm")
- if !strings.Contains(classes, "rounded-full") {
- t.Error("pill classes should use rounded-full")
- }
- if !strings.Contains(classes, "bg-red-100") {
- t.Error("npm pill should use npm colors")
- }
-}
-
-func TestFormatSize(t *testing.T) {
- tests := []struct {
- bytes int64
- want string
- }{
- {0, "0 B"},
- {100, "100 B"},
- {1024, "1.0 KB"},
- {1536, "1.5 KB"},
- {1048576, "1.0 MB"},
- {1073741824, "1.0 GB"},
- }
-
- for _, tt := range tests {
- got := formatSize(tt.bytes)
- if got != tt.want {
- t.Errorf("formatSize(%d) = %q, want %q", tt.bytes, got, tt.want)
- }
- }
-}
-
-func TestCategorizeLicense(t *testing.T) {
- tests := []struct {
- license sql.NullString
- want string
- }{
- {sql.NullString{Valid: false}, "unknown"},
- {sql.NullString{String: "", Valid: true}, "unknown"},
- {sql.NullString{String: "MIT", Valid: true}, "permissive"},
- {sql.NullString{String: "GPL-3.0", Valid: true}, "copyleft"},
- }
-
- for _, tt := range tests {
- got := categorizeLicense(tt.license)
- if got != tt.want {
- t.Errorf("categorizeLicense(%v) = %q, want %q", tt.license, got, tt.want)
- }
- }
-}
-
-func BenchmarkTemplatesParse(b *testing.B) {
- for b.Loop() {
- t := &Templates{}
- if err := t.load(); err != nil {
- b.Fatal(err)
- }
- }
-}
-
-func BenchmarkServerCreate(b *testing.B) {
- for b.Loop() {
- _ = &Server{
- templates: &Templates{},
- }
- }
-}
-
-func BenchmarkFirstRender(b *testing.B) {
- for b.Loop() {
- t := &Templates{}
- w := httptest.NewRecorder()
- if err := t.Render(w, "dashboard", DashboardData{}); err != nil {
- b.Fatal(err)
- }
- }
-}
-
-// thirdPartyAsset matches a script or stylesheet loaded from another origin.
-var thirdPartyAsset = regexp.MustCompile(`<(?:script[^>]*\ssrc|link[^>]*\shref)="(?:https?:)?//[^"]*"`)
-
-// Pages run on the proxy origin, so a script or stylesheet from elsewhere
-// would run with the proxy's privileges and break offline deployments.
-// Vendor it under static/vendor instead.
-func TestTemplatesLoadNoThirdPartyAssets(t *testing.T) {
- err := fs.WalkDir(templatesFS, "templates", func(path string, d fs.DirEntry, err error) error {
- if err != nil || d.IsDir() {
- return err
- }
- content, err := templatesFS.ReadFile(path)
- if err != nil {
- return err
- }
- for _, match := range thirdPartyAsset.FindAllString(string(content), -1) {
- t.Errorf("%s loads a third-party asset: %s", path, match)
- }
- return nil
- })
- if err != nil {
- t.Fatal(err)
- }
-}
diff --git a/internal/server/transport_test.go b/internal/server/transport_test.go
deleted file mode 100644
index 7464282..0000000
--- a/internal/server/transport_test.go
+++ /dev/null
@@ -1,190 +0,0 @@
-package server
-
-import (
- "crypto/tls"
- "io"
- "net"
- "net/http"
- "net/http/httptest"
- "strings"
- "sync"
- "testing"
- "time"
-
- "github.com/git-pkgs/proxy/internal/config"
- "github.com/git-pkgs/registries/safehttp"
-)
-
-// tlsUpstream starts a TLS test server that counts accepted connections.
-func tlsUpstream(t *testing.T, handler http.HandlerFunc) (*httptest.Server, func() int) {
- t.Helper()
- var mu sync.Mutex
- accepted := 0
- srv := httptest.NewUnstartedServer(handler)
- srv.Config.ConnState = func(_ net.Conn, state http.ConnState) {
- if state == http.StateNew {
- mu.Lock()
- accepted++
- mu.Unlock()
- }
- }
- srv.StartTLS()
- t.Cleanup(srv.Close)
- return srv, func() int {
- mu.Lock()
- defer mu.Unlock()
- return accepted
- }
-}
-
-// trustUpstream makes transport trust srv's certificate and pins HTTP/1.1 so
-// every in-flight request needs its own connection.
-func trustUpstream(t *testing.T, transport *http.Transport, srv *httptest.Server) {
- t.Helper()
- transport.TLSClientConfig = &tls.Config{
- RootCAs: srv.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs,
- NextProtos: []string{"http/1.1"},
- MinVersion: tls.VersionTLS12,
- }
- transport.ForceAttemptHTTP2 = false
- t.Cleanup(transport.CloseIdleConnections)
-}
-
-// burst issues n concurrent GETs and drains every body. The transport hands a
-// connection back to the idle pool before the body's final Read returns, so
-// the pool is settled when burst returns.
-func burst(t *testing.T, client *http.Client, url string, n int) {
- t.Helper()
- var wg sync.WaitGroup
- errs := make(chan error, n)
- for range n {
- wg.Add(1)
- go func() {
- defer wg.Done()
- resp, err := client.Get(url)
- if err != nil {
- errs <- err
- return
- }
- _, _ = io.Copy(io.Discard, resp.Body)
- _ = resp.Body.Close()
- }()
- }
- wg.Wait()
- close(errs)
- for err := range errs {
- t.Errorf("burst request: %v", err)
- }
-}
-
-// TestUpstreamClientReusesConnectionsAcrossBursts measures how many
-// connections a second burst of concurrent requests reuses. With Go's default
-// of two idle connections per host most of them are re-dialled; with the
-// tuned transport the second burst reuses all of them.
-func TestUpstreamClientReusesConnectionsAcrossBursts(t *testing.T) {
- const burstSize = 8
-
- // holdBurst returns a handler that answers a request only once burstSize
- // of them are waiting at the same time. With HTTP/1.1 pinned that puts
- // every burst on burstSize distinct connections, whatever the scheduling.
- holdBurst := func() http.HandlerFunc {
- var mu sync.Mutex
- waiting := 0
- release := make(chan struct{})
- return func(w http.ResponseWriter, r *http.Request) {
- mu.Lock()
- gate := release
- waiting++
- if waiting == burstSize {
- close(gate)
- waiting = 0
- release = make(chan struct{})
- }
- mu.Unlock()
- select {
- case <-gate:
- case <-r.Context().Done():
- }
- _, _ = w.Write([]byte("ok"))
- }
- }
-
- tests := []struct {
- name string
- client *http.Client
- // Bounds on how many connections the second burst has to dial.
- minNew, maxNew int
- }{
- {
- name: "go default keeps two idle connections",
- client: safehttp.New(nil, safehttp.Options{AllowLoopback: true}),
- minNew: burstSize - 2,
- maxNew: burstSize,
- },
- {
- name: "tuned transport reuses the whole burst",
- client: newUpstreamClient(config.UpstreamConfig{AllowLoopback: true}),
- minNew: 0,
- maxNew: 0,
- },
- }
- for _, tc := range tests {
- t.Run(tc.name, func(t *testing.T) {
- srv, accepted := tlsUpstream(t, holdBurst())
- transport := tc.client.Transport.(*http.Transport)
- trustUpstream(t, transport, srv)
-
- burst(t, tc.client, srv.URL, burstSize)
- afterFirst := accepted()
- if afterFirst < burstSize {
- t.Fatalf("first burst opened %d connections, want at least %d", afterFirst, burstSize)
- }
-
- burst(t, tc.client, srv.URL, burstSize)
- newInSecond := accepted() - afterFirst
- t.Logf("second burst: %d new connections, %d reused", newInSecond, burstSize-newInSecond)
-
- if newInSecond < tc.minNew || newInSecond > tc.maxNew {
- t.Errorf("second burst opened %d new connections, want between %d and %d", newInSecond, tc.minNew, tc.maxNew)
- }
- })
- }
-}
-
-// TestUpstreamClientBoundsStallBeforeHeaders pins the production transport
-// values, then lowers the header timeout so it can show within milliseconds
-// that this is what cuts off an upstream which accepts a request but never
-// sends headers.
-func TestUpstreamClientBoundsStallBeforeHeaders(t *testing.T) {
- client := newUpstreamClient(config.UpstreamConfig{AllowLoopback: true})
- transport := client.Transport.(*http.Transport)
- if transport.MaxIdleConnsPerHost != upstreamMaxIdleConnsPerHost {
- t.Fatalf("MaxIdleConnsPerHost = %d, want %d", transport.MaxIdleConnsPerHost, upstreamMaxIdleConnsPerHost)
- }
- if transport.ResponseHeaderTimeout != upstreamResponseHeaderTimeout {
- t.Fatalf("ResponseHeaderTimeout = %v, want %v", transport.ResponseHeaderTimeout, upstreamResponseHeaderTimeout)
- }
-
- stall := make(chan struct{})
- srv, _ := tlsUpstream(t, func(_ http.ResponseWriter, r *http.Request) {
- select {
- case <-stall:
- case <-r.Context().Done():
- }
- })
- t.Cleanup(func() { close(stall) })
- trustUpstream(t, transport, srv)
-
- // Far below the client's overall timeout, so the header timeout ends the
- // request; the error text tells the two timeouts apart.
- transport.ResponseHeaderTimeout = 200 * time.Millisecond
-
- resp, err := client.Get(srv.URL)
- if err == nil {
- _ = resp.Body.Close()
- t.Fatal("request to a stalled upstream succeeded, want a timeout")
- }
- if !strings.Contains(err.Error(), "timeout awaiting response headers") {
- t.Fatalf("error = %v, want a response-header timeout", err)
- }
-}
diff --git a/internal/server/untrusted_content_test.go b/internal/server/untrusted_content_test.go
deleted file mode 100644
index 5982402..0000000
--- a/internal/server/untrusted_content_test.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package server
-
-import (
- "context"
- "io"
- "log/slog"
- "net/http"
- "net/http/httptest"
- "testing"
-
- "github.com/git-pkgs/proxy/internal/config"
- "github.com/git-pkgs/proxy/internal/database"
- "github.com/git-pkgs/proxy/internal/handler"
- "github.com/git-pkgs/proxy/internal/storage"
- "github.com/git-pkgs/registries/fetch"
- "github.com/go-chi/chi/v5"
-)
-
-// TestPackageRoutesSandboxUpstreamContent checks that responses on package
-// routes cannot run as pages on the proxy's origin, which also serves /ui.
-// Their bodies and Content-Type come from upstreams, or from whoever pushed a
-// hosted package, so an archive served as text/html would otherwise run its
-// scripts with access to the UI.
-func TestPackageRoutesSandboxUpstreamContent(t *testing.T) {
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.Header().Set("Content-Type", "text/html")
- _, _ = io.WriteString(w, "")
- }))
- t.Cleanup(upstream.Close)
-
- db, err := database.Create(t.TempDir() + "/test.db")
- if err != nil {
- t.Fatalf("creating database: %v", err)
- }
- t.Cleanup(func() { _ = db.Close() })
- store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir())
- if err != nil {
- t.Fatalf("opening storage: %v", err)
- }
- fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
- t.Cleanup(func() { _ = fetcher.Close() })
- proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil)))
- proxy.HTTPClient = upstream.Client()
-
- cfg := config.Default()
- cfg.BaseURL = "http://proxy.example"
- cfg.Upstream.Generic = map[string]string{"site": upstream.URL}
- cfg.Upstream.APK = map[string]string{"alpine": upstream.URL}
- s := &Server{cfg: cfg}
- r := chi.NewRouter()
- s.mountProtocolHandlers(r, proxy)
-
- for _, target := range []string{
- "/generic/site/files/page-1.0.html", // cached artifact
- "/generic/site/files/page-1.0.html", // the same, from the cache
- "/apk/alpine/v3.22/main/x86_64/README.html", // relayed as is
- "/npm/does-not-exist%2F..", // an error response
- } {
- w := httptest.NewRecorder()
- r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil))
- if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" {
- t.Errorf("%s (status %d): X-Content-Type-Options = %q, want nosniff", target, w.Code, got)
- }
- if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" {
- t.Errorf("%s (status %d): Content-Security-Policy = %q, want sandbox", target, w.Code, got)
- }
- }
-}
diff --git a/internal/server/upstream_auth_redirect_test.go b/internal/server/upstream_auth_redirect_test.go
deleted file mode 100644
index 7c2f0c4..0000000
--- a/internal/server/upstream_auth_redirect_test.go
+++ /dev/null
@@ -1,125 +0,0 @@
-package server
-
-import (
- "context"
- "errors"
- "io"
- "log/slog"
- "net"
- "net/http"
- "net/http/httptest"
- "path/filepath"
- "sync"
- "testing"
- "time"
-
- "github.com/git-pkgs/proxy/internal/config"
-)
-
-const redirectAuthHeader = "X-Api-Key"
-
-// headerRecorder remembers the auth header value each request carried.
-type headerRecorder struct {
- mu sync.Mutex
- values []string
-}
-
-func (h *headerRecorder) record(r *http.Request) {
- h.mu.Lock()
- defer h.mu.Unlock()
- h.values = append(h.values, r.Header.Get(redirectAuthHeader))
-}
-
-func (h *headerRecorder) seen() []string {
- h.mu.Lock()
- defer h.mu.Unlock()
- return append([]string(nil), h.values...)
-}
-
-func TestHeaderAuthIsNotForwardedAcrossRedirects(t *testing.T) {
- var atOther headerRecorder
- other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- atOther.record(r)
- w.Header().Set("Content-Type", "application/json")
- _, _ = io.WriteString(w, `{"name":"leftpad","versions":{}}`)
- }))
- defer other.Close()
-
- var atRegistry headerRecorder
- registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- atRegistry.record(r)
- http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)
- }))
- defer registry.Close()
-
- listener, err := net.Listen("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatalf("reserving proxy address: %v", err)
- }
- t.Cleanup(func() { _ = listener.Close() })
-
- tempDir := t.TempDir()
- cfg := config.Default()
- cfg.Listen = listener.Addr().String()
- cfg.BaseURL = "http://" + cfg.Listen
- cfg.Database.Path = filepath.Join(tempDir, "proxy.db")
- cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts")
- cfg.Upstream.AllowLoopback = true
- cfg.Upstream.NPM = registry.URL
- cfg.Upstream.Auth = map[string]config.AuthConfig{
- registry.URL: {Type: "header", HeaderName: redirectAuthHeader, HeaderValue: "registry-secret"},
- }
- if err := cfg.Validate(); err != nil {
- t.Fatalf("validating config: %v", err)
- }
-
- proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"})
- if err != nil {
- t.Fatalf("creating server: %v", err)
- }
- startErr := make(chan error, 1)
- go func() { startErr <- proxyServer.Start(listener) }()
- defer func() {
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- if err := proxyServer.Shutdown(ctx); err != nil {
- t.Errorf("shutting down server: %v", err)
- }
- if err := <-startErr; !errors.Is(err, http.ErrServerClosed) {
- t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed)
- }
- }()
-
- resp := getWhenListening(t, cfg.BaseURL+"/npm/leftpad")
- _ = resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
- }
-
- registrySeen := atRegistry.seen()
- if len(registrySeen) == 0 || registrySeen[0] != "registry-secret" {
- t.Errorf("registry received %s = %q, want the configured value", redirectAuthHeader, registrySeen)
- }
- for _, value := range atOther.seen() {
- if value != "" {
- t.Errorf("redirect target received %s = %q, want none", redirectAuthHeader, value)
- }
- }
-}
-
-// getWhenListening retries a GET until the proxy accepts connections.
-func getWhenListening(t *testing.T, url string) *http.Response {
- t.Helper()
- client := &http.Client{Timeout: 5 * time.Second}
- deadline := time.Now().Add(5 * time.Second)
- for {
- resp, err := client.Get(url)
- if err == nil {
- return resp
- }
- if time.Now().After(deadline) {
- t.Fatalf("proxy did not start: %v", err)
- }
- time.Sleep(10 * time.Millisecond)
- }
-}
diff --git a/internal/storage/blob.go b/internal/storage/blob.go
index 81af78f..22643fa 100644
--- a/internal/storage/blob.go
+++ b/internal/storage/blob.go
@@ -6,45 +6,22 @@ import (
"encoding/hex"
"fmt"
"io"
- "net/http"
"os"
"path/filepath"
"runtime"
"strings"
- "time"
"gocloud.dev/blob"
- _ "gocloud.dev/blob/azureblob"
_ "gocloud.dev/blob/fileblob"
_ "gocloud.dev/blob/s3blob"
"gocloud.dev/gcerrors"
)
-const osWindows = "windows"
-
-// Upload buffering for cloud backends. The S3 driver otherwise allocates
-// (5+1) x 8 MiB part buffers up front for every object over 16 MiB, and the
-// Azure driver uses 5 x 8 MiB blocks; with several large artifacts being
-// cached at once that adds up to hundreds of MiB. 5 MiB is the smallest part
-// S3 accepts. The file backend ignores both.
-const (
- uploadBufferSize = 5 << 20
- uploadConcurrency = 2
-)
-
-// attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an
-// earlier version wrote.
-const attrsExt = ".attrs"
-
// Blob implements Storage using gocloud.dev/blob.
// Supports local filesystem (file://) and S3 (s3://) URLs.
type Blob struct {
bucket *blob.Bucket
url string
-
- // fileRoot is the directory backing a file:// bucket, empty for cloud
- // backends. Used only to clear sidecars an earlier version wrote.
- fileRoot string
}
// OpenBucket opens a blob bucket from a URL.
@@ -53,29 +30,18 @@ type Blob struct {
// - file:///path/to/dir - Local filesystem storage
// - s3://bucket-name - Amazon S3 (uses AWS_* environment variables)
// - s3://bucket-name?region=us-east-1&endpoint=http://localhost:9000 - S3-compatible (MinIO, etc.)
-// - gs://bucket-name - Google Cloud Storage (uses Application Default Credentials;
-// supports Workload Identity on GKE/GCE without any extra configuration)
-// - azblob://container-name - Azure Blob Storage
//
// For local filesystem, the directory is created if it doesn't exist.
-//
-//nolint:ireturn // The URL scheme selects the storage implementation.
-func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
- if strings.HasPrefix(urlStr, "gs://") {
- return OpenGCS(ctx, urlStr)
- }
-
- var fileRoot string
-
+func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) {
// Handle file:// URLs specially to create the directory
if strings.HasPrefix(urlStr, "file://") {
path := strings.TrimPrefix(urlStr, "file://")
// Handle file:/// (three slashes) for absolute paths
- if strings.HasPrefix(path, "/") && runtime.GOOS != osWindows {
+ if strings.HasPrefix(path, "/") && runtime.GOOS != "windows" {
// Unix: file:///path -> /path
// path is already correct
- } else if strings.HasPrefix(path, "/") && runtime.GOOS == osWindows {
+ } else if strings.HasPrefix(path, "/") && runtime.GOOS == "windows" {
// Windows: file:///C:/path -> C:/path
path = strings.TrimPrefix(path, "/")
}
@@ -84,7 +50,7 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
nativePath := filepath.FromSlash(path)
// Ensure directory exists
- if err := os.MkdirAll(nativePath, dirPermissions); err != nil {
+ if err := os.MkdirAll(nativePath, 0755); err != nil {
return nil, fmt.Errorf("creating directory: %w", err)
}
@@ -94,29 +60,14 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
return nil, fmt.Errorf("resolving path: %w", err)
}
- fileRoot = absPath
-
// Convert back to URL format with forward slashes
urlPath := filepath.ToSlash(absPath)
- if runtime.GOOS == osWindows {
+ if runtime.GOOS == "windows" {
// Windows needs file:///C:/path format
urlStr = "file:///" + urlPath
} else {
urlStr = "file://" + urlPath
}
-
- // Create temp files next to the final path instead of in os.TempDir.
- // This avoids "invalid cross-device link" errors from os.Rename when
- // the bucket directory and os.TempDir are on different filesystems
- // (e.g. Docker volume mounts).
- //
- // Do not write fileblob's ".attrs" sidecar. It is rewritten with
- // os.Create, truncating in place outside the atomic rename that
- // protects the blob, so a read overlapping a write can decode a
- // partial file; a missing one defaults cleanly, a truncated one does
- // not. Nothing in the proxy needs it: Store sets no ContentType, and
- // Size reads os.Stat via Attributes.
- urlStr += "?no_tmp_dir=true&metadata=skip"
}
bucket, err := blob.OpenBucket(ctx, urlStr)
@@ -124,116 +75,22 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
return nil, fmt.Errorf("opening bucket: %w", err)
}
- return &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}, nil
-}
-
-// legacySidecarPath gives the ".attrs" path an earlier version wrote for key,
-// or "" when that path would not be a file inside fileRoot.
-func (b *Blob) legacySidecarPath(key string) string {
- if p := b.localPath(key); p != "" {
- return p + attrsExt
- }
- return ""
-}
-
-// localPath gives the file fileblob keeps key in, or "" when that would not
-// be a file inside fileRoot.
-//
-// The key is escaped the way fileblob escapes it on the way to disk.
-// filepath.Localize then validates the escaped form: it rejects an empty,
-// absolute or ".." path, and "." would name fileRoot itself. What it declines
-// are keys the proxy never produces.
-func (b *Blob) localPath(key string) string {
- if b.fileRoot == "" {
- return ""
- }
- rel, err := filepath.Localize(escapeKey(key))
- if err != nil || rel == "." {
- return ""
- }
- return filepath.Join(b.fileRoot, rel)
-}
-
-// escapeKey mirrors fileblob's unexported escapeKey, which hex-escapes a rune
-// as "__0x__". Slashes stay as "/" for filepath.Localize to convert.
-func escapeKey(key string) string {
- runes := []rune(key)
- var out strings.Builder
- for i, r := range runes {
- if escapeRune(runes, i) {
- fmt.Fprintf(&out, "__%#x__", r)
- } else {
- out.WriteRune(r)
- }
- }
- return out.String()
-}
-
-// escapeRune is fileblob's rule for which runes of a key to escape: control
-// characters, a raw path separator, a slash that would form "../", "//" or
-// end the key, and on Windows the characters its filesystem reserves.
-func escapeRune(r []rune, i int) bool {
- c := r[i]
- switch {
- case c < ' ':
- return true
- case os.PathSeparator != '/' && c == os.PathSeparator:
- return true
- case i > 1 && c == '/' && r[i-1] == '.' && r[i-2] == '.':
- return true
- case i > 0 && c == '/' && r[i-1] == '/':
- return true
- case c == '/' && i == len(r)-1:
- return true
- case os.PathSeparator == '\\' && strings.ContainsRune(`<>:"|?*`, c):
- return true
- }
- return false
-}
-
-// clearLegacySidecar removes the ".attrs" file an earlier version wrote for
-// key. Nothing rewrites one now, so a sidecar left partial by an interrupted
-// write would fail every read of that key for good. Removing is atomic where
-// the rewrite was not, so a concurrent reader gets the whole old file or
-// nothing.
-//
-// Failure is deliberately not fatal. Usually the key never had a sidecar and
-// os.Remove reports not-exist. A real failure leaves exactly the state this
-// change inherited, while failing the write would turn a cleanup miss into a
-// failed request. Windows makes that concrete: Go opens files without
-// FILE_SHARE_DELETE, so a reader holding the sidecar open blocks deletion, and
-// that reader is the very workload this change protects. The next store of the
-// key retries.
-func (b *Blob) clearLegacySidecar(key string) {
- if sidecar := b.legacySidecarPath(key); sidecar != "" {
- _ = os.Remove(sidecar)
- }
+ return &Blob{bucket: bucket, url: urlStr}, nil
}
func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
- b.clearLegacySidecar(path)
-
// Compute hash while writing
h := sha256.New()
tee := io.TeeReader(r, h)
- // gocloud commits whatever was written when the writer closes, unless
- // its context is cancelled first, so a failed copy cancels it to abort.
- writeCtx, abort := context.WithCancel(ctx)
- defer abort()
-
- opts := &blob.WriterOptions{
- BufferSize: uploadBufferSize,
- MaxConcurrency: uploadConcurrency,
- }
- w, err := b.bucket.NewWriter(writeCtx, path, opts)
+ opts := &blob.WriterOptions{}
+ w, err := b.bucket.NewWriter(ctx, path, opts)
if err != nil {
return 0, "", fmt.Errorf("creating writer: %w", err)
}
size, err := io.Copy(w, tee)
if err != nil {
- abort()
_ = w.Close()
return 0, "", fmt.Errorf("writing content: %w", err)
}
@@ -247,17 +104,6 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri
}
func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) {
- if localPath := b.localPath(path); localPath != "" {
- r, err := os.Open(localPath)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, ErrNotFound
- }
- return nil, fmt.Errorf("opening local reader: %w", err)
- }
- return r, nil
- }
-
r, err := b.bucket.NewReader(ctx, path, nil)
if err != nil {
if isNotExist(err) {
@@ -268,11 +114,6 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) {
return r, nil
}
-// ReadsSeekable reports that Open returns seekable readers: an *os.File for
-// file:// buckets and a *blob.Reader, which seeks with a ranged read,
-// otherwise.
-func (b *Blob) ReadsSeekable() bool { return true }
-
func (b *Blob) Exists(ctx context.Context, path string) (bool, error) {
exists, err := b.bucket.Exists(ctx, path)
if err != nil {
@@ -281,37 +122,14 @@ func (b *Blob) Exists(ctx context.Context, path string) (bool, error) {
return exists, nil
}
-// Delete removes the object at path. On a file:// bucket it also removes the
-// object's fetch directory once empty, since fileblob leaves directories
-// behind. Any other directory, such as a version directory another fetch may
-// be creating its own directory in, is left alone.
func (b *Blob) Delete(ctx context.Context, path string) error {
err := b.bucket.Delete(ctx, path)
if err != nil && !isNotExist(err) {
return fmt.Errorf("deleting object: %w", err)
}
- if p := b.localPath(path); p != "" {
- if dir := filepath.Dir(p); dir != b.fileRoot && isFetchDir(filepath.Base(dir)) {
- _ = os.Remove(dir) // fails, harmlessly, while the directory holds anything
- }
- }
return nil
}
-func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
- url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
- Method: http.MethodGet,
- Expiry: expiry,
- })
- if err != nil {
- if gcerrors.Code(err) == gcerrors.Unimplemented {
- return "", ErrSignedURLUnsupported
- }
- return "", fmt.Errorf("signing URL: %w", err)
- }
- return url, nil
-}
-
func (b *Blob) Size(ctx context.Context, path string) (int64, error) {
attrs, err := b.bucket.Attributes(ctx, path)
if err != nil {
@@ -341,35 +159,6 @@ func (b *Blob) UsedSpace(ctx context.Context) (int64, error) {
return total, nil
}
-// ListPrefix returns object metadata for keys under a prefix.
-func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
- iter := b.bucket.List(&blob.ListOptions{Prefix: prefix})
- objects := make([]ObjectInfo, 0)
-
- for {
- obj, err := iter.Next(ctx)
- if err == io.EOF {
- break
- }
- if err != nil {
- return nil, fmt.Errorf("listing objects: %w", err)
- }
- if obj.IsDir {
- continue
- }
-
- info := ObjectInfo{
- Path: obj.Key,
- Size: obj.Size,
- ModTime: obj.ModTime,
- }
-
- objects = append(objects, info)
- }
-
- return objects, nil
-}
-
func (b *Blob) Close() error {
return b.bucket.Close()
}
diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go
index 1a764de..3add0a7 100644
--- a/internal/storage/blob_test.go
+++ b/internal/storage/blob_test.go
@@ -1,22 +1,16 @@
package storage
import (
+ "bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
- "io/fs"
- "os"
"path/filepath"
"runtime"
"strings"
- "sync"
- "sync/atomic"
"testing"
- "time"
-
- "gocloud.dev/blob"
)
func TestOpenBucket(t *testing.T) {
@@ -67,31 +61,16 @@ func TestBlobOpen(t *testing.T) {
t.Fatalf("Open failed: %v", err)
}
defer func() { _ = r.Close() }()
- seeker, ok := r.(io.Seeker)
- if !ok {
- t.Fatal("local file reader does not implement io.Seeker")
- }
- if _, err := seeker.Seek(9, io.SeekStart); err != nil {
- t.Fatalf("Seek failed: %v", err)
- }
data, err := io.ReadAll(r)
if err != nil {
t.Fatalf("ReadAll failed: %v", err)
}
- if string(data) != "content" {
- t.Errorf("content after seek = %q, want %q", string(data), "content")
+ if string(data) != content {
+ t.Errorf("content = %q, want %q", string(data), content)
}
}
-// Blob.ReadsSeekable promises seekable readers for every bucket: file://
-// objects open as *os.File (checked in TestBlobOpen) and remote ones as
-// *blob.Reader.
-var (
- _ io.Seeker = (*blob.Reader)(nil)
- _ SeekableReads = (*Blob)(nil)
-)
-
func TestBlobOpenNotFound(t *testing.T) {
b := createTestBlob(t)
ctx := context.Background()
@@ -142,44 +121,6 @@ func TestBlobDelete(t *testing.T) {
}
}
-// A fetch stores its object in a directory of its own, so Delete removes that
-// directory once it is empty rather than leave one behind per deleted object.
-// A version directory from the layout before fetch directories is left alone,
-// since a new fetch may be creating its directory inside it.
-func TestBlobDeleteRemovesEmptyFetchDirectory(t *testing.T) {
- dir := t.TempDir()
- b := openFileBlob(t, dir)
- ctx := context.Background()
- const (
- emptied = "npm/pkg/1.0.0/0123456789abcdef/pkg.tgz"
- shared = "npm/pkg/1.0.0/fedcba9876543210/pkg.tgz"
- kept = "npm/pkg/1.0.0/fedcba9876543210/other.tgz"
- legacy = "npm/pkg/2.0.0/pkg.tgz"
- )
- for _, key := range []string{emptied, shared, kept, legacy} {
- if _, _, err := b.Store(ctx, key, strings.NewReader("content")); err != nil {
- t.Fatalf("Store(%q): %v", key, err)
- }
- }
-
- for _, key := range []string{emptied, shared, legacy} {
- if err := b.Delete(ctx, key); err != nil {
- t.Fatalf("Delete(%q): %v", key, err)
- }
- }
-
- pkg := filepath.Join(dir, "npm", "pkg")
- if _, err := os.Stat(filepath.Join(pkg, "1.0.0", "0123456789abcdef")); !os.IsNotExist(err) {
- t.Errorf("emptied fetch directory left behind (stat err %v)", err)
- }
- for _, d := range []string{filepath.Join("1.0.0", "fedcba9876543210"), "1.0.0", "2.0.0"} {
- if _, err := os.Stat(filepath.Join(pkg, d)); err != nil {
- t.Errorf("directory %s removed: %v", d, err)
- }
- }
- assertReadsBack(t, b, kept, "content")
-}
-
func TestBlobDeleteNotFound(t *testing.T) {
b := createTestBlob(t)
ctx := context.Background()
@@ -245,18 +186,32 @@ func TestBlobUsedSpace(t *testing.T) {
}
func TestBlobLargeFile(t *testing.T) {
- assertLargeFileRoundTrip(t, createTestBlob(t))
-}
-
-func TestBlobSignedURLUnsupported(t *testing.T) {
b := createTestBlob(t)
ctx := context.Background()
- // fileblob has no URL signer configured, so this must surface as
- // ErrSignedURLUnsupported rather than a generic error.
- _, err := b.SignedURL(ctx, "test/file.txt", time.Minute)
- if !errors.Is(err, ErrSignedURLUnsupported) {
- t.Errorf("SignedURL on fileblob = %v, want ErrSignedURLUnsupported", err)
+ // 1MB of data
+ data := bytes.Repeat([]byte("x"), 1024*1024)
+
+ size, hash, err := b.Store(ctx, "large/file.bin", bytes.NewReader(data))
+ if err != nil {
+ t.Fatalf("Store large file failed: %v", err)
+ }
+ if size != int64(len(data)) {
+ t.Errorf("size = %d, want %d", size, len(data))
+ }
+
+ h := sha256.Sum256(data)
+ wantHash := hex.EncodeToString(h[:])
+ if hash != wantHash {
+ t.Errorf("hash mismatch for large file")
+ }
+
+ // Read it back
+ r, _ := b.Open(ctx, "large/file.bin")
+ defer func() { _ = r.Close() }()
+ readBack, _ := io.ReadAll(r)
+ if !bytes.Equal(readBack, data) {
+ t.Error("large file content mismatch")
}
}
@@ -289,44 +244,6 @@ func TestBlobOverwrite(t *testing.T) {
}
}
-func TestOpenBucketSetsNoTmpDir(t *testing.T) {
- dir := t.TempDir()
- ctx := context.Background()
-
- b, err := OpenBucket(ctx, fileURLFromPath(dir))
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
- defer func() { _ = b.Close() }()
-
- // fileblob uses os.TempDir() by default for temp files, then os.Rename to
- // the final path. This fails with "invalid cross-device link" when the bucket
- // dir and os.TempDir() are on different filesystems (e.g. Docker volumes).
- // OpenBucket must set no_tmp_dir=true so temp files are created next to the
- // final path instead.
- if !strings.Contains(b.URL(), "no_tmp_dir=true") {
- t.Errorf("URL should contain no_tmp_dir=true to avoid cross-device rename errors, got %q", b.URL())
- }
-
- // Verify Store still works with the parameter set
- content := "cross-device test"
- _, _, err = b.Store(ctx, "test/cross-device.txt", strings.NewReader(content))
- if err != nil {
- t.Fatalf("Store failed with no_tmp_dir=true: %v", err)
- }
-
- r, err := b.Open(ctx, "test/cross-device.txt")
- if err != nil {
- t.Fatalf("Open failed: %v", err)
- }
- defer func() { _ = r.Close() }()
-
- data, _ := io.ReadAll(r)
- if string(data) != content {
- t.Errorf("content = %q, want %q", string(data), content)
- }
-}
-
func createTestBlob(t *testing.T) *Blob {
t.Helper()
dir := t.TempDir()
@@ -337,372 +254,14 @@ func createTestBlob(t *testing.T) *Blob {
t.Fatalf("OpenBucket failed: %v", err)
}
t.Cleanup(func() { _ = b.Close() })
- blob, ok := b.(*Blob)
- if !ok {
- t.Fatalf("OpenBucket returned %T, want *Blob", b)
- }
- return blob
+ return b
}
func fileURLFromPath(path string) string {
- if runtime.GOOS == osWindows {
+ if runtime.GOOS == "windows" {
// Windows paths need file:///C:/path format
path = filepath.ToSlash(path)
return "file:///" + path
}
return "file://" + path
}
-
-func TestOpenBucketWritesNoAttrsSidecar(t *testing.T) {
- dir := t.TempDir()
- ctx := context.Background()
-
- b, err := OpenBucket(ctx, fileURLFromPath(dir))
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
- defer func() { _ = b.Close() }()
-
- if _, _, err := b.Store(ctx, "pkg/thing-1.0.0.tgz", strings.NewReader("content")); err != nil {
- t.Fatalf("Store failed: %v", err)
- }
-
- sidecars, err := filepath.Glob(filepath.Join(dir, "*", "*.attrs"))
- if err != nil {
- t.Fatalf("Glob failed: %v", err)
- }
- if len(sidecars) != 0 {
- t.Errorf("got sidecar files %v, want none: a truncated sidecar fails reads that overlap a write", sidecars)
- }
-}
-
-// A read overlapping a write to the same key must not fail. fileblob rewrote
-// its ".attrs" sidecar in place, so a reader decoding it mid-write saw a
-// partial file, which the proxy served as a 502 on an artifact it held.
-func TestConcurrentReadsSurviveWritesToSameKey(t *testing.T) {
- if runtime.GOOS == "windows" {
- // Go opens files without FILE_SHARE_DELETE, so a writer cannot replace
- // a file a reader holds open: its rename fails with access denied
- // instead of contending. The other platforms exercise this race.
- t.Skip("Windows refuses to replace a file readers hold open")
- }
- const (
- key = "pkg/thing-1.0.0.tgz"
- readers = 4
- readsPerRead = 500
- )
- dir := t.TempDir()
- ctx := context.Background()
-
- b, err := OpenBucket(ctx, fileURLFromPath(dir))
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
- defer func() { _ = b.Close() }()
-
- payload := strings.Repeat("x", 4096)
- if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil {
- t.Fatalf("seeding Store failed: %v", err)
- }
-
- // The writer reports how it ended: a Store failure would otherwise stop
- // the writes silently and let zero read failures pass for a test that
- // never contended anything.
- done := make(chan struct{})
- var writers sync.WaitGroup
- var writes int
- var writeErr error
- writers.Add(1)
- go func() {
- defer writers.Done()
- for {
- select {
- case <-done:
- return
- default:
- }
- if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil {
- writeErr = err
- return
- }
- writes++
- }
- }()
-
- var failures atomic.Int64
- var reading sync.WaitGroup
- for range readers {
- reading.Add(1)
- go func() {
- defer reading.Done()
- for range readsPerRead {
- r, err := b.Open(ctx, key)
- if err != nil {
- failures.Add(1)
- continue
- }
- if _, err := io.Copy(io.Discard, r); err != nil {
- failures.Add(1)
- }
- _ = r.Close()
- }
- }()
- }
- reading.Wait()
- close(done)
- writers.Wait()
-
- if writeErr != nil {
- t.Fatalf("writer stopped early: %v", writeErr)
- }
- if writes == 0 {
- t.Fatal("no write completed, so the reads were never contended")
- }
- if got := failures.Load(); got != 0 {
- t.Errorf("%d of %d reads failed while one writer rewrote the same key, want 0", got, readers*readsPerRead)
- }
-}
-
-// seedLegacySidecar stores key through a bucket that still writes sidecars, as
-// an earlier version did, and returns the path fileblob actually used. It is
-// discovered rather than assumed, so callers test the real mapping.
-func seedLegacySidecar(t *testing.T, dir, key, payload string) string {
- t.Helper()
- ctx := context.Background()
-
- legacy, err := blob.OpenBucket(ctx, fileURLFromPath(dir)+"?no_tmp_dir=true")
- if err != nil {
- t.Fatalf("opening legacy bucket: %v", err)
- }
- if err := legacy.WriteAll(ctx, key, []byte(payload), nil); err != nil {
- t.Fatalf("legacy WriteAll: %v", err)
- }
- if err := legacy.Close(); err != nil {
- t.Fatalf("closing legacy bucket: %v", err)
- }
-
- var found []string
- walkErr := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
- if err != nil {
- return err
- }
- if !d.IsDir() && strings.HasSuffix(path, ".attrs") {
- found = append(found, path)
- }
- return nil
- })
- if walkErr != nil {
- t.Fatalf("walking %s: %v", dir, walkErr)
- }
- if len(found) != 1 {
- t.Fatalf("got sidecars %v, want exactly one", found)
- }
- return found[0]
-}
-
-// An interrupted setAttrs leaves a partial sidecar that fails every read of the
-// key, and nothing rewrites one now, so a store has to clear it.
-//
-// One key per storage path the proxy builds: legacy artifact keys across ecosystems,
-// metadata blobs, and the Gradle build cache. Scoped npm names, Go's "!" case
-// escaping and the ":" in OCI digests and Debian epochs are the characters
-// most likely to part fileblob's mapping from a plain path join.
-func TestStoreClearsLegacyAttrsSidecar(t *testing.T) {
- keys := []string{
- "npm/@babel/core/7.24.0/core-7.24.0.tgz",
- "maven/org.apache.commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar",
- "golang/github.com/!burnt!sushi/toml/v1.3.2/v1.3.2.zip",
- "oci/library/nginx/sha256:abc123def456/manifest",
- "debian/tzdata/1:2024a-1/tzdata_2024a-1_all.deb",
- "pypi/requests/2.31.0/requests-2.31.0-py3-none-any.whl",
- "cargo/serde/1.0.197/serde-1.0.197.crate",
- "julia/Example/a1b2c3/a1b2c3.tar.gz",
- "conda/numpy/1.26.4/numpy-1.26.4-py311.conda",
- "_metadata/npm/@babel/core/metadata",
- "_gradle/http-build-cache/0a1b2c3d4e5f",
- // Keys fileblob escapes on every platform.
- "npm/pkg//1.0.0/x.tgz",
- "npm/pkg/../1.0.0/x.tgz",
- }
-
- for _, key := range keys {
- t.Run(key, func(t *testing.T) {
- assertStoreClearsSidecar(t, key)
- })
- }
-}
-
-func assertStoreClearsSidecar(t *testing.T, key string) {
- t.Helper()
- const payload = "payload"
- ctx := context.Background()
- dir := t.TempDir()
-
- sidecar := seedLegacySidecar(t, dir, key, payload)
- if err := os.WriteFile(sidecar, []byte(`{"user.content_type":"appl`), 0o600); err != nil {
- t.Fatalf("corrupting sidecar: %v", err)
- }
-
- b := openFileBlob(t, dir)
- reader, err := b.bucket.NewReader(ctx, key, nil)
- if err == nil {
- _ = reader.Close()
- t.Fatal("corrupt sidecar did not fail the read, so it is not the file fileblob reads for this key")
- }
-
- derived := b.legacySidecarPath(key)
- if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil {
- t.Fatalf("Store failed: %v", err)
- }
-
- if derived == "" {
- t.Fatalf("legacySidecarPath declined %q, but fileblob wrote %q", key, sidecar)
- }
- if derived != sidecar {
- t.Fatalf("derived %q, but fileblob wrote %q", derived, sidecar)
- }
- if _, err := os.Stat(sidecar); !os.IsNotExist(err) {
- t.Errorf("sidecar still present after Store, stat err = %v", err)
- }
- assertReadsBack(t, b, key, payload)
-}
-
-func assertReadsBack(t *testing.T, b *Blob, key, want string) {
- t.Helper()
- r, err := b.Open(context.Background(), key)
- if err != nil {
- t.Fatalf("read still failing after Store cleared the sidecar: %v", err)
- }
- defer func() { _ = r.Close() }()
- got, err := io.ReadAll(r)
- if err != nil {
- t.Fatalf("ReadAll failed: %v", err)
- }
- if string(got) != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func openFileBlob(t *testing.T, dir string) *Blob {
- t.Helper()
- s, err := OpenBucket(context.Background(), fileURLFromPath(dir))
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
- t.Cleanup(func() { _ = s.Close() })
- b, ok := s.(*Blob)
- if !ok {
- t.Fatalf("got %T, want *Blob", s)
- }
- return b
-}
-
-// fileblob escapes a non-local key in a way this cannot reproduce, and one
-// holding ".." resolves outside the cache directory. Removal declines both
-// rather than delete the wrong file.
-func TestLegacySidecarPathEscapesLikeFileblob(t *testing.T) {
- root := filepath.FromSlash("/var/cache/proxy")
- b := &Blob{fileRoot: root}
- windows := runtime.GOOS == osWindows
-
- for _, tc := range []struct{ key, unix, windows string }{
- {"npm/pkg/1.0.0/x.tgz", "npm/pkg/1.0.0/x.tgz", "npm/pkg/1.0.0/x.tgz"},
- {"npm/pkg//1.0.0/x.tgz", "npm/pkg/__0x2f__1.0.0/x.tgz", "npm/pkg/__0x2f__1.0.0/x.tgz"},
- {"npm/pkg/../../etc/passwd", "npm/pkg/..__0x2f__..__0x2f__etc/passwd", "npm/pkg/..__0x2f__..__0x2f__etc/passwd"},
- {"npm/pkg/1.0.0/", "npm/pkg/1.0.0__0x2f__", "npm/pkg/1.0.0__0x2f__"},
- {"npm/a\x01b", "npm/a__0x1__b", "npm/a__0x1__b"},
- {"oci/nginx/sha256:abc/manifest", "oci/nginx/sha256:abc/manifest", "oci/nginx/sha256__0x3a__abc/manifest"},
- {"debian/tzdata/1:2024a-1/x.deb", "debian/tzdata/1:2024a-1/x.deb", "debian/tzdata/1__0x3a__2024a-1/x.deb"},
- {`npm/a\b`, `npm/a\b`, "npm/a__0x5c__b"},
- } {
- want := tc.unix
- if windows {
- want = tc.windows
- }
- want = filepath.Join(root, filepath.FromSlash(want)) + attrsExt
- if got := b.legacySidecarPath(tc.key); got != want {
- t.Errorf("legacySidecarPath(%q) = %q, want %q", tc.key, got, want)
- }
- }
-}
-
-func TestLegacySidecarPathDeclinesNonLocalKeys(t *testing.T) {
- b := &Blob{fileRoot: filepath.FromSlash("/var/cache/proxy")}
-
- for _, key := range []string{"", ".", "..", "/etc/passwd"} {
- if got := b.legacySidecarPath(key); got != "" {
- t.Errorf("legacySidecarPath(%q) = %q, want \"\"", key, got)
- }
- }
-}
-
-// Cloud backends have no local directory, so nothing is removed for them.
-func TestLegacySidecarPathEmptyForCloudBackends(t *testing.T) {
- b := &Blob{}
- if got := b.legacySidecarPath("npm/pkg/1.0.0/x.tgz"); got != "" {
- t.Errorf("legacySidecarPath = %q, want \"\" when there is no file root", got)
- }
-}
-
-// Cleanup that cannot complete must not fail the write. A non-empty directory
-// at the sidecar path makes os.Remove fail with something other than not-exist
-// on every platform, which is what a Windows sharing violation would look like
-// here.
-func TestStoreSucceedsWhenSidecarCannotBeRemoved(t *testing.T) {
- const key = "npm/pkg/1.0.0/pkg-1.0.0.tgz"
- const payload = "payload"
- dir := t.TempDir()
- ctx := context.Background()
-
- b := openFileBlob(t, dir)
-
- sidecar := filepath.Join(dir, filepath.FromSlash(key)) + ".attrs"
- if err := os.MkdirAll(filepath.Join(sidecar, "blocker"), 0o750); err != nil {
- t.Fatalf("seeding an unremovable sidecar: %v", err)
- }
- if err := os.Remove(sidecar); err == nil {
- t.Fatal("sidecar path was removable, so the test proves nothing")
- }
-
- if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil {
- t.Errorf("Store failed because cleanup could not complete: %v", err)
- }
-}
-
-type failingReader struct {
- data []byte
- err error
-}
-
-func (r *failingReader) Read(p []byte) (int, error) {
- if len(r.data) == 0 {
- return 0, r.err
- }
- n := copy(p, r.data)
- r.data = r.data[n:]
- return n, nil
-}
-
-// A download that breaks off midway must not leave its first half behind as
-// an object, or every failed fetch adds an orphan nothing ever deletes.
-func TestBlobStoreFailedCopyLeavesNoObject(t *testing.T) {
- b := createTestBlob(t)
- ctx := context.Background()
- readErr := errors.New("connection reset")
-
- _, _, err := b.Store(ctx, "npm/lodash/4.17.21/lodash.tgz", &failingReader{
- data: []byte(strings.Repeat("partial", 1024)),
- err: readErr,
- })
- if !errors.Is(err, readErr) {
- t.Fatalf("Store error = %v, want %v", err, readErr)
- }
-
- exists, err := b.Exists(ctx, "npm/lodash/4.17.21/lodash.tgz")
- if err != nil {
- t.Fatalf("Exists failed: %v", err)
- }
- if exists {
- t.Error("failed Store left a partial object behind")
- }
-}
diff --git a/internal/storage/filesystem.go b/internal/storage/filesystem.go
new file mode 100644
index 0000000..01be90d
--- /dev/null
+++ b/internal/storage/filesystem.go
@@ -0,0 +1,174 @@
+package storage
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+)
+
+// Filesystem implements Storage using the local filesystem.
+type Filesystem struct {
+ root string
+}
+
+// NewFilesystem creates a new filesystem storage rooted at the given directory.
+// The directory will be created if it does not exist.
+func NewFilesystem(root string) (*Filesystem, error) {
+ absRoot, err := filepath.Abs(root)
+ if err != nil {
+ return nil, fmt.Errorf("resolving root path: %w", err)
+ }
+
+ if err := os.MkdirAll(absRoot, 0755); err != nil {
+ return nil, fmt.Errorf("creating root directory: %w", err)
+ }
+
+ return &Filesystem{root: absRoot}, nil
+}
+
+func (fs *Filesystem) fullPath(path string) string {
+ return filepath.Join(fs.root, filepath.FromSlash(path))
+}
+
+func (fs *Filesystem) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
+ fullPath := fs.fullPath(path)
+
+ dir := filepath.Dir(fullPath)
+ if err := os.MkdirAll(dir, 0755); err != nil {
+ return 0, "", fmt.Errorf("creating directory: %w", err)
+ }
+
+ // Write to temp file first for atomic operation
+ tmpFile, err := os.CreateTemp(dir, ".tmp-*")
+ if err != nil {
+ return 0, "", fmt.Errorf("creating temp file: %w", err)
+ }
+ tmpPath := tmpFile.Name()
+
+ // Clean up temp file on error
+ success := false
+ defer func() {
+ if !success {
+ _ = tmpFile.Close()
+ _ = os.Remove(tmpPath)
+ }
+ }()
+
+ // Write content and compute hash
+ h := sha256.New()
+ w := io.MultiWriter(tmpFile, h)
+
+ size, err := io.Copy(w, r)
+ if err != nil {
+ return 0, "", fmt.Errorf("writing content: %w", err)
+ }
+
+ if err := tmpFile.Close(); err != nil {
+ return 0, "", fmt.Errorf("closing temp file: %w", err)
+ }
+
+ // Atomic rename
+ if err := os.Rename(tmpPath, fullPath); err != nil {
+ return 0, "", fmt.Errorf("renaming temp file: %w", err)
+ }
+
+ success = true
+ hash := hex.EncodeToString(h.Sum(nil))
+ return size, hash, nil
+}
+
+func (fs *Filesystem) Open(ctx context.Context, path string) (io.ReadCloser, error) {
+ fullPath := fs.fullPath(path)
+
+ f, err := os.Open(fullPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, ErrNotFound
+ }
+ return nil, fmt.Errorf("opening file: %w", err)
+ }
+
+ return f, nil
+}
+
+func (fs *Filesystem) Exists(ctx context.Context, path string) (bool, error) {
+ fullPath := fs.fullPath(path)
+
+ _, err := os.Stat(fullPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return false, nil
+ }
+ return false, fmt.Errorf("checking file: %w", err)
+ }
+
+ return true, nil
+}
+
+func (fs *Filesystem) Delete(ctx context.Context, path string) error {
+ fullPath := fs.fullPath(path)
+
+ err := os.Remove(fullPath)
+ if err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("removing file: %w", err)
+ }
+
+ // Try to clean up empty parent directories
+ dir := filepath.Dir(fullPath)
+ for dir != fs.root {
+ if err := os.Remove(dir); err != nil {
+ break // Directory not empty or other error
+ }
+ dir = filepath.Dir(dir)
+ }
+
+ return nil
+}
+
+func (fs *Filesystem) Size(ctx context.Context, path string) (int64, error) {
+ fullPath := fs.fullPath(path)
+
+ info, err := os.Stat(fullPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return 0, ErrNotFound
+ }
+ return 0, fmt.Errorf("stat file: %w", err)
+ }
+
+ return info.Size(), nil
+}
+
+func (fs *Filesystem) UsedSpace(ctx context.Context) (int64, error) {
+ var total int64
+
+ err := filepath.Walk(fs.root, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.IsDir() {
+ total += info.Size()
+ }
+ return nil
+ })
+ if err != nil {
+ return 0, fmt.Errorf("walking directory: %w", err)
+ }
+
+ return total, nil
+}
+
+// Root returns the root directory of the storage.
+func (fs *Filesystem) Root() string {
+ return fs.root
+}
+
+// FullPath returns the full filesystem path for a storage path.
+// Useful for serving files directly or debugging.
+func (fs *Filesystem) FullPath(path string) string {
+ return fs.fullPath(path)
+}
diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go
new file mode 100644
index 0000000..16c7447
--- /dev/null
+++ b/internal/storage/filesystem_test.go
@@ -0,0 +1,275 @@
+package storage
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestNewFilesystem(t *testing.T) {
+ dir := t.TempDir()
+ root := filepath.Join(dir, "cache")
+
+ fs, err := NewFilesystem(root)
+ if err != nil {
+ t.Fatalf("NewFilesystem failed: %v", err)
+ }
+
+ if _, err := os.Stat(root); err != nil {
+ t.Errorf("root directory not created: %v", err)
+ }
+
+ if fs.Root() != root {
+ t.Errorf("Root() = %q, want %q", fs.Root(), root)
+ }
+}
+
+func TestFilesystemStore(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+ content := "test content for storage"
+
+ size, hash, err := fs.Store(ctx, "npm/lodash/4.17.21/lodash.tgz", strings.NewReader(content))
+ if err != nil {
+ t.Fatalf("Store failed: %v", err)
+ }
+
+ if size != int64(len(content)) {
+ t.Errorf("size = %d, want %d", size, len(content))
+ }
+
+ h := sha256.Sum256([]byte(content))
+ wantHash := hex.EncodeToString(h[:])
+ if hash != wantHash {
+ t.Errorf("hash = %s, want %s", hash, wantHash)
+ }
+
+ // Verify file exists on disk
+ fullPath := fs.FullPath("npm/lodash/4.17.21/lodash.tgz")
+ data, err := os.ReadFile(fullPath)
+ if err != nil {
+ t.Fatalf("reading stored file: %v", err)
+ }
+ if string(data) != content {
+ t.Errorf("stored content = %q, want %q", string(data), content)
+ }
+}
+
+func TestFilesystemStoreAtomic(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ // Store initial content
+ _, _, err := fs.Store(ctx, "test/file.txt", strings.NewReader("initial"))
+ if err != nil {
+ t.Fatalf("initial Store failed: %v", err)
+ }
+
+ // Overwrite with new content
+ _, _, err = fs.Store(ctx, "test/file.txt", strings.NewReader("updated"))
+ if err != nil {
+ t.Fatalf("update Store failed: %v", err)
+ }
+
+ // Verify updated content
+ r, err := fs.Open(ctx, "test/file.txt")
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ defer func() { _ = r.Close() }()
+
+ data, _ := io.ReadAll(r)
+ if string(data) != "updated" {
+ t.Errorf("content = %q, want %q", string(data), "updated")
+ }
+}
+
+func TestFilesystemOpen(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+ content := "readable content"
+
+ _, _, _ = fs.Store(ctx, "test/read.txt", strings.NewReader(content))
+
+ r, err := fs.Open(ctx, "test/read.txt")
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ defer func() { _ = r.Close() }()
+
+ data, err := io.ReadAll(r)
+ if err != nil {
+ t.Fatalf("ReadAll failed: %v", err)
+ }
+ if string(data) != content {
+ t.Errorf("content = %q, want %q", string(data), content)
+ }
+}
+
+func TestFilesystemOpenNotFound(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ _, err := fs.Open(ctx, "does/not/exist.txt")
+ if !errors.Is(err, ErrNotFound) {
+ t.Errorf("Open non-existent = %v, want ErrNotFound", err)
+ }
+}
+
+func TestFilesystemExists(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ exists, err := fs.Exists(ctx, "test/exists.txt")
+ if err != nil {
+ t.Fatalf("Exists failed: %v", err)
+ }
+ if exists {
+ t.Error("Exists returned true for non-existent file")
+ }
+
+ _, _, _ = fs.Store(ctx, "test/exists.txt", strings.NewReader("content"))
+
+ exists, err = fs.Exists(ctx, "test/exists.txt")
+ if err != nil {
+ t.Fatalf("Exists after store failed: %v", err)
+ }
+ if !exists {
+ t.Error("Exists returned false for existing file")
+ }
+}
+
+func TestFilesystemDelete(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ _, _, _ = fs.Store(ctx, "test/delete/nested/file.txt", strings.NewReader("content"))
+
+ err := fs.Delete(ctx, "test/delete/nested/file.txt")
+ if err != nil {
+ t.Fatalf("Delete failed: %v", err)
+ }
+
+ exists, _ := fs.Exists(ctx, "test/delete/nested/file.txt")
+ if exists {
+ t.Error("file still exists after delete")
+ }
+
+ // Empty parent directories should be cleaned up
+ nestedDir := fs.FullPath("test/delete/nested")
+ if _, err := os.Stat(nestedDir); !os.IsNotExist(err) {
+ t.Error("empty nested directory not cleaned up")
+ }
+}
+
+func TestFilesystemDeleteNotFound(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ // Delete non-existent file should not error
+ err := fs.Delete(ctx, "does/not/exist.txt")
+ if err != nil {
+ t.Errorf("Delete non-existent = %v, want nil", err)
+ }
+}
+
+func TestFilesystemSize(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+ content := "size test content"
+
+ _, _, _ = fs.Store(ctx, "test/size.txt", strings.NewReader(content))
+
+ size, err := fs.Size(ctx, "test/size.txt")
+ if err != nil {
+ t.Fatalf("Size failed: %v", err)
+ }
+ if size != int64(len(content)) {
+ t.Errorf("Size = %d, want %d", size, len(content))
+ }
+}
+
+func TestFilesystemSizeNotFound(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ _, err := fs.Size(ctx, "does/not/exist.txt")
+ if !errors.Is(err, ErrNotFound) {
+ t.Errorf("Size non-existent = %v, want ErrNotFound", err)
+ }
+}
+
+func TestFilesystemUsedSpace(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ // Empty storage
+ used, err := fs.UsedSpace(ctx)
+ if err != nil {
+ t.Fatalf("UsedSpace failed: %v", err)
+ }
+ if used != 0 {
+ t.Errorf("UsedSpace empty = %d, want 0", used)
+ }
+
+ // Add some files
+ _, _, _ = fs.Store(ctx, "a.txt", strings.NewReader("aaaa")) // 4 bytes
+ _, _, _ = fs.Store(ctx, "b.txt", strings.NewReader("bbbbbb")) // 6 bytes
+ _, _, _ = fs.Store(ctx, "c/d.txt", strings.NewReader("ccccc")) // 5 bytes
+
+ used, err = fs.UsedSpace(ctx)
+ if err != nil {
+ t.Fatalf("UsedSpace failed: %v", err)
+ }
+ if used != 15 {
+ t.Errorf("UsedSpace = %d, want 15", used)
+ }
+}
+
+func TestFilesystemLargeFile(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ // 1MB of data
+ data := bytes.Repeat([]byte("x"), 1024*1024)
+
+ size, hash, err := fs.Store(ctx, "large/file.bin", bytes.NewReader(data))
+ if err != nil {
+ t.Fatalf("Store large file failed: %v", err)
+ }
+ if size != int64(len(data)) {
+ t.Errorf("size = %d, want %d", size, len(data))
+ }
+
+ h := sha256.Sum256(data)
+ wantHash := hex.EncodeToString(h[:])
+ if hash != wantHash {
+ t.Errorf("hash mismatch for large file")
+ }
+
+ // Read it back
+ r, _ := fs.Open(ctx, "large/file.bin")
+ defer func() { _ = r.Close() }()
+ readBack, _ := io.ReadAll(r)
+ if !bytes.Equal(readBack, data) {
+ t.Error("large file content mismatch")
+ }
+}
+
+func createTestFilesystem(t *testing.T) *Filesystem {
+ t.Helper()
+ dir := t.TempDir()
+
+ fs, err := NewFilesystem(dir)
+ if err != nil {
+ t.Fatalf("NewFilesystem failed: %v", err)
+ }
+ return fs
+}
diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go
deleted file mode 100644
index a096c18..0000000
--- a/internal/storage/gcs.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package storage
-
-import (
- "context"
- "crypto/sha256"
- "encoding/hex"
- "errors"
- "io"
- "time"
-
- gcstorage "github.com/git-pkgs/gcs"
-)
-
-// GCS adapts a Google Cloud Storage bucket to Storage.
-type GCS struct {
- bucket *gcstorage.Bucket
- url string
-}
-
-// OpenGCS opens a Google Cloud Storage bucket from a gs:// URL.
-func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) {
- bucket, err := gcstorage.OpenBucket(ctx, urlStr)
- if err != nil {
- return nil, err
- }
- return &GCS{bucket: bucket, url: urlStr}, nil
-}
-
-func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
- h := sha256.New()
- size, err := g.bucket.Write(ctx, path, io.TeeReader(r, h))
- if err != nil {
- return 0, "", err
- }
- return size, hex.EncodeToString(h.Sum(nil)), nil
-}
-
-func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) {
- r, err := g.bucket.Open(ctx, path)
- if errors.Is(err, gcstorage.ErrNotFound) {
- return nil, ErrNotFound
- }
- return r, err
-}
-
-func (g *GCS) Exists(ctx context.Context, path string) (bool, error) {
- return g.bucket.Exists(ctx, path)
-}
-
-func (g *GCS) Delete(ctx context.Context, path string) error {
- return g.bucket.Delete(ctx, path)
-}
-
-func (g *GCS) Size(ctx context.Context, path string) (int64, error) {
- size, err := g.bucket.Size(ctx, path)
- if errors.Is(err, gcstorage.ErrNotFound) {
- return 0, ErrNotFound
- }
- return size, err
-}
-
-func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
- u, err := g.bucket.SignedURL(ctx, path, expiry)
- if errors.Is(err, gcstorage.ErrSignedURLUnsupported) {
- return "", ErrSignedURLUnsupported
- }
- return u, err
-}
-
-func (g *GCS) UsedSpace(ctx context.Context) (int64, error) {
- return g.bucket.UsedSpace(ctx)
-}
-
-func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
- objects, err := g.bucket.ListPrefix(ctx, prefix)
- if err != nil {
- return nil, err
- }
-
- result := make([]ObjectInfo, 0, len(objects))
- for _, object := range objects {
- result = append(result, ObjectInfo{
- Path: object.Name,
- Size: object.Size,
- ModTime: object.ModTime,
- })
- }
- return result, nil
-}
-
-func (g *GCS) Close() error {
- return nil
-}
-
-func (g *GCS) URL() string {
- return g.url
-}
diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go
deleted file mode 100644
index 1bd8a40..0000000
--- a/internal/storage/gcs_test.go
+++ /dev/null
@@ -1,160 +0,0 @@
-package storage
-
-import (
- "context"
- "crypto/sha256"
- "encoding/hex"
- "encoding/json"
- "errors"
- "io"
- "net/http"
- "net/http/httptest"
- "net/url"
- "sort"
- "strconv"
- "strings"
- "testing"
- "time"
-)
-
-func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) {
- server := httptest.NewServer(&fakeGCSServer{t: t, objects: map[string]string{}})
- defer server.Close()
- t.Setenv("STORAGE_EMULATOR_HOST", server.URL)
-
- ctx := context.Background()
- store, err := OpenBucket(ctx, "gs://test-bucket")
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
-
- size, hash, err := store.Store(ctx, "npm/pkg/file.tgz", strings.NewReader("content"))
- if err != nil {
- t.Fatalf("Store failed: %v", err)
- }
- wantHash := sha256.Sum256([]byte("content"))
- if size != int64(len("content")) || hash != hex.EncodeToString(wantHash[:]) {
- t.Fatalf("Store returned size=%d hash=%q", size, hash)
- }
-
- exists, err := store.Exists(ctx, "npm/pkg/file.tgz")
- if err != nil || !exists {
- t.Fatalf("Exists = %v, %v; want true, nil", exists, err)
- }
-
- r, err := store.Open(ctx, "npm/pkg/file.tgz")
- if err != nil {
- t.Fatalf("Open failed: %v", err)
- }
- data, _ := io.ReadAll(r)
- _ = r.Close()
- if string(data) != "content" {
- t.Fatalf("Open content = %q, want content", data)
- }
-
- lister, ok := store.(interface {
- ListPrefix(context.Context, string) ([]ObjectInfo, error)
- })
- if !ok {
- t.Fatal("GCS storage does not support prefix listing")
- }
- list, err := lister.ListPrefix(ctx, "npm/")
- if err != nil {
- t.Fatalf("ListPrefix failed: %v", err)
- }
- if len(list) != 1 || list[0].Path != "npm/pkg/file.tgz" {
- t.Fatalf("ListPrefix = %#v", list)
- }
-
- if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil {
- t.Fatalf("Delete failed: %v", err)
- }
- exists, err = store.Exists(ctx, "npm/pkg/file.tgz")
- if err != nil || exists {
- t.Fatalf("Exists after delete = %v, %v; want false, nil", exists, err)
- }
- if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil {
- t.Fatalf("Delete missing object = %v, want nil", err)
- }
-
- reader, err := store.Open(ctx, "npm/pkg/file.tgz")
- if reader != nil || !errors.Is(err, ErrNotFound) {
- t.Fatalf("Open missing object = %v, %v; want nil, ErrNotFound", reader, err)
- }
- if _, err := store.Size(ctx, "npm/pkg/file.tgz"); !errors.Is(err, ErrNotFound) {
- t.Fatalf("Size missing object = %v, want ErrNotFound", err)
- }
- if _, err := store.SignedURL(ctx, "npm/pkg/file.tgz", time.Minute); !errors.Is(err, ErrSignedURLUnsupported) {
- t.Fatalf("SignedURL with emulator = %v, want ErrSignedURLUnsupported", err)
- }
-}
-
-type fakeGCSServer struct {
- t *testing.T
- objects map[string]string
-}
-
-func (f *fakeGCSServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- switch {
- case r.Method == http.MethodPost && r.URL.Path == "/upload/storage/v1/b/test-bucket/o":
- name := r.URL.Query().Get("name")
- data, _ := io.ReadAll(r.Body)
- f.objects[name] = string(data)
- writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
- case r.Method == http.MethodGet && r.URL.Path == "/storage/v1/b/test-bucket/o":
- prefix := r.URL.Query().Get("prefix")
- page := fakeGCSListResponse{}
- for name, data := range f.objects {
- if strings.HasPrefix(name, prefix) {
- page.Items = append(page.Items, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
- }
- }
- sort.Slice(page.Items, func(i, j int) bool { return page.Items[i].Name < page.Items[j].Name })
- writeJSON(w, page)
- case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"):
- name := objectNameFromPath(r.URL.Path)
- data, ok := f.objects[name]
- if !ok {
- http.NotFound(w, r)
- return
- }
- if r.URL.Query().Get("alt") == "media" {
- _, _ = io.WriteString(w, data)
- return
- }
- writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
- case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"):
- name := objectNameFromPath(r.URL.Path)
- if _, ok := f.objects[name]; !ok {
- http.NotFound(w, r)
- return
- }
- delete(f.objects, name)
- w.WriteHeader(http.StatusNoContent)
- default:
- f.t.Errorf("unexpected request: %s %s", r.Method, r.URL.String())
- http.Error(w, "unexpected request", http.StatusInternalServerError)
- }
-}
-
-type fakeGCSObject struct {
- Name string `json:"name"`
- Size string `json:"size"`
- Updated string `json:"updated"`
-}
-
-type fakeGCSListResponse struct {
- NextPageToken string `json:"nextPageToken"`
- Items []fakeGCSObject `json:"items"`
-}
-
-func writeJSON(w http.ResponseWriter, v any) {
- w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode(v)
-}
-
-func objectNameFromPath(p string) string {
- escaped := strings.TrimPrefix(p, "/storage/v1/b/test-bucket/o/")
- name, _ := url.PathUnescape(escaped)
- return name
-}
diff --git a/internal/storage/s3_upload_test.go b/internal/storage/s3_upload_test.go
deleted file mode 100644
index 898b1f7..0000000
--- a/internal/storage/s3_upload_test.go
+++ /dev/null
@@ -1,120 +0,0 @@
-package storage
-
-import (
- "bytes"
- "context"
- "fmt"
- "io"
- "maps"
- "net/http"
- "net/http/httptest"
- "slices"
- "strconv"
- "strings"
- "sync"
- "testing"
-)
-
-// fakeS3 implements just enough of the S3 API for PutObject and multipart
-// uploads, recording the size of each part it receives.
-type fakeS3 struct {
- mu sync.Mutex
- objects map[string][]byte
- parts map[int][]byte
- partSizes []int
- // discard drops bodies so a benchmark measures only the client.
- discard bool
-}
-
-func newFakeS3(t testing.TB) (*fakeS3, string) {
- t.Helper()
- f := &fakeS3{objects: map[string][]byte{}, parts: map[int][]byte{}}
- server := httptest.NewServer(f)
- t.Cleanup(server.Close)
- url := "s3://bucket?endpoint=" + server.URL + "®ion=us-east-1&use_path_style=true&anonymous=true&request_checksum_calculation=when_required"
- return f, url
-}
-
-func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- key := strings.TrimPrefix(r.URL.Path, "/bucket/")
- q := r.URL.Query()
- if f.discard {
- _, _ = io.Copy(io.Discard, r.Body)
- r.Body = http.NoBody
- }
- body, err := io.ReadAll(r.Body)
- if err != nil {
- http.Error(w, err.Error(), http.StatusBadRequest)
- return
- }
- f.mu.Lock()
- defer f.mu.Unlock()
- switch {
- case r.Method == http.MethodPost && q.Has("uploads"):
- f.parts = map[int][]byte{}
- _, _ = fmt.Fprintf(w, `bucket %s upload bucket %s "object"