diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 700f28e..9712038 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,12 +17,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: ${{ matrix.go-version }} @@ -35,12 +35,12 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: '1.25' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 93b0a1e..4844dde 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,7 +20,7 @@ jobs: contents: read steps: - name: Check out the repo - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 with: persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7833fb..dc15164 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 persist-credentials: false @@ -22,7 +22,7 @@ jobs: - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version-file: go.mod cache: false diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index f947c7e..d18ba0f 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -12,12 +12,12 @@ jobs: swagger: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: '1.25' diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index cce6e4f..1c212f7 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -21,9 +21,9 @@ jobs: security-events: write steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 diff --git a/Dockerfile b/Dockerfile index 7b2795c..f31cd05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY . . # Build the binary RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy -FROM alpine:3.24.1 +FROM alpine:3.24.0 RUN apk add --no-cache ca-certificates diff --git a/README.md b/README.md index 4609b66..3270212 100644 --- a/README.md +++ b/README.md @@ -409,7 +409,7 @@ The proxy can be configured via: -config string Path to configuration file -listen string Address to listen on (default ":8080") -base-url string Public URL of this proxy (default "http://localhost:8080") --storage-url string Storage URL (file:// or s3://) +-storage-url string Storage URL (file://, s3://, gs://, azblob://) -storage-path string Path to artifact storage directory (deprecated, use -storage-url) -database-driver string Database driver: sqlite or postgres (default "sqlite") -database-path string Path to SQLite database file (default "./cache/proxy.db") @@ -504,6 +504,57 @@ storage: Set credentials via standard AWS environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`). +### Google Cloud Storage + +The proxy can store cached artifacts in a GCS bucket using the `gs://` URL scheme. + +```yaml +storage: + url: "gs://my-bucket-name" +``` + +Authentication uses [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials), which means no credentials need to be embedded in the config or environment. Supported sources, in order: + +- **GKE Workload Identity** — bind the Kubernetes service account running the proxy to a Google service account that has `roles/storage.objectAdmin` on the bucket. The proxy will use the workload's token automatically. +- **Attached service account** on GCE, Cloud Run, Cloud Functions, etc. +- **`GOOGLE_APPLICATION_CREDENTIALS`** environment variable pointing at a service account JSON key file. +- **`gcloud auth application-default login`** for local development. + +#### GKE Workload Identity setup + +```bash +# 1. Create a Google service account +gcloud iam service-accounts create git-pkgs-proxy \ + --project=PROJECT_ID + +# 2. Grant it access to the bucket +gsutil iam ch \ + serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com:objectAdmin \ + gs://my-bucket-name + +# 3. Bind the Kubernetes service account to it +gcloud iam service-accounts add-iam-policy-binding \ + git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/iam.workloadIdentityUser \ + --member="serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/KSA_NAME]" + +# 4. Annotate the Kubernetes service account +kubectl annotate serviceaccount KSA_NAME \ + --namespace=NAMESPACE \ + iam.gke.io/gcp-service-account=git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com +``` + +#### Direct serve (signed URLs) with Workload Identity + +When `direct_serve: true` is enabled, the proxy issues HTTP 302 redirects to presigned GCS URLs. Because Workload Identity provides no private key, the gcsblob driver falls back to the [IAM Credentials `signBlob` API](https://cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob). For this to work, grant the service account the token-creator role on itself: + +```bash +gcloud iam service-accounts add-iam-policy-binding \ + git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/iam.serviceAccountTokenCreator \ + --member="serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com" +``` + ## CLI Commands ### serve (default) diff --git a/config.example.yaml b/config.example.yaml index 62b4105..bb080ec 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -22,9 +22,20 @@ storage: # - file:///path/to/dir - Local filesystem (default) # - s3://bucket-name - Amazon S3 # - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) + # - gs://bucket-name - Google Cloud Storage + # - azblob://container-name - Azure Blob Storage # # For S3, configure credentials via environment variables: # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION + # + # For GCS, authentication uses Application Default Credentials. On GKE with + # Workload Identity, bind the Kubernetes service account to a Google service + # account that has roles/storage.objectAdmin on the bucket. No extra config + # is needed in this file. For local development, run: + # gcloud auth application-default login + # If direct_serve is enabled, the service account also needs + # roles/iam.serviceAccountTokenCreator on itself so the IAM Credentials + # signBlob API can sign URLs without a private key. url: "" # Local filesystem path (used when url is empty) @@ -37,7 +48,7 @@ storage: max_size: "" # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) - # instead of streaming through the proxy. Only effective for S3 and Azure. + # instead of streaming through the proxy. Only effective for S3, GCS, and Azure. # Leave disabled if clients reach the proxy through an authenticating gateway, # since presigned URLs bypass it. direct_serve: false diff --git a/go.mod b/go.mod index 471ab53..44aec66 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,17 @@ module github.com/git-pkgs/proxy go 1.25.6 require ( + cloud.google.com/go/compute/metadata v0.9.0 github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.11.0 github.com/git-pkgs/archives v0.3.0 github.com/git-pkgs/cooldown v0.1.1 - github.com/git-pkgs/enrichment v0.4.1 - github.com/git-pkgs/purl v0.1.13 - github.com/git-pkgs/registries v0.6.2 + github.com/git-pkgs/enrichment v0.3.0 + github.com/git-pkgs/purl v0.1.12 + github.com/git-pkgs/registries v0.6.1 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.2.6 - github.com/git-pkgs/vulns v0.1.6 + github.com/git-pkgs/vulns v0.1.5 github.com/go-chi/chi/v5 v5.3.0 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 @@ -21,10 +22,11 @@ require ( github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 gocloud.dev v0.46.0 - golang.org/x/sync v0.21.0 + golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.20.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.53.0 + modernc.org/sqlite v1.51.0 ) require ( @@ -32,7 +34,6 @@ require ( 4d63.com/gochecknoglobals v0.2.2 // indirect cloud.google.com/go/auth v0.18.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/compute/metadata v0.9.0 // indirect codeberg.org/chavacava/garif v0.2.0 // indirect codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect @@ -111,11 +112,11 @@ require ( github.com/curioswitch/go-reassign v0.3.0 // indirect github.com/daixiang0/gci v0.13.7 // indirect github.com/dave/dst v0.27.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ecosyste-ms/ecosystems-go v0.2.0 // indirect + github.com/ecosyste-ms/ecosystems-go v0.1.1 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/fatih/color v1.18.0 // indirect @@ -125,7 +126,7 @@ require ( github.com/fzipp/gocyclo v0.6.0 // indirect github.com/ghostiam/protogetter v0.3.20 // indirect github.com/git-pkgs/packageurl-go v0.3.1 // indirect - github.com/git-pkgs/pom v0.1.5 // indirect + github.com/git-pkgs/pom v0.1.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-critic/go-critic v0.14.3 // indirect github.com/go-logr/logr v1.4.3 // indirect @@ -217,13 +218,13 @@ require ( github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect - github.com/oapi-codegen/runtime v1.4.1 // indirect + github.com/oapi-codegen/runtime v1.2.0 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect github.com/quasilyte/go-ruleguard v0.4.5 // indirect @@ -291,15 +292,14 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect + golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect - golang.org/x/mod v0.36.0 // indirect - golang.org/x/net v0.54.0 // indirect - golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/mod v0.33.0 // indirect + golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/text v0.35.0 // indirect + golang.org/x/tools v0.42.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect @@ -307,7 +307,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect - modernc.org/libc v1.73.4 // indirect + modernc.org/libc v1.72.3 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect diff --git a/go.sum b/go.sum index 8d8c96d..ff69d3b 100644 --- a/go.sum +++ b/go.sum @@ -209,16 +209,17 @@ github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEy github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ecosyste-ms/ecosystems-go v0.2.0 h1:Nhpg54C+St8Sd/mf8bNJmQqx35ZgHw33TfFoxaXMQI8= -github.com/ecosyste-ms/ecosystems-go v0.2.0/go.mod h1:CCdzT1iAZirbEZAbFSnWpK88eKKaIWex7gjtZ0UudXA= +github.com/ecosyste-ms/ecosystems-go v0.1.1 h1:YYiBK9TCCTeE+BtmpN2FssaRFcmF+T0v4LrupIOjehQ= +github.com/ecosyste-ms/ecosystems-go v0.1.1/go.mod h1:VczXs1CO9nL8XbL1NwvgmwIaqzMsAxcsXnTpRtwi9gU= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= @@ -248,22 +249,22 @@ github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= -github.com/git-pkgs/enrichment v0.4.1 h1:A8BKs0XwvpF1sF5qviZy4fkJAe18qB9OgpbRnmwnT34= -github.com/git-pkgs/enrichment v0.4.1/go.mod h1:stHqZUitV9ZkwACqHzBysLMSe6T4QZn81hxTdSroNhM= +github.com/git-pkgs/enrichment v0.3.0 h1:tsATMAwR/qcSIw4JV37uH2TBgTv415RfJC7w3nzWfD4= +github.com/git-pkgs/enrichment v0.3.0/go.mod h1:MBv5nhHzjwLxeSgx2+7waCcpReUjhCD+9B0bvufpMO0= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= -github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= -github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= -github.com/git-pkgs/purl v0.1.13 h1:at8BU6vnP5oonHFHAPA064BzgRqij+SZcOUDgNT2DC8= -github.com/git-pkgs/purl v0.1.13/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= -github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+u0a8= -github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= +github.com/git-pkgs/pom v0.1.4 h1:C6st+XSbF75eKuwfdkDZZtYHoTcaWRIEQYar5VtszUo= +github.com/git-pkgs/pom v0.1.4/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= +github.com/git-pkgs/purl v0.1.12 h1:qCskrEU1LWQhCkIVZd992W5++Bsxazvx2Cx1/65qCvU= +github.com/git-pkgs/purl v0.1.12/go.mod h1:ofp4mHsR0cUeVONQaf33n6Wxg2QTEvtUdRfCedI8ouA= +github.com/git-pkgs/registries v0.6.1 h1:xZfVZQmffIfdeJthn5o2EozbVJ6gBeImYwKQnfdKUfU= +github.com/git-pkgs/registries v0.6.1/go.mod h1:a3BP/56VW3O/CFRqiJCtSy+OqRrSH25wF1PWHP76ka0= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8= github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= -github.com/git-pkgs/vulns v0.1.6 h1:8RRSgdlxp4JMU0Zykr63XTOMo5CyZKwt/PwaQxrx9Yg= -github.com/git-pkgs/vulns v0.1.6/go.mod h1:TsZC4MjoCkKJslgmbcmRCnytwnFcjESC2N8b0a2xDWc= +github.com/git-pkgs/vulns v0.1.5 h1:mtX88/27toFl+B95kaH5QbAdOCQ3YIDGjJrlrrnqQTE= +github.com/git-pkgs/vulns v0.1.5/go.mod h1:bZFikfrR/5gC0ZMwXh7qcEu2gpKfXMBhVsy4kF12Ae0= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= @@ -510,10 +511,8 @@ github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= github.com/nunnatsa/ginkgolinter v0.23.0 h1:x3o4DGYOWbBMP/VdNQKgSj+25aJKx2Pe6lHr8gBcgf8= github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= -github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= -github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= -github.com/oapi-codegen/runtime v1.4.1 h1:9nwLoI+KrWxzbBcp0jO/R8uXqbik/HUyCvPeU68Y/qo= -github.com/oapi-codegen/runtime v1.4.1/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/oapi-codegen/runtime v1.2.0 h1:RvKc1CVS1QeKSNzO97FBQbSMZyQ8s6rZd+LpmzwHMP4= +github.com/oapi-codegen/runtime v1.2.0/go.mod h1:Y7ZhmmlE8ikZOmuHRRndiIm7nf3xcVv+YMweKgG1DT0= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= @@ -539,8 +538,9 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -738,8 +738,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= @@ -755,8 +755,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -771,8 +771,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -784,8 +784,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -823,8 +823,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -839,8 +839,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= @@ -882,20 +882,20 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= -modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= -modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= -modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= +modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= -modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= -modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= +modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -904,8 +904,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= -modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/sqlite v1.51.0 h1:aH/MMSoayAIhozZ7uJbVTT9QO/VhzBf0J9tymmmuC/U= +modernc.org/sqlite v1.51.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/internal/config/config.go b/internal/config/config.go index 16928dc..d70ff2e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,10 +24,23 @@ // storage: // url: "s3://bucket?endpoint=http://localhost:9000" // +// Google Cloud Storage: +// +// storage: +// url: "gs://bucket-name" +// // For S3, configure credentials via AWS environment variables: // // AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION // +// For GCS, authentication uses Application Default Credentials. This works +// transparently on GKE with Workload Identity, GCE/Cloud Run with attached +// service accounts, or locally via `gcloud auth application-default login`. +// When the proxy is signing URLs (direct_serve) without a private key, +// gcsblob automatically falls back to the IAM Credentials signBlob API, +// which requires the service account to hold roles/iam.serviceAccountTokenCreator +// on itself. +// // Database Configuration: // // The proxy supports two database backends: @@ -141,6 +154,8 @@ type StorageConfig struct { // - file:///path/to/dir - Local filesystem (default) // - s3://bucket-name - Amazon S3 // - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) + // - gs://bucket-name - Google Cloud Storage (Workload Identity supported) + // - azblob://container-name - Azure Blob Storage // If empty, defaults to file:// with the Path value. URL string `json:"url" yaml:"url"` @@ -157,7 +172,7 @@ type StorageConfig struct { // DirectServe enables redirecting cached artifact downloads to presigned // storage URLs (HTTP 302) instead of streaming bytes through the proxy. - // Only effective for backends that support URL signing (S3, Azure). + // Only effective for backends that support URL signing (S3, GCS, Azure). DirectServe bool `json:"direct_serve" yaml:"direct_serve"` // DirectServeTTL is how long presigned URLs remain valid. @@ -219,21 +234,6 @@ type DatabaseConfig struct { URL string `json:"url" yaml:"url"` } -// String returns a human-readable description of the configured database -// suitable for logging. For postgres the password in the connection URL is -// redacted; if the URL cannot be parsed only the driver name is returned to -// avoid leaking credentials. -func (d DatabaseConfig) String() string { - if d.Driver == "postgres" { - u, err := url.Parse(d.URL) - if err != nil || u.Host == "" { - return "postgres" - } - return u.Redacted() - } - return d.Path -} - // LogConfig configures logging. type LogConfig struct { // Level is the minimum log level: "debug", "info", "warn", "error". diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bb3ec74..2f4fdd2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -676,24 +676,3 @@ func TestValidateDirectServeBaseURL(t *testing.T) { t.Errorf("unexpected error for valid direct_serve_base_url: %v", err) } } - -func TestDatabaseConfigString(t *testing.T) { - tests := []struct { - name string - cfg DatabaseConfig - want string - }{ - {"sqlite", DatabaseConfig{Driver: "sqlite", Path: "./cache/proxy.db"}, "./cache/proxy.db"}, - {"default driver", DatabaseConfig{Path: "/var/lib/proxy.db"}, "/var/lib/proxy.db"}, - {"postgres no password", DatabaseConfig{Driver: "postgres", URL: "postgres://user@localhost:5432/proxy"}, "postgres://user@localhost:5432/proxy"}, - {"postgres redacts password", DatabaseConfig{Driver: "postgres", URL: "postgres://user:secret@localhost:5432/proxy?sslmode=disable"}, "postgres://user:xxxxx@localhost:5432/proxy?sslmode=disable"}, - {"postgres unparseable url", DatabaseConfig{Driver: "postgres", URL: "host=localhost user=foo password=bar"}, "postgres"}, - {"postgres ignores sqlite path", DatabaseConfig{Driver: "postgres", URL: "postgres://localhost/db", Path: "./cache/proxy.db"}, "postgres://localhost/db"}, - } - - for _, tt := range tests { - if got := tt.cfg.String(); got != tt.want { - t.Errorf("%s: String() = %q, want %q", tt.name, got, tt.want) - } - } -} diff --git a/internal/handler/composer.go b/internal/handler/composer.go index bc3bc1d..065ddf9 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1,7 +1,6 @@ package handler import ( - "context" "encoding/json" "errors" "fmt" @@ -308,43 +307,41 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) h.proxy.Logger.Info("composer download request", "package", packageName, "version", version, "filename", filename) - // We need to fetch the metadata to get the actual download URL since - // Packagist URLs include a hash. Packagist serves dev versions (e.g. - // "3.x-dev", "dev-master") from a separate "~dev" metadata file, while - // tagged releases live in the regular file. Try the file most likely to - // contain this version first, then fall back to the other so that both - // stable and dev versions resolve correctly. - metaURLs := h.metadataURLsForVersion(vendor, pkg, version) + // We need to fetch the metadata to get the actual download URL + // since Packagist URLs include a hash + metaURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) - h.proxy.Logger.Debug("resolving download URL", - "package", packageName, "version", version, - "metadata_urls", metaURLs) - - var downloadURL string - for _, metaURL := range metaURLs { - url, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) - if err != nil { - h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL) - http.Error(w, "failed to fetch metadata", http.StatusBadGateway) - return - } - if url != "" { - downloadURL = url - break - } - } - - if downloadURL == "" { - h.proxy.Logger.Debug("version not found in any metadata source", - "package", packageName, "version", version, - "tried_urls", metaURLs) - http.Error(w, "version not found", http.StatusNotFound) + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, metaURL, nil) + if err != nil { + http.Error(w, "failed to create request", http.StatusInternalServerError) return } - h.proxy.Logger.Debug("resolved download URL", - "package", packageName, "version", version, - "download_url", downloadURL) + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.proxy.Logger.Error("failed to fetch metadata", "error", err) + http.Error(w, "failed to fetch metadata", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + http.Error(w, "package not found", http.StatusNotFound) + return + } + + var metadata map[string]any + if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { + http.Error(w, "failed to parse metadata", http.StatusInternalServerError) + return + } + + // Find the download URL for this version + downloadURL := h.findDownloadURL(metadata, packageName, version) + if downloadURL == "" { + http.Error(w, "version not found", http.StatusNotFound) + return + } result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL) if err != nil { @@ -356,82 +353,6 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) ServeArtifact(w, result) } -// isDevVersion reports whether a Composer version string refers to a -// development (unstable, branch) version rather than a tagged release. -// Composer formats these as either "dev-" (e.g. "dev-master") or -// "-dev" (e.g. "3.x-dev"). -func isDevVersion(version string) bool { - return strings.HasPrefix(version, "dev-") || strings.HasSuffix(version, "-dev") -} - -// metadataURLsForVersion returns the upstream metadata URLs to consult for a -// given version, in priority order. Dev versions are served from the "~dev" -// file, tagged releases from the regular file; the other file is included as a -// fallback so an unexpected classification still resolves. -func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string { - stable := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) - dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", h.repoURL, vendor, pkg) - - if isDevVersion(version) { - return []string{dev, stable} - } - return []string{stable, dev} -} - -// findDownloadURLFromMetadata fetches a metadata document and returns the dist -// URL for the given version, or an empty string if the version is not present. -// An error is returned only on transport failure; a missing document (non-200) -// or a missing version both yield an empty string so the caller can fall back. -func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, error) { - h.proxy.Logger.Debug("fetching upstream metadata for download lookup", - "url", metaURL, "package", packageName, "version", version) - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil) - if err != nil { - return "", err - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return "", err - } - defer func() { _ = resp.Body.Close() }() - - h.proxy.Logger.Debug("upstream metadata response", - "url", metaURL, "status", resp.StatusCode) - - if resp.StatusCode != http.StatusOK { - return "", nil - } - - var metadata map[string]any - if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { - return "", err - } - - // Expand minified Composer v2 format so that inherited fields (including - // dist) are present on every version entry. Without this, versions that - // inherit dist from a previous entry will appear to have no download URL. - if metadata["minified"] == "composer/2.0" { - h.proxy.Logger.Debug("expanding minified metadata", "url", metaURL) - if packages, ok := metadata["packages"].(map[string]any); ok { - for pkgName, versions := range packages { - versionList, ok := versions.([]any) - if !ok { - continue - } - packages[pkgName] = expandMinifiedVersions(versionList) - } - } - } - - url := h.findDownloadURL(metadata, packageName, version) - h.proxy.Logger.Debug("download URL lookup result", - "url", metaURL, "package", packageName, "version", version, - "download_url", url) - return url, nil -} - // findDownloadURL finds the dist URL for a specific version in metadata. func (h *ComposerHandler) findDownloadURL(metadata map[string]any, packageName, version string) string { packages, ok := metadata["packages"].(map[string]any) diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index 9898664..baf13b6 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -1,11 +1,8 @@ package handler import ( - "context" "encoding/json" "log/slog" - "net/http" - "net/http/httptest" "strings" "testing" "time" @@ -468,100 +465,6 @@ func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) { } } -// TestComposerDownloadDevVersionUsesDevMetadata is a regression test for the -// bug that made it impossible to install a *-dev dependency from dist. -// -// Packagist serves development versions (e.g. "3.x-dev", "dev-master") from a -// separate "{package}~dev.json" metadata file; the regular "{package}.json" -// file contains only tagged releases. The download handler used to fetch only -// the regular file, so it could never find the dist URL for a dev version and -// returned 404 — causing Composer to silently fall back to a git clone. -// -// This test serves both files from a mock upstream and asserts that: -// - the OLD behavior (regular file only) cannot resolve the dev version, and -// - the FIXED behavior (consulting the ~dev file) does. -func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { - const ( - pkg = "phpmd/phpmd" - vendor = "phpmd" - name = "phpmd" - version = "3.x-dev" - distURL = "https://api.github.com/repos/phpmd/phpmd/zipball/2a9217f60aaf27bf6ddad9188f254d020ab70745" - ) - - // Regular metadata: tagged releases only — no dev versions. - stableBody := `{ - "packages": { - "phpmd/phpmd": [ - {"version": "2.15.0", "dist": {"url": "https://example.com/2.15.0.zip", "type": "zip"}} - ] - } - }` - - // ~dev metadata: where the 3.x-dev version actually lives. - devBody := `{ - "packages": { - "phpmd/phpmd": [ - {"version": "3.x-dev", "dist": {"url": "` + distURL + `", "type": "zip"}} - ] - } - }` - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/p2/phpmd/phpmd.json": - _, _ = w.Write([]byte(stableBody)) - case "/p2/phpmd/phpmd~dev.json": - _, _ = w.Write([]byte(devBody)) - default: - http.NotFound(w, r) - } - })) - defer srv.Close() - - h := &ComposerHandler{ - proxy: testProxy(), - repoURL: srv.URL, - proxyURL: "http://localhost:8080", - } - - ctx := context.Background() - - // OLD behavior: fetching only the regular file fails to resolve the dev - // version, which is what produced the 404 before the fix. - stableURL := srv.URL + "/p2/phpmd/phpmd.json" - got, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version) - if err != nil { - t.Fatalf("unexpected error fetching regular metadata: %v", err) - } - if got != "" { - t.Fatalf("regular metadata unexpectedly contained dev version %q (got %q); "+ - "the test no longer reproduces the original bug", version, got) - } - - // FIXED behavior: the handler consults the ~dev file (it is first in the - // candidate list for dev versions) and resolves the dist URL. - urls := h.metadataURLsForVersion(vendor, name, version) - if len(urls) == 0 || !strings.HasSuffix(urls[0], "/p2/phpmd/phpmd~dev.json") { - t.Fatalf("dev version should consult the ~dev metadata file first, got %v", urls) - } - - var resolved string - for _, u := range urls { - resolved, err = h.findDownloadURLFromMetadata(ctx, u, pkg, version) - if err != nil { - t.Fatalf("unexpected error fetching metadata %q: %v", u, err) - } - if resolved != "" { - break - } - } - - if resolved != distURL { - t.Errorf("dev version dist URL = %q, want %q", resolved, distURL) - } -} - func TestComposerRewriteMetadataCooldown(t *testing.T) { now := time.Now() old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339) diff --git a/internal/server/server.go b/internal/server/server.go index 0a46a37..cb99648 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -302,7 +302,7 @@ func (s *Server) Start() error { "base_url", s.cfg.BaseURL, "ui_url", s.cfg.UIBaseURL, "storage", s.storage.URL(), - "database", s.cfg.Database.String()) + "database", s.cfg.Database.Path) go s.updateCacheStatsMetrics() go s.startEvictionLoop(bgCtx) @@ -927,7 +927,7 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { TotalSize: size, TotalSizeHuman: formatSize(size), StorageURL: s.storage.URL(), - DatabasePath: s.cfg.Database.String(), + DatabasePath: s.cfg.Database.Path, } w.Header().Set("Content-Type", "application/json") diff --git a/internal/server/templates/pages/packages_list.html b/internal/server/templates/pages/packages_list.html index 95897db..5ef42c7 100644 --- a/internal/server/templates/pages/packages_list.html +++ b/internal/server/templates/pages/packages_list.html @@ -100,7 +100,7 @@ document.getElementById('ecosystem-filter').addEventListener('change', function( const params = new URLSearchParams(); if (ecosystem) params.set('ecosystem', ecosystem); if (sortBy) params.set('sort', sortBy); - window.location.href = '/ui/packages?' + params.toString(); + window.location.href = '/packages?' + params.toString(); }); document.getElementById('sort-by').addEventListener('change', function(e) { @@ -109,7 +109,7 @@ document.getElementById('sort-by').addEventListener('change', function(e) { const params = new URLSearchParams(); if (ecosystem) params.set('ecosystem', ecosystem); if (sortBy) params.set('sort', sortBy); - window.location.href = '/ui/packages?' + params.toString(); + window.location.href = '/packages?' + params.toString(); }); {{end}} diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 67e91d0..34f6f08 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -25,8 +25,9 @@ const osWindows = "windows" // Blob implements Storage using gocloud.dev/blob. // Supports local filesystem (file://) and S3 (s3://) URLs. type Blob struct { - bucket *blob.Bucket - url string + bucket *blob.Bucket + backend Storage + url string } // OpenBucket opens a blob bucket from a URL. @@ -35,9 +36,20 @@ type Blob struct { // - file:///path/to/dir - Local filesystem storage // - s3://bucket-name - Amazon S3 (uses AWS_* environment variables) // - s3://bucket-name?region=us-east-1&endpoint=http://localhost:9000 - S3-compatible (MinIO, etc.) +// - gs://bucket-name - Google Cloud Storage (uses Application Default Credentials; +// supports Workload Identity on GKE/GCE without any extra configuration) +// - azblob://container-name - Azure Blob Storage // // For local filesystem, the directory is created if it doesn't exist. func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) { + if strings.HasPrefix(urlStr, "gs://") { + backend, err := OpenGCS(ctx, urlStr) + if err != nil { + return nil, err + } + return &Blob{backend: backend, url: urlStr}, nil + } + // Handle file:// URLs specially to create the directory if strings.HasPrefix(urlStr, "file://") { path := strings.TrimPrefix(urlStr, "file://") @@ -90,6 +102,10 @@ func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) { } func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + if b.backend != nil { + return b.backend.Store(ctx, path, r) + } + // Compute hash while writing h := sha256.New() tee := io.TeeReader(r, h) @@ -115,6 +131,10 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri } func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { + if b.backend != nil { + return b.backend.Open(ctx, path) + } + r, err := b.bucket.NewReader(ctx, path, nil) if err != nil { if isNotExist(err) { @@ -126,6 +146,10 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { } func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { + if b.backend != nil { + return b.backend.Exists(ctx, path) + } + exists, err := b.bucket.Exists(ctx, path) if err != nil { return false, fmt.Errorf("checking existence: %w", err) @@ -134,6 +158,10 @@ func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { } func (b *Blob) Delete(ctx context.Context, path string) error { + if b.backend != nil { + return b.backend.Delete(ctx, path) + } + err := b.bucket.Delete(ctx, path) if err != nil && !isNotExist(err) { return fmt.Errorf("deleting object: %w", err) @@ -142,6 +170,10 @@ func (b *Blob) Delete(ctx context.Context, path string) error { } func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { + if b.backend != nil { + return b.backend.SignedURL(ctx, path, expiry) + } + url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ Method: http.MethodGet, Expiry: expiry, @@ -156,6 +188,10 @@ func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) } func (b *Blob) Size(ctx context.Context, path string) (int64, error) { + if b.backend != nil { + return b.backend.Size(ctx, path) + } + attrs, err := b.bucket.Attributes(ctx, path) if err != nil { if isNotExist(err) { @@ -167,6 +203,10 @@ func (b *Blob) Size(ctx context.Context, path string) (int64, error) { } func (b *Blob) UsedSpace(ctx context.Context) (int64, error) { + if b.backend != nil { + return b.backend.UsedSpace(ctx) + } + var total int64 iter := b.bucket.List(nil) @@ -186,6 +226,16 @@ func (b *Blob) UsedSpace(ctx context.Context) (int64, error) { // ListPrefix returns object metadata for keys under a prefix. func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { + if b.backend != nil { + lister, ok := b.backend.(interface { + ListPrefix(context.Context, string) ([]ObjectInfo, error) + }) + if !ok { + return nil, ErrNotFound + } + return lister.ListPrefix(ctx, prefix) + } + iter := b.bucket.List(&blob.ListOptions{Prefix: prefix}) objects := make([]ObjectInfo, 0) @@ -214,10 +264,18 @@ func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, err } func (b *Blob) Close() error { + if b.backend != nil { + return b.backend.Close() + } + return b.bucket.Close() } func (b *Blob) URL() string { + if b.backend != nil { + return b.backend.URL() + } + return b.url } diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go new file mode 100644 index 0000000..e97e772 --- /dev/null +++ b/internal/storage/gcs.go @@ -0,0 +1,455 @@ +package storage + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "cloud.google.com/go/compute/metadata" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" +) + +const ( + gcsScope = "https://www.googleapis.com/auth/cloud-platform" + gcsDefaultHost = "https://storage.googleapis.com" +) + +// GCS implements Storage using the Cloud Storage JSON API. +type GCS struct { + bucket string + url string + client *http.Client + apiBase string + uploadBase string + + accessID string + privateKey []byte + signBytes func(context.Context, []byte) ([]byte, error) +} + +// OpenGCS opens a Google Cloud Storage bucket from a gs:// URL. +func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) { + u, err := url.Parse(urlStr) + if err != nil { + return nil, fmt.Errorf("parsing GCS URL: %w", err) + } + if u.Scheme != "gs" || u.Host == "" { + return nil, fmt.Errorf("invalid GCS URL %q", urlStr) + } + if u.Path != "" && u.Path != "/" { + return nil, fmt.Errorf("GCS URL must name a bucket, got path %q", u.Path) + } + + host := strings.TrimRight(gcsDefaultHost, "/") + client := http.DefaultClient + var accessID string + var privateKey []byte + + if emulator := os.Getenv("STORAGE_EMULATOR_HOST"); emulator != "" { + host = strings.TrimRight(emulator, "/") + if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { + host = "http://" + host + } + } else { + var credsJSON []byte + var err error + client, credsJSON, err = gcsHTTPClient(ctx) + if err != nil { + return nil, err + } + accessID, privateKey = readGCSCredentials(credsJSON) + if accessID == "" && metadata.OnGCE() { + accessID, _ = metadata.Email("") + } + } + + g := &GCS{ + bucket: u.Host, + url: urlStr, + client: client, + apiBase: host + "/storage/v1", + uploadBase: host + "/upload/storage/v1", + accessID: accessID, + privateKey: privateKey, + } + if len(privateKey) == 0 && accessID != "" { + g.signBytes = g.signBlob + } + return g, nil +} + +func gcsHTTPClient(ctx context.Context) (*http.Client, []byte, error) { + creds, err := google.FindDefaultCredentials(ctx, gcsScope) + if err != nil { + return nil, nil, fmt.Errorf("loading GCS default credentials: %w", err) + } + return oauth2.NewClient(ctx, creds.TokenSource), creds.JSON, nil +} + +func readGCSCredentials(credFileAsJSON []byte) (string, []byte) { + var serviceAccount struct { + ClientEmail string `json:"client_email"` + PrivateKey string `json:"private_key"` + } + if err := json.Unmarshal(credFileAsJSON, &serviceAccount); err == nil && serviceAccount.ClientEmail != "" { + return serviceAccount.ClientEmail, []byte(serviceAccount.PrivateKey) + } + + var impersonated struct { + ServiceAccountImpersonationURL string `json:"service_account_impersonation_url"` + } + if err := json.Unmarshal(credFileAsJSON, &impersonated); err == nil { + if email := serviceAccountFromImpersonationURL(impersonated.ServiceAccountImpersonationURL); email != "" { + return email, nil + } + } + + return "", nil +} + +func serviceAccountFromImpersonationURL(raw string) string { + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err != nil { + return "" + } + const marker = "/serviceAccounts/" + idx := strings.Index(u.Path, marker) + if idx == -1 { + return "" + } + email := strings.TrimSuffix(u.Path[idx+len(marker):], ":generateAccessToken") + email, _ = url.PathUnescape(email) + return email +} + +func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + h := sha256.New() + body := &countingReader{r: io.TeeReader(r, h)} + + endpoint := g.uploadBase + "/b/" + url.PathEscape(g.bucket) + "/o" + reqURL, _ := url.Parse(endpoint) + q := reqURL.Query() + q.Set("uploadType", "media") + q.Set("name", path) + reqURL.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), body) + if err != nil { + return 0, "", err + } + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := g.client.Do(req) + if err != nil { + return 0, "", fmt.Errorf("uploading GCS object: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if err := g.checkResponse(resp, http.StatusOK); err != nil { + return 0, "", fmt.Errorf("uploading GCS object: %w", err) + } + + hash := hex.EncodeToString(h.Sum(nil)) + return body.n, hash, nil +} + +func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.objectURL(path)+"?alt=media", nil) + if err != nil { + return nil, err + } + resp, err := g.client.Do(req) + if err != nil { + return nil, fmt.Errorf("opening GCS object: %w", err) + } + if resp.StatusCode == http.StatusNotFound { + _ = resp.Body.Close() + return nil, ErrNotFound + } + if err := g.checkResponse(resp, http.StatusOK); err != nil { + _ = resp.Body.Close() + return nil, fmt.Errorf("opening GCS object: %w", err) + } + return resp.Body, nil +} + +func (g *GCS) Exists(ctx context.Context, path string) (bool, error) { + _, err := g.attrs(ctx, path) + if errors.Is(err, ErrNotFound) { + return false, nil + } + return err == nil, err +} + +func (g *GCS) Delete(ctx context.Context, path string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, g.objectURL(path), nil) + if err != nil { + return err + } + resp, err := g.client.Do(req) + if err != nil { + return fmt.Errorf("deleting GCS object: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusNotFound { + return nil + } + if err := g.checkResponse(resp, http.StatusNoContent); err != nil { + return fmt.Errorf("deleting GCS object: %w", err) + } + return nil +} + +func (g *GCS) Size(ctx context.Context, path string) (int64, error) { + obj, err := g.attrs(ctx, path) + if err != nil { + return 0, err + } + return obj.size(), nil +} + +func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { + if g.accessID == "" { + return "", ErrSignedURLUnsupported + } + + expires := time.Now().Add(expiry) + u := &url.URL{Path: fmt.Sprintf("/%s/%s", g.bucket, path)} + stringToSign := fmt.Sprintf("GET\n\n\n%d\n%s", expires.Unix(), u.String()) + + signed, err := g.sign(ctx, []byte(stringToSign)) + if err != nil { + return "", fmt.Errorf("signing GCS URL: %w", err) + } + + u.Scheme = "https" + u.Host = "storage.googleapis.com" + q := u.Query() + q.Set("GoogleAccessId", g.accessID) + q.Set("Expires", strconv.FormatInt(expires.Unix(), 10)) + q.Set("Signature", base64.StdEncoding.EncodeToString(signed)) + u.RawQuery = q.Encode() + return u.String(), nil +} + +func (g *GCS) UsedSpace(ctx context.Context) (int64, error) { + objects, err := g.ListPrefix(ctx, "") + if err != nil { + return 0, err + } + var total int64 + for _, obj := range objects { + total += obj.Size + } + return total, nil +} + +func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { + var objects []ObjectInfo + pageToken := "" + + for { + reqURL, _ := url.Parse(g.apiBase + "/b/" + url.PathEscape(g.bucket) + "/o") + q := reqURL.Query() + q.Set("prefix", prefix) + if pageToken != "" { + q.Set("pageToken", pageToken) + } + reqURL.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL.String(), nil) + if err != nil { + return nil, err + } + resp, err := g.client.Do(req) + if err != nil { + return nil, fmt.Errorf("listing GCS objects: %w", err) + } + if err := g.checkResponse(resp, http.StatusOK); err != nil { + _ = resp.Body.Close() + return nil, fmt.Errorf("listing GCS objects: %w", err) + } + + var page gcsListResponse + if err := json.NewDecoder(resp.Body).Decode(&page); err != nil { + _ = resp.Body.Close() + return nil, fmt.Errorf("decoding GCS list response: %w", err) + } + _ = resp.Body.Close() + + for _, item := range page.Items { + objects = append(objects, ObjectInfo{ + Path: item.Name, + Size: item.size(), + ModTime: item.updated(), + }) + } + if page.NextPageToken == "" { + return objects, nil + } + pageToken = page.NextPageToken + } +} + +func (g *GCS) Close() error { + return nil +} + +func (g *GCS) URL() string { + return g.url +} + +func (g *GCS) attrs(ctx context.Context, path string) (*gcsObject, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.objectURL(path), nil) + if err != nil { + return nil, err + } + resp, err := g.client.Do(req) + if err != nil { + return nil, fmt.Errorf("getting GCS object attributes: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusNotFound { + return nil, ErrNotFound + } + if err := g.checkResponse(resp, http.StatusOK); err != nil { + return nil, fmt.Errorf("getting GCS object attributes: %w", err) + } + + var obj gcsObject + if err := json.NewDecoder(resp.Body).Decode(&obj); err != nil { + return nil, fmt.Errorf("decoding GCS attributes: %w", err) + } + return &obj, nil +} + +func (g *GCS) objectURL(path string) string { + return g.apiBase + "/b/" + url.PathEscape(g.bucket) + "/o/" + url.PathEscape(path) +} + +func (g *GCS) checkResponse(resp *http.Response, want int) error { + if resp.StatusCode == want { + return nil + } + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) +} + +func (g *GCS) sign(ctx context.Context, b []byte) ([]byte, error) { + if len(g.privateKey) > 0 { + key, err := parseGCSPrivateKey(g.privateKey) + if err != nil { + return nil, err + } + sum := sha256.Sum256(b) + return rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, sum[:]) + } + if g.signBytes != nil { + return g.signBytes(ctx, b) + } + return nil, ErrSignedURLUnsupported +} + +func (g *GCS) signBlob(ctx context.Context, payload []byte) ([]byte, error) { + reqBody, err := json.Marshal(map[string]string{ + "payload": base64.StdEncoding.EncodeToString(payload), + }) + if err != nil { + return nil, err + } + + endpoint := "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" + + url.PathEscape(g.accessID) + ":signBlob" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := g.client.Do(req) + if err != nil { + return nil, fmt.Errorf("calling IAM signBlob: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if err := g.checkResponse(resp, http.StatusOK); err != nil { + return nil, fmt.Errorf("calling IAM signBlob: %w", err) + } + + var out struct { + SignedBlob string `json:"signedBlob"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, fmt.Errorf("decoding IAM signBlob response: %w", err) + } + return base64.StdEncoding.DecodeString(out.SignedBlob) +} + +func parseGCSPrivateKey(key []byte) (*rsa.PrivateKey, error) { + if block, _ := pem.Decode(key); block != nil { + key = block.Bytes + } + parsedKey, err := x509.ParsePKCS8PrivateKey(key) + if err != nil { + parsedKey, err = x509.ParsePKCS1PrivateKey(key) + if err != nil { + return nil, err + } + } + parsed, ok := parsedKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("private key is not RSA") + } + return parsed, nil +} + +type gcsObject struct { + Name string `json:"name"` + Size string `json:"size"` + Updated string `json:"updated"` +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (r *countingReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + r.n += int64(n) + return n, err +} + +func (o gcsObject) size() int64 { + n, _ := strconv.ParseInt(o.Size, 10, 64) + return n +} + +func (o gcsObject) updated() time.Time { + t, _ := time.Parse(time.RFC3339Nano, o.Updated) + return t +} + +type gcsListResponse struct { + NextPageToken string `json:"nextPageToken"` + Items []gcsObject `json:"items"` +} diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go new file mode 100644 index 0000000..44dff00 --- /dev/null +++ b/internal/storage/gcs_test.go @@ -0,0 +1,145 @@ +package storage + +import ( + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "strconv" + "strings" + "testing" + "time" +) + +func TestGCSRoundTripWithEmulator(t *testing.T) { + objects := map[string]string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/upload/storage/v1/b/test-bucket/o": + name := r.URL.Query().Get("name") + data, _ := io.ReadAll(r.Body) + objects[name] = string(data) + writeJSON(w, gcsObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) + case r.Method == http.MethodGet && r.URL.Path == "/storage/v1/b/test-bucket/o": + prefix := r.URL.Query().Get("prefix") + page := gcsListResponse{} + for name, data := range objects { + if strings.HasPrefix(name, prefix) { + page.Items = append(page.Items, gcsObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) + } + } + sort.Slice(page.Items, func(i, j int) bool { return page.Items[i].Name < page.Items[j].Name }) + writeJSON(w, page) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"): + name := objectNameFromPath(r.URL.Path) + data, ok := objects[name] + if !ok { + http.NotFound(w, r) + return + } + if r.URL.Query().Get("alt") == "media" { + _, _ = io.WriteString(w, data) + return + } + writeJSON(w, gcsObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"): + delete(objects, objectNameFromPath(r.URL.Path)) + w.WriteHeader(http.StatusNoContent) + default: + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + t.Setenv("STORAGE_EMULATOR_HOST", server.URL) + + ctx := context.Background() + store, err := OpenGCS(ctx, "gs://test-bucket") + if err != nil { + t.Fatalf("OpenGCS failed: %v", err) + } + + size, hash, err := store.Store(ctx, "npm/pkg/file.tgz", strings.NewReader("content")) + if err != nil { + t.Fatalf("Store failed: %v", err) + } + if size != int64(len("content")) || hash == "" { + t.Fatalf("Store returned size=%d hash=%q", size, hash) + } + + exists, err := store.Exists(ctx, "npm/pkg/file.tgz") + if err != nil || !exists { + t.Fatalf("Exists = %v, %v; want true, nil", exists, err) + } + + r, err := store.Open(ctx, "npm/pkg/file.tgz") + if err != nil { + t.Fatalf("Open failed: %v", err) + } + data, _ := io.ReadAll(r) + _ = r.Close() + if string(data) != "content" { + t.Fatalf("Open content = %q, want content", data) + } + + list, err := store.ListPrefix(ctx, "npm/") + if err != nil { + t.Fatalf("ListPrefix failed: %v", err) + } + if len(list) != 1 || list[0].Path != "npm/pkg/file.tgz" { + t.Fatalf("ListPrefix = %#v", list) + } + + if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil { + t.Fatalf("Delete failed: %v", err) + } + exists, err = store.Exists(ctx, "npm/pkg/file.tgz") + if err != nil || exists { + t.Fatalf("Exists after delete = %v, %v; want false, nil", exists, err) + } +} + +func TestGCSSignedURLUsesSigner(t *testing.T) { + store := &GCS{ + bucket: "test-bucket", + accessID: "service@example.com", + signBytes: func(_ context.Context, b []byte) ([]byte, error) { + if !strings.Contains(string(b), "/test-bucket/npm/pkg/file.tgz") { + t.Fatalf("string to sign = %q", b) + } + return []byte("signed"), nil + }, + } + + got, err := store.SignedURL(context.Background(), "npm/pkg/file.tgz", time.Minute) + if err != nil { + t.Fatalf("SignedURL failed: %v", err) + } + u, err := url.Parse(got) + if err != nil { + t.Fatalf("parsing signed URL: %v", err) + } + if u.Scheme != "https" || u.Host != "storage.googleapis.com" || u.Path != "/test-bucket/npm/pkg/file.tgz" { + t.Fatalf("signed URL location = %s", got) + } + if u.Query().Get("GoogleAccessId") != "service@example.com" { + t.Fatalf("GoogleAccessId = %q", u.Query().Get("GoogleAccessId")) + } + if u.Query().Get("Signature") != base64.StdEncoding.EncodeToString([]byte("signed")) { + t.Fatalf("Signature = %q", u.Query().Get("Signature")) + } +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) +} + +func objectNameFromPath(p string) string { + escaped := strings.TrimPrefix(p, "/storage/v1/b/test-bucket/o/") + name, _ := url.PathUnescape(escaped) + return name +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index e11db53..4bf8f43 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -5,6 +5,9 @@ // - file:///path/to/dir - Local filesystem storage // - s3://bucket-name - Amazon S3 // - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO) +// - gs://bucket-name - Google Cloud Storage (supports GKE Workload Identity +// via Application Default Credentials) +// - azblob://container-name - Azure Blob Storage // // Use OpenBucket to create a storage backend from a URL. package storage