mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-07-06 22:33:19 -04:00
Compare commits
2 commits
main
...
andrew/lig
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
73ba229187 |
||
|
|
f3897749df |
9 changed files with 751 additions and 11 deletions
53
README.md
53
README.md
|
|
@ -409,7 +409,7 @@ The proxy can be configured via:
|
||||||
-config string Path to configuration file
|
-config string Path to configuration file
|
||||||
-listen string Address to listen on (default ":8080")
|
-listen string Address to listen on (default ":8080")
|
||||||
-base-url string Public URL of this proxy (default "http://localhost:8080")
|
-base-url string Public URL of this proxy (default "http://localhost:8080")
|
||||||
-storage-url string Storage URL (file:// or s3://)
|
-storage-url string Storage URL (file://, s3://, gs://, azblob://)
|
||||||
-storage-path string Path to artifact storage directory (deprecated, use -storage-url)
|
-storage-path string Path to artifact storage directory (deprecated, use -storage-url)
|
||||||
-database-driver string Database driver: sqlite or postgres (default "sqlite")
|
-database-driver string Database driver: sqlite or postgres (default "sqlite")
|
||||||
-database-path string Path to SQLite database file (default "./cache/proxy.db")
|
-database-path string Path to SQLite database file (default "./cache/proxy.db")
|
||||||
|
|
@ -504,6 +504,57 @@ storage:
|
||||||
|
|
||||||
Set credentials via standard AWS environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`).
|
Set credentials via standard AWS environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`).
|
||||||
|
|
||||||
|
### Google Cloud Storage
|
||||||
|
|
||||||
|
The proxy can store cached artifacts in a GCS bucket using the `gs://` URL scheme.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
storage:
|
||||||
|
url: "gs://my-bucket-name"
|
||||||
|
```
|
||||||
|
|
||||||
|
Authentication uses [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials), which means no credentials need to be embedded in the config or environment. Supported sources, in order:
|
||||||
|
|
||||||
|
- **GKE Workload Identity** — bind the Kubernetes service account running the proxy to a Google service account that has `roles/storage.objectAdmin` on the bucket. The proxy will use the workload's token automatically.
|
||||||
|
- **Attached service account** on GCE, Cloud Run, Cloud Functions, etc.
|
||||||
|
- **`GOOGLE_APPLICATION_CREDENTIALS`** environment variable pointing at a service account JSON key file.
|
||||||
|
- **`gcloud auth application-default login`** for local development.
|
||||||
|
|
||||||
|
#### GKE Workload Identity setup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Create a Google service account
|
||||||
|
gcloud iam service-accounts create git-pkgs-proxy \
|
||||||
|
--project=PROJECT_ID
|
||||||
|
|
||||||
|
# 2. Grant it access to the bucket
|
||||||
|
gsutil iam ch \
|
||||||
|
serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com:objectAdmin \
|
||||||
|
gs://my-bucket-name
|
||||||
|
|
||||||
|
# 3. Bind the Kubernetes service account to it
|
||||||
|
gcloud iam service-accounts add-iam-policy-binding \
|
||||||
|
git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \
|
||||||
|
--role=roles/iam.workloadIdentityUser \
|
||||||
|
--member="serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/KSA_NAME]"
|
||||||
|
|
||||||
|
# 4. Annotate the Kubernetes service account
|
||||||
|
kubectl annotate serviceaccount KSA_NAME \
|
||||||
|
--namespace=NAMESPACE \
|
||||||
|
iam.gke.io/gcp-service-account=git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Direct serve (signed URLs) with Workload Identity
|
||||||
|
|
||||||
|
When `direct_serve: true` is enabled, the proxy issues HTTP 302 redirects to presigned GCS URLs. Because Workload Identity provides no private key, the gcsblob driver falls back to the [IAM Credentials `signBlob` API](https://cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob). For this to work, grant the service account the token-creator role on itself:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
gcloud iam service-accounts add-iam-policy-binding \
|
||||||
|
git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com \
|
||||||
|
--role=roles/iam.serviceAccountTokenCreator \
|
||||||
|
--member="serviceAccount:git-pkgs-proxy@PROJECT_ID.iam.gserviceaccount.com"
|
||||||
|
```
|
||||||
|
|
||||||
## CLI Commands
|
## CLI Commands
|
||||||
|
|
||||||
### serve (default)
|
### serve (default)
|
||||||
|
|
|
||||||
|
|
@ -22,9 +22,20 @@ storage:
|
||||||
# - file:///path/to/dir - Local filesystem (default)
|
# - file:///path/to/dir - Local filesystem (default)
|
||||||
# - s3://bucket-name - Amazon S3
|
# - s3://bucket-name - Amazon S3
|
||||||
# - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
|
# - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
|
||||||
|
# - gs://bucket-name - Google Cloud Storage
|
||||||
|
# - azblob://container-name - Azure Blob Storage
|
||||||
#
|
#
|
||||||
# For S3, configure credentials via environment variables:
|
# For S3, configure credentials via environment variables:
|
||||||
# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
|
# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
|
||||||
|
#
|
||||||
|
# For GCS, authentication uses Application Default Credentials. On GKE with
|
||||||
|
# Workload Identity, bind the Kubernetes service account to a Google service
|
||||||
|
# account that has roles/storage.objectAdmin on the bucket. No extra config
|
||||||
|
# is needed in this file. For local development, run:
|
||||||
|
# gcloud auth application-default login
|
||||||
|
# If direct_serve is enabled, the service account also needs
|
||||||
|
# roles/iam.serviceAccountTokenCreator on itself so the IAM Credentials
|
||||||
|
# signBlob API can sign URLs without a private key.
|
||||||
url: ""
|
url: ""
|
||||||
|
|
||||||
# Local filesystem path (used when url is empty)
|
# Local filesystem path (used when url is empty)
|
||||||
|
|
@ -37,7 +48,7 @@ storage:
|
||||||
max_size: ""
|
max_size: ""
|
||||||
|
|
||||||
# Redirect cached artifact downloads to presigned storage URLs (HTTP 302)
|
# Redirect cached artifact downloads to presigned storage URLs (HTTP 302)
|
||||||
# instead of streaming through the proxy. Only effective for S3 and Azure.
|
# instead of streaming through the proxy. Only effective for S3, GCS, and Azure.
|
||||||
# Leave disabled if clients reach the proxy through an authenticating gateway,
|
# Leave disabled if clients reach the proxy through an authenticating gateway,
|
||||||
# since presigned URLs bypass it.
|
# since presigned URLs bypass it.
|
||||||
direct_serve: false
|
direct_serve: false
|
||||||
|
|
|
||||||
8
go.mod
8
go.mod
|
|
@ -3,6 +3,7 @@ module github.com/git-pkgs/proxy
|
||||||
go 1.25.6
|
go 1.25.6
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
cloud.google.com/go/compute/metadata v0.9.0
|
||||||
github.com/BurntSushi/toml v1.6.0
|
github.com/BurntSushi/toml v1.6.0
|
||||||
github.com/CycloneDX/cyclonedx-go v0.11.0
|
github.com/CycloneDX/cyclonedx-go v0.11.0
|
||||||
github.com/git-pkgs/archives v0.3.0
|
github.com/git-pkgs/archives v0.3.0
|
||||||
|
|
@ -21,6 +22,7 @@ require (
|
||||||
github.com/spdx/tools-golang v0.5.7
|
github.com/spdx/tools-golang v0.5.7
|
||||||
github.com/swaggo/swag v1.16.6
|
github.com/swaggo/swag v1.16.6
|
||||||
gocloud.dev v0.46.0
|
gocloud.dev v0.46.0
|
||||||
|
golang.org/x/oauth2 v0.36.0
|
||||||
golang.org/x/sync v0.20.0
|
golang.org/x/sync v0.20.0
|
||||||
google.golang.org/protobuf v1.36.11
|
google.golang.org/protobuf v1.36.11
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
|
|
@ -32,7 +34,6 @@ require (
|
||||||
4d63.com/gochecknoglobals v0.2.2 // indirect
|
4d63.com/gochecknoglobals v0.2.2 // indirect
|
||||||
cloud.google.com/go/auth v0.18.2 // indirect
|
cloud.google.com/go/auth v0.18.2 // indirect
|
||||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
|
||||||
codeberg.org/chavacava/garif v0.2.0 // indirect
|
codeberg.org/chavacava/garif v0.2.0 // indirect
|
||||||
codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect
|
codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect
|
||||||
dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect
|
dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect
|
||||||
|
|
@ -111,7 +112,7 @@ require (
|
||||||
github.com/curioswitch/go-reassign v0.3.0 // indirect
|
github.com/curioswitch/go-reassign v0.3.0 // indirect
|
||||||
github.com/daixiang0/gci v0.13.7 // indirect
|
github.com/daixiang0/gci v0.13.7 // indirect
|
||||||
github.com/dave/dst v0.27.3 // indirect
|
github.com/dave/dst v0.27.3 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/denis-tingaikin/go-header v0.5.0 // indirect
|
github.com/denis-tingaikin/go-header v0.5.0 // indirect
|
||||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
github.com/dlclark/regexp2 v1.11.5 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
|
|
@ -223,7 +224,7 @@ require (
|
||||||
github.com/pelletier/go-toml v1.9.5 // indirect
|
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
github.com/prometheus/common v0.67.5 // indirect
|
github.com/prometheus/common v0.67.5 // indirect
|
||||||
github.com/prometheus/procfs v0.20.1 // indirect
|
github.com/prometheus/procfs v0.20.1 // indirect
|
||||||
github.com/quasilyte/go-ruleguard v0.4.5 // indirect
|
github.com/quasilyte/go-ruleguard v0.4.5 // indirect
|
||||||
|
|
@ -296,7 +297,6 @@ require (
|
||||||
golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect
|
golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect
|
||||||
golang.org/x/mod v0.33.0 // indirect
|
golang.org/x/mod v0.33.0 // indirect
|
||||||
golang.org/x/net v0.52.0 // indirect
|
golang.org/x/net v0.52.0 // indirect
|
||||||
golang.org/x/oauth2 v0.36.0 // indirect
|
|
||||||
golang.org/x/sys v0.45.0 // indirect
|
golang.org/x/sys v0.45.0 // indirect
|
||||||
golang.org/x/text v0.35.0 // indirect
|
golang.org/x/text v0.35.0 // indirect
|
||||||
golang.org/x/tools v0.42.0 // indirect
|
golang.org/x/tools v0.42.0 // indirect
|
||||||
|
|
|
||||||
6
go.sum
6
go.sum
|
|
@ -209,8 +209,9 @@ github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEy
|
||||||
github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo=
|
github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo=
|
||||||
github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc=
|
github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8=
|
github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8=
|
||||||
github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY=
|
github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY=
|
||||||
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
|
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
|
||||||
|
|
@ -537,8 +538,9 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd
|
||||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
||||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||||
|
|
|
||||||
|
|
@ -24,10 +24,23 @@
|
||||||
// storage:
|
// storage:
|
||||||
// url: "s3://bucket?endpoint=http://localhost:9000"
|
// url: "s3://bucket?endpoint=http://localhost:9000"
|
||||||
//
|
//
|
||||||
|
// Google Cloud Storage:
|
||||||
|
//
|
||||||
|
// storage:
|
||||||
|
// url: "gs://bucket-name"
|
||||||
|
//
|
||||||
// For S3, configure credentials via AWS environment variables:
|
// For S3, configure credentials via AWS environment variables:
|
||||||
//
|
//
|
||||||
// AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
|
// AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
|
||||||
//
|
//
|
||||||
|
// For GCS, authentication uses Application Default Credentials. This works
|
||||||
|
// transparently on GKE with Workload Identity, GCE/Cloud Run with attached
|
||||||
|
// service accounts, or locally via `gcloud auth application-default login`.
|
||||||
|
// When the proxy is signing URLs (direct_serve) without a private key,
|
||||||
|
// gcsblob automatically falls back to the IAM Credentials signBlob API,
|
||||||
|
// which requires the service account to hold roles/iam.serviceAccountTokenCreator
|
||||||
|
// on itself.
|
||||||
|
//
|
||||||
// Database Configuration:
|
// Database Configuration:
|
||||||
//
|
//
|
||||||
// The proxy supports two database backends:
|
// The proxy supports two database backends:
|
||||||
|
|
@ -141,6 +154,8 @@ type StorageConfig struct {
|
||||||
// - file:///path/to/dir - Local filesystem (default)
|
// - file:///path/to/dir - Local filesystem (default)
|
||||||
// - s3://bucket-name - Amazon S3
|
// - s3://bucket-name - Amazon S3
|
||||||
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
|
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
|
||||||
|
// - gs://bucket-name - Google Cloud Storage (Workload Identity supported)
|
||||||
|
// - azblob://container-name - Azure Blob Storage
|
||||||
// If empty, defaults to file:// with the Path value.
|
// If empty, defaults to file:// with the Path value.
|
||||||
URL string `json:"url" yaml:"url"`
|
URL string `json:"url" yaml:"url"`
|
||||||
|
|
||||||
|
|
@ -157,7 +172,7 @@ type StorageConfig struct {
|
||||||
|
|
||||||
// DirectServe enables redirecting cached artifact downloads to presigned
|
// DirectServe enables redirecting cached artifact downloads to presigned
|
||||||
// storage URLs (HTTP 302) instead of streaming bytes through the proxy.
|
// storage URLs (HTTP 302) instead of streaming bytes through the proxy.
|
||||||
// Only effective for backends that support URL signing (S3, Azure).
|
// Only effective for backends that support URL signing (S3, GCS, Azure).
|
||||||
DirectServe bool `json:"direct_serve" yaml:"direct_serve"`
|
DirectServe bool `json:"direct_serve" yaml:"direct_serve"`
|
||||||
|
|
||||||
// DirectServeTTL is how long presigned URLs remain valid.
|
// DirectServeTTL is how long presigned URLs remain valid.
|
||||||
|
|
|
||||||
|
|
@ -25,8 +25,9 @@ const osWindows = "windows"
|
||||||
// Blob implements Storage using gocloud.dev/blob.
|
// Blob implements Storage using gocloud.dev/blob.
|
||||||
// Supports local filesystem (file://) and S3 (s3://) URLs.
|
// Supports local filesystem (file://) and S3 (s3://) URLs.
|
||||||
type Blob struct {
|
type Blob struct {
|
||||||
bucket *blob.Bucket
|
bucket *blob.Bucket
|
||||||
url string
|
backend Storage
|
||||||
|
url string
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenBucket opens a blob bucket from a URL.
|
// OpenBucket opens a blob bucket from a URL.
|
||||||
|
|
@ -35,9 +36,20 @@ type Blob struct {
|
||||||
// - file:///path/to/dir - Local filesystem storage
|
// - file:///path/to/dir - Local filesystem storage
|
||||||
// - s3://bucket-name - Amazon S3 (uses AWS_* environment variables)
|
// - s3://bucket-name - Amazon S3 (uses AWS_* environment variables)
|
||||||
// - s3://bucket-name?region=us-east-1&endpoint=http://localhost:9000 - S3-compatible (MinIO, etc.)
|
// - s3://bucket-name?region=us-east-1&endpoint=http://localhost:9000 - S3-compatible (MinIO, etc.)
|
||||||
|
// - gs://bucket-name - Google Cloud Storage (uses Application Default Credentials;
|
||||||
|
// supports Workload Identity on GKE/GCE without any extra configuration)
|
||||||
|
// - azblob://container-name - Azure Blob Storage
|
||||||
//
|
//
|
||||||
// For local filesystem, the directory is created if it doesn't exist.
|
// For local filesystem, the directory is created if it doesn't exist.
|
||||||
func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) {
|
func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) {
|
||||||
|
if strings.HasPrefix(urlStr, "gs://") {
|
||||||
|
backend, err := OpenGCS(ctx, urlStr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &Blob{backend: backend, url: urlStr}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Handle file:// URLs specially to create the directory
|
// Handle file:// URLs specially to create the directory
|
||||||
if strings.HasPrefix(urlStr, "file://") {
|
if strings.HasPrefix(urlStr, "file://") {
|
||||||
path := strings.TrimPrefix(urlStr, "file://")
|
path := strings.TrimPrefix(urlStr, "file://")
|
||||||
|
|
@ -90,6 +102,10 @@ func OpenBucket(ctx context.Context, urlStr string) (*Blob, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
|
func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.Store(ctx, path, r)
|
||||||
|
}
|
||||||
|
|
||||||
// Compute hash while writing
|
// Compute hash while writing
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
tee := io.TeeReader(r, h)
|
tee := io.TeeReader(r, h)
|
||||||
|
|
@ -115,6 +131,10 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) {
|
func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.Open(ctx, path)
|
||||||
|
}
|
||||||
|
|
||||||
r, err := b.bucket.NewReader(ctx, path, nil)
|
r, err := b.bucket.NewReader(ctx, path, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotExist(err) {
|
if isNotExist(err) {
|
||||||
|
|
@ -126,6 +146,10 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) Exists(ctx context.Context, path string) (bool, error) {
|
func (b *Blob) Exists(ctx context.Context, path string) (bool, error) {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.Exists(ctx, path)
|
||||||
|
}
|
||||||
|
|
||||||
exists, err := b.bucket.Exists(ctx, path)
|
exists, err := b.bucket.Exists(ctx, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("checking existence: %w", err)
|
return false, fmt.Errorf("checking existence: %w", err)
|
||||||
|
|
@ -134,6 +158,10 @@ func (b *Blob) Exists(ctx context.Context, path string) (bool, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) Delete(ctx context.Context, path string) error {
|
func (b *Blob) Delete(ctx context.Context, path string) error {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.Delete(ctx, path)
|
||||||
|
}
|
||||||
|
|
||||||
err := b.bucket.Delete(ctx, path)
|
err := b.bucket.Delete(ctx, path)
|
||||||
if err != nil && !isNotExist(err) {
|
if err != nil && !isNotExist(err) {
|
||||||
return fmt.Errorf("deleting object: %w", err)
|
return fmt.Errorf("deleting object: %w", err)
|
||||||
|
|
@ -142,6 +170,10 @@ func (b *Blob) Delete(ctx context.Context, path string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
|
func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.SignedURL(ctx, path, expiry)
|
||||||
|
}
|
||||||
|
|
||||||
url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
|
url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
|
||||||
Method: http.MethodGet,
|
Method: http.MethodGet,
|
||||||
Expiry: expiry,
|
Expiry: expiry,
|
||||||
|
|
@ -156,6 +188,10 @@ func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) Size(ctx context.Context, path string) (int64, error) {
|
func (b *Blob) Size(ctx context.Context, path string) (int64, error) {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.Size(ctx, path)
|
||||||
|
}
|
||||||
|
|
||||||
attrs, err := b.bucket.Attributes(ctx, path)
|
attrs, err := b.bucket.Attributes(ctx, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isNotExist(err) {
|
if isNotExist(err) {
|
||||||
|
|
@ -167,6 +203,10 @@ func (b *Blob) Size(ctx context.Context, path string) (int64, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) UsedSpace(ctx context.Context) (int64, error) {
|
func (b *Blob) UsedSpace(ctx context.Context) (int64, error) {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.UsedSpace(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
var total int64
|
var total int64
|
||||||
|
|
||||||
iter := b.bucket.List(nil)
|
iter := b.bucket.List(nil)
|
||||||
|
|
@ -186,6 +226,16 @@ func (b *Blob) UsedSpace(ctx context.Context) (int64, error) {
|
||||||
|
|
||||||
// ListPrefix returns object metadata for keys under a prefix.
|
// ListPrefix returns object metadata for keys under a prefix.
|
||||||
func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
|
if b.backend != nil {
|
||||||
|
lister, ok := b.backend.(interface {
|
||||||
|
ListPrefix(context.Context, string) ([]ObjectInfo, error)
|
||||||
|
})
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
return lister.ListPrefix(ctx, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
iter := b.bucket.List(&blob.ListOptions{Prefix: prefix})
|
iter := b.bucket.List(&blob.ListOptions{Prefix: prefix})
|
||||||
objects := make([]ObjectInfo, 0)
|
objects := make([]ObjectInfo, 0)
|
||||||
|
|
||||||
|
|
@ -214,10 +264,18 @@ func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) Close() error {
|
func (b *Blob) Close() error {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.Close()
|
||||||
|
}
|
||||||
|
|
||||||
return b.bucket.Close()
|
return b.bucket.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Blob) URL() string {
|
func (b *Blob) URL() string {
|
||||||
|
if b.backend != nil {
|
||||||
|
return b.backend.URL()
|
||||||
|
}
|
||||||
|
|
||||||
return b.url
|
return b.url
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
455
internal/storage/gcs.go
Normal file
455
internal/storage/gcs.go
Normal file
|
|
@ -0,0 +1,455 @@
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/rsa"
|
||||||
|
"crypto/sha256"
|
||||||
|
"crypto/x509"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"encoding/pem"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"cloud.google.com/go/compute/metadata"
|
||||||
|
"golang.org/x/oauth2"
|
||||||
|
"golang.org/x/oauth2/google"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
gcsScope = "https://www.googleapis.com/auth/cloud-platform"
|
||||||
|
gcsDefaultHost = "https://storage.googleapis.com"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GCS implements Storage using the Cloud Storage JSON API.
|
||||||
|
type GCS struct {
|
||||||
|
bucket string
|
||||||
|
url string
|
||||||
|
client *http.Client
|
||||||
|
apiBase string
|
||||||
|
uploadBase string
|
||||||
|
|
||||||
|
accessID string
|
||||||
|
privateKey []byte
|
||||||
|
signBytes func(context.Context, []byte) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenGCS opens a Google Cloud Storage bucket from a gs:// URL.
|
||||||
|
func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) {
|
||||||
|
u, err := url.Parse(urlStr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing GCS URL: %w", err)
|
||||||
|
}
|
||||||
|
if u.Scheme != "gs" || u.Host == "" {
|
||||||
|
return nil, fmt.Errorf("invalid GCS URL %q", urlStr)
|
||||||
|
}
|
||||||
|
if u.Path != "" && u.Path != "/" {
|
||||||
|
return nil, fmt.Errorf("GCS URL must name a bucket, got path %q", u.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
host := strings.TrimRight(gcsDefaultHost, "/")
|
||||||
|
client := http.DefaultClient
|
||||||
|
var accessID string
|
||||||
|
var privateKey []byte
|
||||||
|
|
||||||
|
if emulator := os.Getenv("STORAGE_EMULATOR_HOST"); emulator != "" {
|
||||||
|
host = strings.TrimRight(emulator, "/")
|
||||||
|
if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
|
||||||
|
host = "http://" + host
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var credsJSON []byte
|
||||||
|
var err error
|
||||||
|
client, credsJSON, err = gcsHTTPClient(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
accessID, privateKey = readGCSCredentials(credsJSON)
|
||||||
|
if accessID == "" && metadata.OnGCE() {
|
||||||
|
accessID, _ = metadata.Email("")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
g := &GCS{
|
||||||
|
bucket: u.Host,
|
||||||
|
url: urlStr,
|
||||||
|
client: client,
|
||||||
|
apiBase: host + "/storage/v1",
|
||||||
|
uploadBase: host + "/upload/storage/v1",
|
||||||
|
accessID: accessID,
|
||||||
|
privateKey: privateKey,
|
||||||
|
}
|
||||||
|
if len(privateKey) == 0 && accessID != "" {
|
||||||
|
g.signBytes = g.signBlob
|
||||||
|
}
|
||||||
|
return g, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func gcsHTTPClient(ctx context.Context) (*http.Client, []byte, error) {
|
||||||
|
creds, err := google.FindDefaultCredentials(ctx, gcsScope)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("loading GCS default credentials: %w", err)
|
||||||
|
}
|
||||||
|
return oauth2.NewClient(ctx, creds.TokenSource), creds.JSON, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readGCSCredentials(credFileAsJSON []byte) (string, []byte) {
|
||||||
|
var serviceAccount struct {
|
||||||
|
ClientEmail string `json:"client_email"`
|
||||||
|
PrivateKey string `json:"private_key"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(credFileAsJSON, &serviceAccount); err == nil && serviceAccount.ClientEmail != "" {
|
||||||
|
return serviceAccount.ClientEmail, []byte(serviceAccount.PrivateKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
var impersonated struct {
|
||||||
|
ServiceAccountImpersonationURL string `json:"service_account_impersonation_url"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(credFileAsJSON, &impersonated); err == nil {
|
||||||
|
if email := serviceAccountFromImpersonationURL(impersonated.ServiceAccountImpersonationURL); email != "" {
|
||||||
|
return email, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func serviceAccountFromImpersonationURL(raw string) string {
|
||||||
|
if raw == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
const marker = "/serviceAccounts/"
|
||||||
|
idx := strings.Index(u.Path, marker)
|
||||||
|
if idx == -1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
email := strings.TrimSuffix(u.Path[idx+len(marker):], ":generateAccessToken")
|
||||||
|
email, _ = url.PathUnescape(email)
|
||||||
|
return email
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
|
||||||
|
h := sha256.New()
|
||||||
|
body := &countingReader{r: io.TeeReader(r, h)}
|
||||||
|
|
||||||
|
endpoint := g.uploadBase + "/b/" + url.PathEscape(g.bucket) + "/o"
|
||||||
|
reqURL, _ := url.Parse(endpoint)
|
||||||
|
q := reqURL.Query()
|
||||||
|
q.Set("uploadType", "media")
|
||||||
|
q.Set("name", path)
|
||||||
|
reqURL.RawQuery = q.Encode()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), body)
|
||||||
|
if err != nil {
|
||||||
|
return 0, "", err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/octet-stream")
|
||||||
|
|
||||||
|
resp, err := g.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return 0, "", fmt.Errorf("uploading GCS object: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if err := g.checkResponse(resp, http.StatusOK); err != nil {
|
||||||
|
return 0, "", fmt.Errorf("uploading GCS object: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hash := hex.EncodeToString(h.Sum(nil))
|
||||||
|
return body.n, hash, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.objectURL(path)+"?alt=media", nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := g.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("opening GCS object: %w", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
if err := g.checkResponse(resp, http.StatusOK); err != nil {
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return nil, fmt.Errorf("opening GCS object: %w", err)
|
||||||
|
}
|
||||||
|
return resp.Body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) Exists(ctx context.Context, path string) (bool, error) {
|
||||||
|
_, err := g.attrs(ctx, path)
|
||||||
|
if errors.Is(err, ErrNotFound) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) Delete(ctx context.Context, path string) error {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, g.objectURL(path), nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp, err := g.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("deleting GCS object: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := g.checkResponse(resp, http.StatusNoContent); err != nil {
|
||||||
|
return fmt.Errorf("deleting GCS object: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) Size(ctx context.Context, path string) (int64, error) {
|
||||||
|
obj, err := g.attrs(ctx, path)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return obj.size(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
|
||||||
|
if g.accessID == "" {
|
||||||
|
return "", ErrSignedURLUnsupported
|
||||||
|
}
|
||||||
|
|
||||||
|
expires := time.Now().Add(expiry)
|
||||||
|
u := &url.URL{Path: fmt.Sprintf("/%s/%s", g.bucket, path)}
|
||||||
|
stringToSign := fmt.Sprintf("GET\n\n\n%d\n%s", expires.Unix(), u.String())
|
||||||
|
|
||||||
|
signed, err := g.sign(ctx, []byte(stringToSign))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("signing GCS URL: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
u.Scheme = "https"
|
||||||
|
u.Host = "storage.googleapis.com"
|
||||||
|
q := u.Query()
|
||||||
|
q.Set("GoogleAccessId", g.accessID)
|
||||||
|
q.Set("Expires", strconv.FormatInt(expires.Unix(), 10))
|
||||||
|
q.Set("Signature", base64.StdEncoding.EncodeToString(signed))
|
||||||
|
u.RawQuery = q.Encode()
|
||||||
|
return u.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) UsedSpace(ctx context.Context) (int64, error) {
|
||||||
|
objects, err := g.ListPrefix(ctx, "")
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
var total int64
|
||||||
|
for _, obj := range objects {
|
||||||
|
total += obj.Size
|
||||||
|
}
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
|
var objects []ObjectInfo
|
||||||
|
pageToken := ""
|
||||||
|
|
||||||
|
for {
|
||||||
|
reqURL, _ := url.Parse(g.apiBase + "/b/" + url.PathEscape(g.bucket) + "/o")
|
||||||
|
q := reqURL.Query()
|
||||||
|
q.Set("prefix", prefix)
|
||||||
|
if pageToken != "" {
|
||||||
|
q.Set("pageToken", pageToken)
|
||||||
|
}
|
||||||
|
reqURL.RawQuery = q.Encode()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL.String(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := g.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listing GCS objects: %w", err)
|
||||||
|
}
|
||||||
|
if err := g.checkResponse(resp, http.StatusOK); err != nil {
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return nil, fmt.Errorf("listing GCS objects: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var page gcsListResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&page); err != nil {
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return nil, fmt.Errorf("decoding GCS list response: %w", err)
|
||||||
|
}
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
for _, item := range page.Items {
|
||||||
|
objects = append(objects, ObjectInfo{
|
||||||
|
Path: item.Name,
|
||||||
|
Size: item.size(),
|
||||||
|
ModTime: item.updated(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if page.NextPageToken == "" {
|
||||||
|
return objects, nil
|
||||||
|
}
|
||||||
|
pageToken = page.NextPageToken
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) URL() string {
|
||||||
|
return g.url
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) attrs(ctx context.Context, path string) (*gcsObject, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.objectURL(path), nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := g.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("getting GCS object attributes: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
if err := g.checkResponse(resp, http.StatusOK); err != nil {
|
||||||
|
return nil, fmt.Errorf("getting GCS object attributes: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var obj gcsObject
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&obj); err != nil {
|
||||||
|
return nil, fmt.Errorf("decoding GCS attributes: %w", err)
|
||||||
|
}
|
||||||
|
return &obj, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) objectURL(path string) string {
|
||||||
|
return g.apiBase + "/b/" + url.PathEscape(g.bucket) + "/o/" + url.PathEscape(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) checkResponse(resp *http.Response, want int) error {
|
||||||
|
if resp.StatusCode == want {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
|
return fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) sign(ctx context.Context, b []byte) ([]byte, error) {
|
||||||
|
if len(g.privateKey) > 0 {
|
||||||
|
key, err := parseGCSPrivateKey(g.privateKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(b)
|
||||||
|
return rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, sum[:])
|
||||||
|
}
|
||||||
|
if g.signBytes != nil {
|
||||||
|
return g.signBytes(ctx, b)
|
||||||
|
}
|
||||||
|
return nil, ErrSignedURLUnsupported
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GCS) signBlob(ctx context.Context, payload []byte) ([]byte, error) {
|
||||||
|
reqBody, err := json.Marshal(map[string]string{
|
||||||
|
"payload": base64.StdEncoding.EncodeToString(payload),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" +
|
||||||
|
url.PathEscape(g.accessID) + ":signBlob"
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(reqBody))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := g.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("calling IAM signBlob: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if err := g.checkResponse(resp, http.StatusOK); err != nil {
|
||||||
|
return nil, fmt.Errorf("calling IAM signBlob: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out struct {
|
||||||
|
SignedBlob string `json:"signedBlob"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||||
|
return nil, fmt.Errorf("decoding IAM signBlob response: %w", err)
|
||||||
|
}
|
||||||
|
return base64.StdEncoding.DecodeString(out.SignedBlob)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseGCSPrivateKey(key []byte) (*rsa.PrivateKey, error) {
|
||||||
|
if block, _ := pem.Decode(key); block != nil {
|
||||||
|
key = block.Bytes
|
||||||
|
}
|
||||||
|
parsedKey, err := x509.ParsePKCS8PrivateKey(key)
|
||||||
|
if err != nil {
|
||||||
|
parsedKey, err = x509.ParsePKCS1PrivateKey(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parsed, ok := parsedKey.(*rsa.PrivateKey)
|
||||||
|
if !ok {
|
||||||
|
return nil, errors.New("private key is not RSA")
|
||||||
|
}
|
||||||
|
return parsed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type gcsObject struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Size string `json:"size"`
|
||||||
|
Updated string `json:"updated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type countingReader struct {
|
||||||
|
r io.Reader
|
||||||
|
n int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *countingReader) Read(p []byte) (int, error) {
|
||||||
|
n, err := r.r.Read(p)
|
||||||
|
r.n += int64(n)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o gcsObject) size() int64 {
|
||||||
|
n, _ := strconv.ParseInt(o.Size, 10, 64)
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o gcsObject) updated() time.Time {
|
||||||
|
t, _ := time.Parse(time.RFC3339Nano, o.Updated)
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
type gcsListResponse struct {
|
||||||
|
NextPageToken string `json:"nextPageToken"`
|
||||||
|
Items []gcsObject `json:"items"`
|
||||||
|
}
|
||||||
145
internal/storage/gcs_test.go
Normal file
145
internal/storage/gcs_test.go
Normal file
|
|
@ -0,0 +1,145 @@
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGCSRoundTripWithEmulator(t *testing.T) {
|
||||||
|
objects := map[string]string{}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case r.Method == http.MethodPost && r.URL.Path == "/upload/storage/v1/b/test-bucket/o":
|
||||||
|
name := r.URL.Query().Get("name")
|
||||||
|
data, _ := io.ReadAll(r.Body)
|
||||||
|
objects[name] = string(data)
|
||||||
|
writeJSON(w, gcsObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
case r.Method == http.MethodGet && r.URL.Path == "/storage/v1/b/test-bucket/o":
|
||||||
|
prefix := r.URL.Query().Get("prefix")
|
||||||
|
page := gcsListResponse{}
|
||||||
|
for name, data := range objects {
|
||||||
|
if strings.HasPrefix(name, prefix) {
|
||||||
|
page.Items = append(page.Items, gcsObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(page.Items, func(i, j int) bool { return page.Items[i].Name < page.Items[j].Name })
|
||||||
|
writeJSON(w, page)
|
||||||
|
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"):
|
||||||
|
name := objectNameFromPath(r.URL.Path)
|
||||||
|
data, ok := objects[name]
|
||||||
|
if !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.URL.Query().Get("alt") == "media" {
|
||||||
|
_, _ = io.WriteString(w, data)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, gcsObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"):
|
||||||
|
delete(objects, objectNameFromPath(r.URL.Path))
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected request: %s %s", r.Method, r.URL.String())
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
t.Setenv("STORAGE_EMULATOR_HOST", server.URL)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
store, err := OpenGCS(ctx, "gs://test-bucket")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OpenGCS failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
size, hash, err := store.Store(ctx, "npm/pkg/file.tgz", strings.NewReader("content"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Store failed: %v", err)
|
||||||
|
}
|
||||||
|
if size != int64(len("content")) || hash == "" {
|
||||||
|
t.Fatalf("Store returned size=%d hash=%q", size, hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
exists, err := store.Exists(ctx, "npm/pkg/file.tgz")
|
||||||
|
if err != nil || !exists {
|
||||||
|
t.Fatalf("Exists = %v, %v; want true, nil", exists, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
r, err := store.Open(ctx, "npm/pkg/file.tgz")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Open failed: %v", err)
|
||||||
|
}
|
||||||
|
data, _ := io.ReadAll(r)
|
||||||
|
_ = r.Close()
|
||||||
|
if string(data) != "content" {
|
||||||
|
t.Fatalf("Open content = %q, want content", data)
|
||||||
|
}
|
||||||
|
|
||||||
|
list, err := store.ListPrefix(ctx, "npm/")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListPrefix failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(list) != 1 || list[0].Path != "npm/pkg/file.tgz" {
|
||||||
|
t.Fatalf("ListPrefix = %#v", list)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil {
|
||||||
|
t.Fatalf("Delete failed: %v", err)
|
||||||
|
}
|
||||||
|
exists, err = store.Exists(ctx, "npm/pkg/file.tgz")
|
||||||
|
if err != nil || exists {
|
||||||
|
t.Fatalf("Exists after delete = %v, %v; want false, nil", exists, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGCSSignedURLUsesSigner(t *testing.T) {
|
||||||
|
store := &GCS{
|
||||||
|
bucket: "test-bucket",
|
||||||
|
accessID: "service@example.com",
|
||||||
|
signBytes: func(_ context.Context, b []byte) ([]byte, error) {
|
||||||
|
if !strings.Contains(string(b), "/test-bucket/npm/pkg/file.tgz") {
|
||||||
|
t.Fatalf("string to sign = %q", b)
|
||||||
|
}
|
||||||
|
return []byte("signed"), nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := store.SignedURL(context.Background(), "npm/pkg/file.tgz", time.Minute)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SignedURL failed: %v", err)
|
||||||
|
}
|
||||||
|
u, err := url.Parse(got)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parsing signed URL: %v", err)
|
||||||
|
}
|
||||||
|
if u.Scheme != "https" || u.Host != "storage.googleapis.com" || u.Path != "/test-bucket/npm/pkg/file.tgz" {
|
||||||
|
t.Fatalf("signed URL location = %s", got)
|
||||||
|
}
|
||||||
|
if u.Query().Get("GoogleAccessId") != "service@example.com" {
|
||||||
|
t.Fatalf("GoogleAccessId = %q", u.Query().Get("GoogleAccessId"))
|
||||||
|
}
|
||||||
|
if u.Query().Get("Signature") != base64.StdEncoding.EncodeToString([]byte("signed")) {
|
||||||
|
t.Fatalf("Signature = %q", u.Query().Get("Signature"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJSON(w http.ResponseWriter, v any) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectNameFromPath(p string) string {
|
||||||
|
escaped := strings.TrimPrefix(p, "/storage/v1/b/test-bucket/o/")
|
||||||
|
name, _ := url.PathUnescape(escaped)
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
@ -5,6 +5,9 @@
|
||||||
// - file:///path/to/dir - Local filesystem storage
|
// - file:///path/to/dir - Local filesystem storage
|
||||||
// - s3://bucket-name - Amazon S3
|
// - s3://bucket-name - Amazon S3
|
||||||
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
|
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
|
||||||
|
// - gs://bucket-name - Google Cloud Storage (supports GKE Workload Identity
|
||||||
|
// via Application Default Credentials)
|
||||||
|
// - azblob://container-name - Azure Blob Storage
|
||||||
//
|
//
|
||||||
// Use OpenBucket to create a storage backend from a URL.
|
// Use OpenBucket to create a storage backend from a URL.
|
||||||
package storage
|
package storage
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue