mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-08-23 12:24:57 -04:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
37cc7abfc7 |
127 changed files with 3090 additions and 12291 deletions
13
.github/workflows/ci.yml
vendored
13
.github/workflows/ci.yml
vendored
|
|
@ -13,17 +13,18 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
go-version: ['1.25']
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
go-version: ${{ matrix.go-version }}
|
||||
|
||||
- name: Build
|
||||
run: go build -v ./...
|
||||
|
|
@ -34,14 +35,14 @@ jobs:
|
|||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
go-version: '1.25'
|
||||
|
||||
- name: golangci-lint
|
||||
run: go tool golangci-lint run ./...
|
||||
|
|
|
|||
60
.github/workflows/publish.yml
vendored
60
.github/workflows/publish.yml
vendored
|
|
@ -18,23 +18,14 @@ jobs:
|
|||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
|
@ -42,57 +33,14 @@ jobs:
|
|||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf
|
||||
with:
|
||||
images: ghcr.io/${{ github.repository }}
|
||||
|
||||
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: build
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
|
||||
- name: Sign image by digest
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
IMAGE: ghcr.io/${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
cosign sign --yes "${IMAGE}@${DIGEST}"
|
||||
|
||||
- name: Verify BuildKit attestations and extract SPDX predicates
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
IMAGE: ghcr.io/${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
reference="${IMAGE}@${DIGEST}"
|
||||
docker buildx imagetools inspect "$reference" --format '{{ json .Provenance }}' > provenance.json
|
||||
docker buildx imagetools inspect "$reference" --format '{{ json .SBOM }}' > sbom.json
|
||||
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
jq -e --arg p "$platform" '.[$p].SLSA | type == "object" and length > 0' \
|
||||
provenance.json >/dev/null
|
||||
jq -e --arg p "$platform" '.[$p].SPDX' sbom.json > "sbom-${platform//\//-}.spdx.json"
|
||||
done
|
||||
|
||||
- name: Attest platform SBOMs by digest
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
IMAGE: ghcr.io/${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
reference="${IMAGE}@${DIGEST}"
|
||||
for predicate in sbom-linux-amd64.spdx.json sbom-linux-arm64.spdx.json; do
|
||||
cosign attest --yes --type spdxjson --predicate "$predicate" "$reference"
|
||||
done
|
||||
|
|
|
|||
8
.github/workflows/release.yml
vendored
8
.github/workflows/release.yml
vendored
|
|
@ -14,20 +14,20 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
- uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: false
|
||||
|
||||
- uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
- uses: goreleaser/goreleaser-action@e24998b8b67b290c2fa8b7c14fcfa7de2c5c9b8c # v7.1.0
|
||||
with:
|
||||
version: "~> v2"
|
||||
args: release --clean
|
||||
|
|
|
|||
6
.github/workflows/swagger.yml
vendored
6
.github/workflows/swagger.yml
vendored
|
|
@ -12,14 +12,14 @@ jobs:
|
|||
swagger:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
go-version: '1.25'
|
||||
|
||||
- name: Install swag
|
||||
run: go install github.com/swaggo/swag/cmd/swag@latest
|
||||
|
|
|
|||
4
.github/workflows/zizmor.yml
vendored
4
.github/workflows/zizmor.yml
vendored
|
|
@ -21,9 +21,9 @@ jobs:
|
|||
security-events: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2
|
||||
uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
|
||||
|
|
|
|||
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -14,8 +14,8 @@ proxy
|
|||
coverage.html
|
||||
coverage.txt
|
||||
|
||||
# Go vendor directory (repo root only; embedded UI vendor dirs are tracked)
|
||||
/vendor/
|
||||
# Dependency directories
|
||||
vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
version: "2"
|
||||
|
||||
linters:
|
||||
enable:
|
||||
- gocritic
|
||||
- gocognit
|
||||
- gocyclo
|
||||
- maintidx
|
||||
- dupl
|
||||
- mnd
|
||||
- unparam
|
||||
- ireturn
|
||||
- goconst
|
||||
- errcheck
|
||||
settings:
|
||||
goconst:
|
||||
min-len: 4
|
||||
min-occurrences: 5
|
||||
ignore-tests: true
|
||||
ignore-string-values:
|
||||
- "^[a-z]+$"
|
||||
exclusions:
|
||||
rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- goconst
|
||||
- dupl
|
||||
- mnd
|
||||
|
|
@ -39,7 +39,7 @@ proxy/
|
|||
│ │ └── queries.go # CRUD operations
|
||||
│ ├── storage/ # Artifact file storage
|
||||
│ │ ├── storage.go # Storage interface
|
||||
│ │ └── blob.go # gocloud.dev/blob backends (file, S3, Azure)
|
||||
│ │ └── filesystem.go # Local filesystem impl
|
||||
│ ├── upstream/ # Upstream registry clients
|
||||
│ │ ├── fetcher.go # HTTP artifact fetching
|
||||
│ │ └── resolver.go # Download URL resolution
|
||||
|
|
@ -72,7 +72,7 @@ Key types:
|
|||
|
||||
### `internal/storage`
|
||||
|
||||
Artifact file storage abstraction backed by `gocloud.dev/blob`. Supports local filesystem (`file://`), S3 (`s3://`), and Azure (`azblob://`) URLs.
|
||||
Artifact file storage abstraction. Currently implements local filesystem storage. Designed to allow future backends (S3, GCS).
|
||||
|
||||
Interface:
|
||||
```go
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM --platform=$BUILDPLATFORM golang:1.26.6-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
|
|
@ -12,11 +12,10 @@ RUN go mod download
|
|||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the binary for the target platform
|
||||
ARG TARGETARCH
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy
|
||||
# Build the binary
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy
|
||||
|
||||
FROM alpine:3.24.1
|
||||
FROM alpine:3.21
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
|
|
|
|||
167
README.md
167
README.md
|
|
@ -32,13 +32,11 @@ Resolution order: package override, then ecosystem override, then global default
|
|||
| pub.dev | Dart | Yes | ✓ |
|
||||
| PyPI | Python | Yes | ✓ |
|
||||
| Maven | Java | | ✓ |
|
||||
| Gradle Build Cache | Java/Kotlin | | ✓ |
|
||||
| NuGet | .NET | Yes | ✓ |
|
||||
| Composer | PHP | Yes | ✓ |
|
||||
| Conan | C/C++ | | ✓ |
|
||||
| Conda | Python/R | Yes | ✓ |
|
||||
| CRAN | R | | ✓ |
|
||||
| Julia | Julia | | ✓ |
|
||||
| Container | Docker/OCI | | ✓ |
|
||||
| Debian | Debian/Ubuntu | | ✓ |
|
||||
| RPM | RHEL/Fedora | | ✓ |
|
||||
|
|
@ -210,34 +208,6 @@ Add to your `~/.m2/settings.xml`:
|
|||
</settings>
|
||||
```
|
||||
|
||||
The `/maven/` endpoint uses Maven Central as primary upstream and falls back to the Gradle Plugin Portal for Gradle plugin marker metadata and related artifacts when the primary upstream returns not found.
|
||||
|
||||
For Gradle plugin resolution via the same proxy endpoint:
|
||||
|
||||
```kotlin
|
||||
pluginManagement {
|
||||
repositories {
|
||||
maven(url = "http://localhost:8080/maven/")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Gradle HTTP Build Cache
|
||||
|
||||
Configure in `settings.gradle(.kts)`:
|
||||
|
||||
```kotlin
|
||||
buildCache {
|
||||
local {
|
||||
enabled = false
|
||||
}
|
||||
remote<HttpBuildCache> {
|
||||
url = uri("http://localhost:8080/gradle/")
|
||||
push = true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### NuGet
|
||||
|
||||
Configure in `nuget.config`:
|
||||
|
|
@ -325,21 +295,6 @@ local({
|
|||
})
|
||||
```
|
||||
|
||||
### Julia
|
||||
|
||||
Set the Pkg server before starting Julia:
|
||||
|
||||
```bash
|
||||
export JULIA_PKG_SERVER=http://localhost:8080/julia
|
||||
```
|
||||
|
||||
Or inside a running session:
|
||||
|
||||
```julia
|
||||
ENV["JULIA_PKG_SERVER"] = "http://localhost:8080/julia"
|
||||
using Pkg; Pkg.update()
|
||||
```
|
||||
|
||||
### Docker / Container Registry
|
||||
|
||||
Configure Docker to use the proxy as a registry mirror in `/etc/docker/daemon.json`:
|
||||
|
|
@ -362,39 +317,6 @@ Or pull images directly:
|
|||
docker pull localhost:8080/library/nginx:latest
|
||||
```
|
||||
|
||||
### Helm
|
||||
|
||||
Configure each HTTP chart repository with a name, then add the matching proxy
|
||||
URL to Helm:
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
helm:
|
||||
bitnami: "https://charts.bitnami.com/bitnami"
|
||||
```
|
||||
|
||||
```bash
|
||||
helm repo add bitnami http://localhost:8080/helm/bitnami
|
||||
helm repo update
|
||||
helm pull bitnami/nginx
|
||||
```
|
||||
|
||||
The proxy caches `index.yaml` using the normal metadata-cache settings and
|
||||
caches chart archives after verifying their SHA-256 digest from the index.
|
||||
|
||||
For charts stored in an OCI registry, configure a named OCI upstream and add
|
||||
the reserved `upstream/{name}` prefix to the chart reference:
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
oci:
|
||||
ghcr: "https://ghcr.io"
|
||||
```
|
||||
|
||||
```bash
|
||||
helm pull oci://localhost:8080/upstream/ghcr/owner/charts/mychart --version 1.0.0 --plain-http
|
||||
```
|
||||
|
||||
### Debian / APT
|
||||
|
||||
Configure APT to use the proxy in `/etc/apt/sources.list.d/proxy.list`:
|
||||
|
|
@ -409,13 +331,6 @@ Replace your existing sources.list entries, then:
|
|||
sudo apt update
|
||||
```
|
||||
|
||||
The upstream defaults to `http://deb.debian.org/debian`. To proxy a different APT repository (e.g. Ubuntu), set `upstream.debian` in the config file or `PROXY_UPSTREAM_DEBIAN` in the environment:
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
debian: "http://archive.ubuntu.com/ubuntu"
|
||||
```
|
||||
|
||||
### RPM / Yum / DNF
|
||||
|
||||
Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`:
|
||||
|
|
@ -438,7 +353,6 @@ sudo dnf update
|
|||
## Configuration
|
||||
|
||||
The proxy can be configured via:
|
||||
|
||||
1. Command line flags (highest priority)
|
||||
2. Environment variables
|
||||
3. Configuration file (YAML or JSON)
|
||||
|
|
@ -456,7 +370,6 @@ The proxy can be configured via:
|
|||
-database-url string PostgreSQL connection URL
|
||||
-log-level string Log level: debug, info, warn, error (default "info")
|
||||
-log-format string Log format: text, json (default "text")
|
||||
-access-log string Path to the JSONL access log
|
||||
-version Print version and exit
|
||||
```
|
||||
|
||||
|
|
@ -465,14 +378,12 @@ The proxy can be configured via:
|
|||
```bash
|
||||
PROXY_LISTEN=:8080
|
||||
PROXY_BASE_URL=http://localhost:8080
|
||||
PROXY_UI_URL=http://localhost:8080 # Optional; defaults to PROXY_BASE_URL
|
||||
PROXY_STORAGE_URL=file:///var/cache/proxy/artifacts
|
||||
PROXY_DATABASE_DRIVER=sqlite
|
||||
PROXY_DATABASE_PATH=./cache/proxy.db
|
||||
PROXY_DATABASE_URL=postgres://user:pass@localhost/proxy?sslmode=disable
|
||||
PROXY_LOG_LEVEL=info
|
||||
PROXY_LOG_FORMAT=text
|
||||
PROXY_ACCESS_LOG_PATH=/var/log/proxy/access.jsonl
|
||||
```
|
||||
|
||||
### Configuration File
|
||||
|
|
@ -493,9 +404,6 @@ log:
|
|||
level: "info"
|
||||
format: "text"
|
||||
|
||||
access_log:
|
||||
path: "/var/log/proxy/access.jsonl" # Optional JSONL activity log
|
||||
|
||||
# Optional: override upstream URLs
|
||||
upstream:
|
||||
npm: "https://registry.npmjs.org"
|
||||
|
|
@ -652,7 +560,7 @@ Recently cached:
|
|||
| Endpoint | Description |
|
||||
|----------|-------------|
|
||||
| `GET /` | Dashboard (web UI) |
|
||||
| `GET /health` | Health check (JSON; HTTP 200 healthy, 503 unhealthy) |
|
||||
| `GET /health` | Health check (returns "ok" if healthy) |
|
||||
| `GET /stats` | Cache statistics (JSON) |
|
||||
| `GET /metrics` | Prometheus metrics |
|
||||
| `GET /npm/*` | npm registry protocol |
|
||||
|
|
@ -668,8 +576,6 @@ Recently cached:
|
|||
| `GET /conan/*` | Conan C/C++ protocol |
|
||||
| `GET /conda/*` | Conda/Anaconda protocol |
|
||||
| `GET /cran/*` | CRAN (R) protocol |
|
||||
| `GET /julia/*` | Julia Pkg server protocol |
|
||||
| `GET /helm/{repository}/*` | HTTP Helm chart repository protocol |
|
||||
| `GET /v2/*` | OCI/Docker registry protocol |
|
||||
| `GET /debian/*` | Debian/APT repository protocol |
|
||||
| `GET /rpm/*` | RPM/Yum repository protocol |
|
||||
|
|
@ -866,16 +772,16 @@ Response:
|
|||
|
||||
## Web Interface
|
||||
|
||||
The proxy serves a web UI under `/ui`. No separate frontend build is needed -- templates and assets are embedded in the binary. `GET /` redirects to `/ui/`. The UI is mounted under its own prefix so a reverse proxy can apply different access rules to it than to the package endpoints (for example, requiring auth for `PathPrefix(/ui)` while leaving `/npm`, `/pypi` etc. open to build machines).
|
||||
The proxy serves a web UI at the root URL. No separate frontend build is needed -- templates and assets are embedded in the binary.
|
||||
|
||||
- **Dashboard** (`/ui/`) -- cache stats, popular packages, recently cached artifacts, and vulnerability overview.
|
||||
- **Install guide** (`/ui/install`) -- per-ecosystem configuration instructions, so you don't have to look them up here.
|
||||
- **Package browser** (`/ui/packages`) -- browse all cached packages with filtering by ecosystem and sorting by hits, size, name, or vulnerability count.
|
||||
- **Search** (`/ui/search?q=...`) -- search cached packages by name.
|
||||
- **Package detail** (`/ui/package/{ecosystem}/{name}`) -- metadata, license, vulnerabilities, and version list for a package. You can select two versions to compare.
|
||||
- **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts.
|
||||
- **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews.
|
||||
- **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files.
|
||||
- **Dashboard** (`/`) -- cache stats, popular packages, recently cached artifacts, and vulnerability overview.
|
||||
- **Install guide** (`/install`) -- per-ecosystem configuration instructions, so you don't have to look them up here.
|
||||
- **Package browser** (`/packages`) -- browse all cached packages with filtering by ecosystem and sorting by hits, size, name, or vulnerability count.
|
||||
- **Search** (`/search?q=...`) -- search cached packages by name.
|
||||
- **Package detail** (`/package/{ecosystem}/{name}`) -- metadata, license, vulnerabilities, and version list for a package. You can select two versions to compare.
|
||||
- **Version detail** (`/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts.
|
||||
- **Source browser** (`/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews.
|
||||
- **Version diff** (`/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files.
|
||||
|
||||
## Monitoring
|
||||
|
||||
|
|
@ -883,8 +789,6 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
|
|||
|
||||
| Metric | Type | Labels | Description |
|
||||
|--------|------|--------|-------------|
|
||||
| `proxy_requests_total` | counter | `ecosystem`, `status` | Proxy responses by package ecosystem and HTTP status |
|
||||
| `proxy_request_duration_seconds` | histogram | `ecosystem`, `status` | Proxy request duration |
|
||||
| `proxy_cache_hits_total` | counter | `ecosystem` | Cache hits |
|
||||
| `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses |
|
||||
| `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts |
|
||||
|
|
@ -894,28 +798,9 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
|
|||
| `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency |
|
||||
| `proxy_storage_errors_total` | counter | `operation` | Storage read/write failures |
|
||||
| `proxy_active_requests` | gauge | | In-flight requests |
|
||||
| `proxy_health_probe_failures_total` | counter | `step` | Storage health probe failures by failing step (`write`, `size`, `read`, `verify`, `delete`). |
|
||||
|
||||
Cache size and artifact count are refreshed every 60 seconds. The remaining metrics update on each request.
|
||||
|
||||
### Health Check
|
||||
|
||||
`/health` returns a structured JSON report of subsystem health. HTTP 200 if all checks pass; 503 if any fail.
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"checks": {
|
||||
"database": {"status": "ok"},
|
||||
"storage": {"status": "ok"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Failing checks include an `"error"` field. Storage failures also include a `"step"` field identifying which probe step failed (`write`, `size`, `read`, `verify`, `delete`). When the database check fails, the storage entry reports `{"status": "skipped"}` so the response always carries the same key set.
|
||||
|
||||
Storage probe results are cached for `health.storage_probe_interval` (default 30s) to bound the cost of probing remote backends. A probe holds an internal mutex for up to 10 seconds (the hardcoded per-probe timeout), so `/health` is intended as a Kubernetes **readiness** probe rather than a liveness probe — a slow S3 round-trip should pull the pod from rotation, not restart it.
|
||||
|
||||
Scrape config for Prometheus:
|
||||
|
||||
```yaml
|
||||
|
|
@ -983,45 +868,20 @@ When running behind nginx, Apache, or another reverse proxy, set `base_url` to y
|
|||
base_url: "https://proxy.example.com"
|
||||
```
|
||||
|
||||
If the UI is reached on a different hostname than the package endpoints — for example, the UI exposed publicly on a domain while build machines hit a Docker network alias — set `ui_base_url` separately. `base_url` is the URL package managers and metadata rewriting use; `ui_base_url` is the URL advertised to humans visiting the web UI (canonical/`og:url` tags and the install guide banner):
|
||||
|
||||
```yaml
|
||||
base_url: "http://pkg-proxy:8080" # internal alias for build machines
|
||||
ui_base_url: "https://proxy.example.com/ui" # public UI URL
|
||||
```
|
||||
|
||||
When unset, `ui_base_url` defaults to `base_url`.
|
||||
|
||||
> **Warning:** the proxy serves the UI and package endpoints on the same listener. Setting `ui_base_url` only changes what URL the UI advertises to humans; it does not stop package endpoints from being reachable on the same hostname and port. When fronting the proxy with a public reverse proxy, restrict the public route to `PathPrefix(/ui)` (or your proxy's equivalent), otherwise `/npm`, `/pypi`, and the other package endpoints stay exposed alongside the UI.
|
||||
|
||||
nginx example, restricting the public host to the UI while leaving package endpoints reachable only on the internal listener:
|
||||
nginx example:
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name proxy.example.com;
|
||||
|
||||
location /ui/ {
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8080;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_buffering off;
|
||||
}
|
||||
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Traefik example using `PathPrefix(/ui)` so the public router only matches UI traffic:
|
||||
|
||||
```yaml
|
||||
labels:
|
||||
traefik.enable: "true"
|
||||
traefik.http.services.pkg-proxy.loadbalancer.server.port: "8080"
|
||||
traefik.http.routers.pkg-proxy.rule: "Host(`proxy.example.com`) && PathPrefix(`/ui`)"
|
||||
traefik.http.routers.pkg-proxy.entrypoints: "websecure"
|
||||
```
|
||||
|
||||
## Cache Management
|
||||
|
|
@ -1064,8 +924,7 @@ The proxy will recreate the database on next start.
|
|||
## Building from Source
|
||||
|
||||
Requirements:
|
||||
|
||||
- Go (the project version is declared in `go.mod`)
|
||||
- Go 1.25 or later
|
||||
|
||||
```bash
|
||||
git clone https://github.com/git-pkgs/proxy.git
|
||||
|
|
|
|||
|
|
@ -40,8 +40,6 @@
|
|||
// Log level: debug, info, warn, error (default "info")
|
||||
// -log-format string
|
||||
// Log format: text, json (default "text")
|
||||
// -access-log string
|
||||
// Path to the JSONL access log (disabled by default)
|
||||
//
|
||||
// Stats Flags:
|
||||
//
|
||||
|
|
@ -74,15 +72,6 @@
|
|||
// PROXY_DATABASE_URL - PostgreSQL connection URL
|
||||
// PROXY_LOG_LEVEL - Log level
|
||||
// PROXY_LOG_FORMAT - Log format
|
||||
// PROXY_ACCESS_LOG_PATH - JSONL access log path
|
||||
// PROXY_UPSTREAM_MAVEN - Maven repository upstream URL
|
||||
// PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL - Gradle Plugin Portal upstream URL
|
||||
// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction
|
||||
// PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size
|
||||
// PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval
|
||||
// PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s")
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
|
|
@ -187,7 +176,6 @@ func runServe() {
|
|||
databaseURL := fs.String("database-url", "", "PostgreSQL connection URL")
|
||||
logLevel := fs.String("log-level", "", "Log level: debug, info, warn, error")
|
||||
logFormat := fs.String("log-format", "", "Log format: text, json")
|
||||
accessLogPath := fs.String("access-log", "", "Path to the JSONL access log")
|
||||
version := fs.Bool("version", false, "Print version and exit")
|
||||
|
||||
fs.Usage = func() {
|
||||
|
|
@ -205,15 +193,6 @@ func runServe() {
|
|||
fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_ACCESS_LOG_PATH JSONL access log path\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL Gradle Plugin Portal upstream URL\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL Gradle cache eviction sweep interval\n")
|
||||
fmt.Fprintf(os.Stderr, " PROXY_HEALTH_STORAGE_PROBE_INTERVAL Storage health probe cache interval\n")
|
||||
}
|
||||
|
||||
_ = fs.Parse(os.Args[1:])
|
||||
|
|
@ -261,9 +240,6 @@ func runServe() {
|
|||
if *logFormat != "" {
|
||||
cfg.Log.Format = *logFormat
|
||||
}
|
||||
if *accessLogPath != "" {
|
||||
cfg.AccessLog.Path = *accessLogPath
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if err := cfg.Validate(); err != nil {
|
||||
|
|
@ -275,10 +251,7 @@ func runServe() {
|
|||
logger := setupLogger(cfg.Log.Level, cfg.Log.Format)
|
||||
|
||||
// Create and start server
|
||||
srv, err := server.New(cfg, logger, server.BuildInfo{
|
||||
Version: Version,
|
||||
Commit: Commit,
|
||||
})
|
||||
srv, err := server.New(cfg, logger)
|
||||
if err != nil {
|
||||
logger.Error("failed to create server", "error", err)
|
||||
os.Exit(1)
|
||||
|
|
@ -481,7 +454,6 @@ func runMirror() {
|
|||
proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
|
||||
proxy.CacheMetadata = true // mirror always caches metadata
|
||||
proxy.MetadataTTL = cfg.ParseMetadataTTL()
|
||||
proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
|
||||
|
||||
m := mirror.New(proxy, db, store, logger, *concurrency)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,22 +4,10 @@
|
|||
# Server listen address
|
||||
listen: ":8080"
|
||||
|
||||
# Public URL where package endpoints are reachable.
|
||||
# Used for rewriting package metadata URLs and shown in install guide snippets
|
||||
# so users know what to point their package manager at.
|
||||
# Public URL where this proxy is accessible
|
||||
# Used for rewriting package metadata URLs
|
||||
base_url: "http://localhost:8080"
|
||||
|
||||
# Timeout for individual upstream HTTP requests made by protocol handlers
|
||||
# (metadata fetches, pass-through file requests). Uses Go duration syntax.
|
||||
# Set to "0" to disable the timeout. Default: "30s".
|
||||
# http_timeout: "30s"
|
||||
|
||||
# Public URL where the web UI is reached. Defaults to base_url when unset.
|
||||
# Set this separately when the UI is served on a different hostname than the
|
||||
# package endpoints — for example, the UI on a public domain behind auth while
|
||||
# build machines hit a Docker network alias for the package endpoints.
|
||||
# ui_base_url: "https://proxy.example.com/ui"
|
||||
|
||||
# Artifact storage configuration
|
||||
storage:
|
||||
# Storage backend URL
|
||||
|
|
@ -78,42 +66,19 @@ log:
|
|||
# Log format: "text" or "json"
|
||||
format: "text"
|
||||
|
||||
# JSONL access log. Leave path empty to disable it.
|
||||
access_log:
|
||||
path: ""
|
||||
|
||||
# Upstream registry URLs and authentication
|
||||
upstream:
|
||||
# npm registry URL
|
||||
npm: "https://registry.npmjs.org"
|
||||
|
||||
# Maven repository URL (used by /maven endpoint)
|
||||
maven: "https://repo1.maven.org/maven2"
|
||||
|
||||
# Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts)
|
||||
gradle_plugin_portal: "https://plugins.gradle.org/m2"
|
||||
|
||||
# Cargo sparse index URL
|
||||
cargo: "https://index.crates.io"
|
||||
|
||||
# Cargo crate download URL
|
||||
cargo_download: "https://static.crates.io/crates"
|
||||
|
||||
# Debian/APT repository URL (used by /debian endpoint)
|
||||
debian: "http://deb.debian.org/debian"
|
||||
|
||||
# Named HTTP Helm chart repositories (used by /helm/{name}/)
|
||||
# helm:
|
||||
# bitnami: "https://charts.bitnami.com/bitnami"
|
||||
|
||||
# Named OCI registries. Use the upstream/{name}/ repository prefix, e.g.
|
||||
# oci://proxy.example.com/upstream/ghcr/owner/chart.
|
||||
# oci:
|
||||
# ghcr: "https://ghcr.io"
|
||||
|
||||
# Authentication for upstream registries
|
||||
# Keys are absolute URL scopes. Scheme, host, effective port, and path
|
||||
# segment boundaries must match; the longest matching scope wins.
|
||||
# Keys are URL prefixes matched against request URLs.
|
||||
# Values can reference environment variables using ${VAR_NAME} syntax.
|
||||
#
|
||||
# Supported auth types:
|
||||
|
|
@ -143,35 +108,6 @@ upstream:
|
|||
# header_name: "X-Auth-Token"
|
||||
# header_value: "${MAVEN_TOKEN}"
|
||||
|
||||
# Gradle HttpBuildCache configuration
|
||||
gradle:
|
||||
build_cache:
|
||||
# Set to true to disable PUT uploads (read-only cache mode)
|
||||
read_only: false
|
||||
|
||||
# Maximum accepted Gradle cache upload body size
|
||||
# Required and must be > 0
|
||||
max_upload_size: "100MB"
|
||||
|
||||
# Evict entries older than this age (set to "0" to disable age-based eviction)
|
||||
max_age: "168h"
|
||||
|
||||
# Cap total Gradle cache size; oldest entries are deleted first
|
||||
# ("0" disables size-based eviction)
|
||||
# max_size: "20GB"
|
||||
|
||||
# How often eviction runs when max_age or max_size is set
|
||||
sweep_interval: "10m"
|
||||
|
||||
# Health endpoint configuration.
|
||||
health:
|
||||
# Minimum time between storage backend probes.
|
||||
# The /health endpoint runs a write/read/verify/delete round-trip
|
||||
# against the configured storage backend and caches the result for
|
||||
# this interval. Set to "0" to probe on every request.
|
||||
# Default: "30s".
|
||||
storage_probe_interval: "30s"
|
||||
|
||||
# Version cooldown configuration
|
||||
# Hides package versions published too recently, giving the community time
|
||||
# to spot malicious releases before they're pulled into projects.
|
||||
|
|
@ -185,8 +121,7 @@ cooldown:
|
|||
# npm: "7d"
|
||||
# cargo: "0"
|
||||
|
||||
# Per-package overrides (keyed by PURL). Keys are normalized, so npm scopes
|
||||
# may use either @scope or the canonical %40scope form.
|
||||
# Per-package overrides (keyed by PURL)
|
||||
# packages:
|
||||
# "pkg:npm/lodash": "0"
|
||||
# "pkg:npm/@babel/core": "14d"
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ The proxy is a caching HTTP server that sits between package manager clients and
|
|||
│ │ /npm/* -> NPMHandler /health -> healthHandler │ │
|
||||
│ │ /cargo/* -> CargoHandler /stats -> statsHandler │ │
|
||||
│ │ /gem/* -> GemHandler /metrics -> prometheus │ │
|
||||
│ │ ...17 ecosystems /api/* -> APIHandler │ │
|
||||
│ │ /ui/* -> Web UI │ │
|
||||
│ │ ...16 ecosystems /api/* -> APIHandler │ │
|
||||
│ │ / -> Web UI │ │
|
||||
│ └──────────────────────────────────────────────────────────┘ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
|
|
@ -240,8 +240,6 @@ Fetches artifacts from upstream registries.
|
|||
- Exponential backoff retry on 429 (rate limit) and 5xx errors
|
||||
- Returns streaming reader (doesn't load into memory)
|
||||
- Configurable user-agent
|
||||
- Shares an authentication-aware transport with metadata requests so URL-scoped credentials apply consistently
|
||||
- Discovers and caches scoped OCI Bearer tokens from registry challenges
|
||||
|
||||
**Resolver:**
|
||||
- Determines download URL for a package/version
|
||||
|
|
@ -276,18 +274,18 @@ HTTP server setup, web UI, and API handlers.
|
|||
- Creates and wires together all components
|
||||
- Mounts protocol handlers at ecosystem-specific paths
|
||||
- Middleware: request ID, real IP, logging, panic recovery, active request tracking
|
||||
- Web UI under `/ui`: dashboard, package browser, source browser, version comparison
|
||||
- Web UI: dashboard, package browser, source browser, version comparison
|
||||
- Templates are embedded in the binary via `//go:embed`
|
||||
- Enrichment API for package metadata, vulnerability scanning, and outdated detection
|
||||
- Health, stats, and Prometheus metrics endpoints. `/health` runs an active write → size-check → read → verify → delete probe against the storage backend and returns a structured JSON response (`HealthResponse`) with `"ok"` / `"error"` status per subsystem. Probe results are cached (default 30 s, configurable via `health.storage_probe_interval`) to avoid overwhelming remote backends.
|
||||
- Health, stats, and Prometheus metrics endpoints
|
||||
|
||||
### `internal/metrics`
|
||||
|
||||
Prometheus metrics for cache performance, upstream latency, storage operations, and active requests. See the Monitoring section of the README for the full metric list.
|
||||
|
||||
### Cooldown
|
||||
### `internal/cooldown`
|
||||
|
||||
Version age filtering for supply chain attack mitigation, provided by [github.com/git-pkgs/cooldown](https://github.com/git-pkgs/cooldown). Configurable at global, ecosystem, and per-package levels. Supported by npm, PyPI, pub.dev, and Composer handlers.
|
||||
Version age filtering for supply chain attack mitigation. Configurable at global, ecosystem, and per-package levels. Supported by npm, PyPI, pub.dev, and Composer handlers.
|
||||
|
||||
### `internal/enrichment`
|
||||
|
||||
|
|
@ -353,7 +351,6 @@ Eviction can be implemented as:
|
|||
- Fresh data - new versions visible immediately
|
||||
- Metadata is small, upstream fetch is fast
|
||||
- Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table
|
||||
- OCI manifests are the exception: they are cached automatically so previously fetched images remain pullable when the registry or token service is unavailable
|
||||
|
||||
**Why stream artifacts?**
|
||||
- Memory efficient - don't load large files into RAM
|
||||
|
|
|
|||
|
|
@ -17,8 +17,7 @@ See `config.example.yaml` in the repository root for a complete example.
|
|||
| Config | Environment | Flag | Default | Description |
|
||||
|--------|-------------|------|---------|-------------|
|
||||
| `listen` | `PROXY_LISTEN` | `-listen` | `:8080` | Address to listen on |
|
||||
| `base_url` | `PROXY_BASE_URL` | `-base-url` | `http://localhost:8080` | Public URL package managers use to reach this proxy |
|
||||
| `ui_base_url` | `PROXY_UI_URL` | - | (defaults to `base_url`) | Public URL where the web UI is reached. Set separately when the UI lives behind a different hostname than package endpoints (e.g. public domain vs Docker network alias). Used for canonical/og:url tags and the install guide banner. The proxy still serves package endpoints on the same listener, so any reverse proxy fronting the UI publicly should restrict the public route to `PathPrefix(/ui)` to avoid exposing package endpoints. |
|
||||
| `base_url` | `PROXY_BASE_URL` | `-base-url` | `http://localhost:8080` | Public URL for the proxy |
|
||||
|
||||
## Storage
|
||||
|
||||
|
|
@ -108,30 +107,6 @@ log:
|
|||
| `log.level` | `PROXY_LOG_LEVEL` | `-log-level` | `debug`, `info`, `warn`, `error` |
|
||||
| `log.format` | `PROXY_LOG_FORMAT` | `-log-format` | `text`, `json` |
|
||||
|
||||
## Access Log
|
||||
|
||||
The optional access log records client requests and each HTTP exchange with an upstream registry. It is always written as JSONL, with one JSON object per line. Records for the same client request share a `request_id`.
|
||||
|
||||
```yaml
|
||||
access_log:
|
||||
path: "/var/log/proxy/access.jsonl"
|
||||
```
|
||||
|
||||
| Config | Environment | Flag | Description |
|
||||
|--------|-------------|------|-------------|
|
||||
| `access_log.path` | `PROXY_ACCESS_LOG_PATH` | `-access-log` | File to append JSONL records to; empty disables the log |
|
||||
|
||||
The parent directory must exist and be writable when the proxy starts. A newly created log file is readable and writable only by the proxy process owner.
|
||||
|
||||
A request that receives a rate limit response from an upstream can produce records like these:
|
||||
|
||||
```json
|
||||
{"time":"2026-08-16T12:00:00Z","event":"upstream","request_id":"host/example-000001","method":"GET","url":"https://registry.example/packages/example","status_code":429,"duration_ms":42}
|
||||
{"time":"2026-08-16T12:00:00Z","event":"request","request_id":"host/example-000001","method":"GET","path":"/npm/example","status_code":502,"duration_ms":43,"remote_addr":"192.0.2.10:41234"}
|
||||
```
|
||||
|
||||
Upstream retries and OCI authentication calls are separate `upstream` records, so the log preserves every status returned over the wire. Network failures have an `error` field and no `status_code`. URL credentials, query strings, and fragments are omitted from both upstream URLs and client paths.
|
||||
|
||||
## Upstream Registries
|
||||
|
||||
Override default upstream registry URLs:
|
||||
|
|
@ -139,37 +114,13 @@ Override default upstream registry URLs:
|
|||
```yaml
|
||||
upstream:
|
||||
npm: "https://registry.npmjs.org"
|
||||
maven: "https://repo1.maven.org/maven2"
|
||||
gradle_plugin_portal: "https://plugins.gradle.org/m2"
|
||||
cargo: "https://index.crates.io"
|
||||
cargo_download: "https://static.crates.io/crates"
|
||||
|
||||
# Named HTTP Helm chart repositories, served at /helm/{name}/.
|
||||
helm:
|
||||
bitnami: "https://charts.bitnami.com/bitnami"
|
||||
|
||||
# Named OCI registries. Select one with the repository prefix
|
||||
# upstream/{name}/, e.g. oci://proxy.example.com/upstream/ghcr/owner/chart.
|
||||
oci:
|
||||
ghcr: "https://ghcr.io"
|
||||
```
|
||||
|
||||
Helm HTTP repositories are read-only. The proxy fetches and rewrites each
|
||||
repository's `index.yaml` so chart archives are downloaded through the proxy.
|
||||
Chart archives are retained only when their SHA-256 digest matches the digest
|
||||
listed in the index. Relative and absolute chart URLs are both supported.
|
||||
|
||||
Named OCI registries preserve the existing unprefixed Docker Hub mirror. A
|
||||
reference such as `oci://proxy.example.com/upstream/ghcr/owner/chart` is sent
|
||||
to the registry configured as `ghcr` with `owner/chart` as its repository.
|
||||
When the proxy uses plain HTTP (for example `localhost:8080`), pass
|
||||
`--plain-http` to Helm OCI commands.
|
||||
|
||||
## Authentication
|
||||
|
||||
Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax.
|
||||
|
||||
OCI registries that return a Bearer challenge from a `/v2/{repository}/…` endpoint are handled automatically. The proxy discovers the token realm from `WWW-Authenticate`, applies any configured credentials for the token URL, and reuses the scoped token until shortly before it expires.
|
||||
Configure authentication for private upstream registries. Auth is matched by URL prefix, and credentials can reference environment variables using `${VAR_NAME}` syntax.
|
||||
|
||||
### Bearer Token
|
||||
|
||||
|
|
@ -218,7 +169,7 @@ upstream:
|
|||
|
||||
### URL Matching
|
||||
|
||||
Auth keys must be absolute URLs. Matching compares the scheme, host, effective port, and path-segment prefix, preventing credentials for `registry.example.com` from being sent to a lookalike host such as `registry.example.com.evil.test`. The longest matching scope wins, so you can configure different credentials for different paths:
|
||||
Auth configs are matched by URL prefix. The longest matching prefix wins, so you can configure different credentials for different paths:
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
|
|
@ -233,30 +184,6 @@ upstream:
|
|||
token: "${PRIVATE_TOKEN}"
|
||||
```
|
||||
|
||||
## Gradle Build Cache
|
||||
|
||||
The `/gradle` endpoint supports optional safeguards for upload control and cache retention.
|
||||
|
||||
```yaml
|
||||
gradle:
|
||||
build_cache:
|
||||
read_only: false
|
||||
max_upload_size: "100MB"
|
||||
max_age: "168h"
|
||||
max_size: "20GB"
|
||||
sweep_interval: "10m"
|
||||
```
|
||||
|
||||
| Config | Environment | Description |
|
||||
|--------|-------------|-------------|
|
||||
| `gradle.build_cache.read_only` | `PROXY_GRADLE_BUILD_CACHE_READ_ONLY` | Disable PUT uploads and keep GET/HEAD read-only |
|
||||
| `gradle.build_cache.max_upload_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE` | Maximum accepted PUT body size (must be > 0) |
|
||||
| `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, set `0` to disable) |
|
||||
| `gradle.build_cache.max_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_SIZE` | Total size cap for `_gradle/http-build-cache`, deleting oldest first (`0` disables) |
|
||||
| `gradle.build_cache.sweep_interval` | `PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL` | Frequency for background eviction sweeps |
|
||||
|
||||
`max_age` and `max_size` are independent and can be combined. When both are set, age-based eviction runs first, then size-based eviction trims remaining entries oldest-first.
|
||||
|
||||
## Cooldown
|
||||
|
||||
The cooldown feature hides package versions published too recently, giving the community time to spot malicious releases before they reach your projects. When a version is within its cooldown period, it's stripped from metadata responses so package managers won't install it.
|
||||
|
|
@ -280,8 +207,6 @@ cooldown:
|
|||
|
||||
Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable.
|
||||
|
||||
Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins.
|
||||
|
||||
Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages.
|
||||
|
||||
Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata.
|
||||
|
|
@ -292,8 +217,6 @@ Note: Hex cooldown requires disabling registry signature verification since the
|
|||
|
||||
By default the proxy fetches metadata fresh from upstream on every request. Enable `cache_metadata` to store metadata responses in the database and storage backend for offline fallback. When upstream is unreachable, the proxy serves the last cached copy. ETag-based revalidation avoids re-downloading unchanged metadata.
|
||||
|
||||
OCI manifests are always cached because cached image blobs cannot be pulled without their manifests. Digest-addressed manifests are immutable and served directly from cache. Tag-addressed manifests follow `metadata_ttl`, revalidate when stale, and fall back to the last cached response when the registry is unavailable.
|
||||
|
||||
```yaml
|
||||
cache_metadata: true
|
||||
```
|
||||
|
|
@ -316,28 +239,6 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests
|
|||
|
||||
When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated.
|
||||
|
||||
### Metadata size limit
|
||||
|
||||
Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs.
|
||||
|
||||
```yaml
|
||||
metadata_max_size: "100MB" # default
|
||||
```
|
||||
|
||||
Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`.
|
||||
|
||||
## Upstream HTTP timeout
|
||||
|
||||
Protocol handlers use a shared HTTP client for upstream requests such as metadata fetches and pass-through file downloads. `http_timeout` sets that client's per-request timeout. Raise it if slow upstreams or large metadata responses cause `context deadline exceeded` errors.
|
||||
|
||||
```yaml
|
||||
http_timeout: "30s" # default
|
||||
```
|
||||
|
||||
Or via environment variable: `PROXY_HTTP_TIMEOUT=2m`.
|
||||
|
||||
Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout).
|
||||
|
||||
## Mirror API
|
||||
|
||||
The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP:
|
||||
|
|
|
|||
|
|
@ -15,246 +15,7 @@ const docTemplate = `{
|
|||
"host": "{{.Host}}",
|
||||
"basePath": "{{.BasePath}}",
|
||||
"paths": {
|
||||
"/api/bulk": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Bulk package lookup by PURL",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "PURLs",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.BulkRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.BulkResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/outdated": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Check outdated packages",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "Packages to check",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.OutdatedRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.OutdatedResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/packages": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "List cached packages",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Ecosystem",
|
||||
"name": "ecosystem",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"enum": [
|
||||
"hits",
|
||||
"name",
|
||||
"size",
|
||||
"cached_at",
|
||||
"ecosystem",
|
||||
"vulns"
|
||||
],
|
||||
"type": "string",
|
||||
"description": "Sort",
|
||||
"name": "sort",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.PackagesListResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/search": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Search cached packages",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Query",
|
||||
"name": "q",
|
||||
"in": "query",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Ecosystem",
|
||||
"name": "ecosystem",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.SearchResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/health": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"meta"
|
||||
],
|
||||
"summary": "Health check",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.HealthResponse"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.HealthResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/stats": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"meta"
|
||||
],
|
||||
"summary": "Cache statistics",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.StatsResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ui/api/browse/{ecosystem}/{name}/{version}": {
|
||||
"/api/browse/{ecosystem}/{name}/{version}": {
|
||||
"get": {
|
||||
"description": "Lists files from the first cached artifact for a package version.",
|
||||
"produces": [
|
||||
|
|
@ -303,19 +64,19 @@ const docTemplate = `{
|
|||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ui/api/browse/{ecosystem}/{name}/{version}/file/{filepath}": {
|
||||
"/api/browse/{ecosystem}/{name}/{version}/file/{filepath}": {
|
||||
"get": {
|
||||
"description": "Streams a single file from the cached artifact. The file path may contain slashes.",
|
||||
"produces": [
|
||||
|
|
@ -365,25 +126,70 @@ const docTemplate = `{
|
|||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ui/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion}": {
|
||||
"/api/bulk": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Bulk package lookup by PURL",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "PURLs",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.BulkRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.BulkResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion}": {
|
||||
"get": {
|
||||
"description": "Returns a structured diff for two cached versions.",
|
||||
"produces": [
|
||||
|
|
@ -434,13 +240,207 @@ const docTemplate = `{
|
|||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/outdated": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Check outdated packages",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "Packages to check",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.OutdatedRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.OutdatedResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/packages": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "List cached packages",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Ecosystem",
|
||||
"name": "ecosystem",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"enum": [
|
||||
"hits",
|
||||
"name",
|
||||
"size",
|
||||
"cached_at",
|
||||
"ecosystem",
|
||||
"vulns"
|
||||
],
|
||||
"type": "string",
|
||||
"description": "Sort",
|
||||
"name": "sort",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.PackagesListResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/search": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Search cached packages",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Query",
|
||||
"name": "q",
|
||||
"in": "query",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Ecosystem",
|
||||
"name": "ecosystem",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.SearchResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/health": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"text/plain"
|
||||
],
|
||||
"tags": [
|
||||
"meta"
|
||||
],
|
||||
"summary": "Health check",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/stats": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"meta"
|
||||
],
|
||||
"summary": "Cache statistics",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.StatsResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -504,45 +504,6 @@ const docTemplate = `{
|
|||
}
|
||||
}
|
||||
},
|
||||
"server.ErrorResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {
|
||||
"type": "string"
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.HealthCheck": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"step": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.HealthResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"checks": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/server.HealthCheck"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.OutdatedPackage": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
|
|||
|
|
@ -8,246 +8,7 @@
|
|||
},
|
||||
"basePath": "/",
|
||||
"paths": {
|
||||
"/api/bulk": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Bulk package lookup by PURL",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "PURLs",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.BulkRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.BulkResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/outdated": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Check outdated packages",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "Packages to check",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.OutdatedRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.OutdatedResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/packages": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "List cached packages",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Ecosystem",
|
||||
"name": "ecosystem",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"enum": [
|
||||
"hits",
|
||||
"name",
|
||||
"size",
|
||||
"cached_at",
|
||||
"ecosystem",
|
||||
"vulns"
|
||||
],
|
||||
"type": "string",
|
||||
"description": "Sort",
|
||||
"name": "sort",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.PackagesListResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/search": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Search cached packages",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Query",
|
||||
"name": "q",
|
||||
"in": "query",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Ecosystem",
|
||||
"name": "ecosystem",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.SearchResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/health": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"meta"
|
||||
],
|
||||
"summary": "Health check",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.HealthResponse"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.HealthResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/stats": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"meta"
|
||||
],
|
||||
"summary": "Cache statistics",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.StatsResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ui/api/browse/{ecosystem}/{name}/{version}": {
|
||||
"/api/browse/{ecosystem}/{name}/{version}": {
|
||||
"get": {
|
||||
"description": "Lists files from the first cached artifact for a package version.",
|
||||
"produces": [
|
||||
|
|
@ -296,19 +57,19 @@
|
|||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ui/api/browse/{ecosystem}/{name}/{version}/file/{filepath}": {
|
||||
"/api/browse/{ecosystem}/{name}/{version}/file/{filepath}": {
|
||||
"get": {
|
||||
"description": "Streams a single file from the cached artifact. The file path may contain slashes.",
|
||||
"produces": [
|
||||
|
|
@ -358,25 +119,70 @@
|
|||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ui/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion}": {
|
||||
"/api/bulk": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Bulk package lookup by PURL",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "PURLs",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.BulkRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.BulkResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion}": {
|
||||
"get": {
|
||||
"description": "Returns a structured diff for two cached versions.",
|
||||
"produces": [
|
||||
|
|
@ -427,13 +233,207 @@
|
|||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/outdated": {
|
||||
"post": {
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Check outdated packages",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "Packages to check",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.OutdatedRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.OutdatedResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/packages": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "List cached packages",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Ecosystem",
|
||||
"name": "ecosystem",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"enum": [
|
||||
"hits",
|
||||
"name",
|
||||
"size",
|
||||
"cached_at",
|
||||
"ecosystem",
|
||||
"vulns"
|
||||
],
|
||||
"type": "string",
|
||||
"description": "Sort",
|
||||
"name": "sort",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.PackagesListResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/search": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"api"
|
||||
],
|
||||
"summary": "Search cached packages",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Query",
|
||||
"name": "q",
|
||||
"in": "query",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Ecosystem",
|
||||
"name": "ecosystem",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.SearchResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/health": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"text/plain"
|
||||
],
|
||||
"tags": [
|
||||
"meta"
|
||||
],
|
||||
"summary": "Health check",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/stats": {
|
||||
"get": {
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"meta"
|
||||
],
|
||||
"summary": "Cache statistics",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.StatsResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -497,45 +497,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"server.ErrorResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {
|
||||
"type": "string"
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.HealthCheck": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"step": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.HealthResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"checks": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/server.HealthCheck"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.OutdatedPackage": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
|
|||
277
go.mod
277
go.mod
|
|
@ -1,141 +1,135 @@
|
|||
module github.com/git-pkgs/proxy
|
||||
|
||||
go 1.26.0
|
||||
|
||||
toolchain go1.26.6
|
||||
go 1.25.6
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.6.0
|
||||
github.com/CycloneDX/cyclonedx-go v0.11.0
|
||||
github.com/git-pkgs/archives v0.5.1
|
||||
github.com/git-pkgs/cooldown v0.1.1
|
||||
github.com/git-pkgs/enrichment v0.6.5
|
||||
github.com/git-pkgs/integrity v0.1.1
|
||||
github.com/git-pkgs/magic v0.2.0
|
||||
github.com/git-pkgs/purl v0.1.17
|
||||
github.com/git-pkgs/registries v0.8.1
|
||||
github.com/git-pkgs/spdx v0.3.1
|
||||
github.com/git-pkgs/vers v0.3.1
|
||||
github.com/git-pkgs/vulns v0.2.2
|
||||
github.com/go-chi/chi/v5 v5.3.1
|
||||
github.com/CycloneDX/cyclonedx-go v0.10.0
|
||||
github.com/git-pkgs/archives v0.2.3
|
||||
github.com/git-pkgs/enrichment v0.2.2
|
||||
github.com/git-pkgs/purl v0.1.12
|
||||
github.com/git-pkgs/registries v0.5.1
|
||||
github.com/git-pkgs/spdx v0.1.3
|
||||
github.com/git-pkgs/vers v0.2.5
|
||||
github.com/git-pkgs/vulns v0.1.5
|
||||
github.com/go-chi/chi/v5 v5.2.5
|
||||
github.com/jmoiron/sqlx v1.4.0
|
||||
github.com/lib/pq v1.12.3
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2
|
||||
github.com/spdx/tools-golang v0.5.7
|
||||
github.com/swaggo/swag v1.16.6
|
||||
gocloud.dev v0.46.0
|
||||
golang.org/x/sync v0.22.0
|
||||
google.golang.org/protobuf v1.36.12
|
||||
gocloud.dev v0.45.0
|
||||
golang.org/x/sync v0.20.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.56.0
|
||||
modernc.org/sqlite v1.50.0
|
||||
)
|
||||
|
||||
require (
|
||||
4d63.com/gocheckcompilerdirectives v1.4.0 // indirect
|
||||
4d63.com/gocheckcompilerdirectives v1.3.0 // indirect
|
||||
4d63.com/gochecknoglobals v0.2.2 // indirect
|
||||
charm.land/lipgloss/v2 v2.0.6 // indirect
|
||||
cloud.google.com/go/auth v0.21.0 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
codeberg.org/chavacava/garif v0.2.0 // indirect
|
||||
codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect
|
||||
dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect
|
||||
dev.gaijin.team/go/exhaustruct/v5 v5.0.3 // indirect
|
||||
dev.gaijin.team/go/golib v0.8.1 // indirect
|
||||
dev.gaijin.team/go/golib v0.6.0 // indirect
|
||||
github.com/4meepo/tagalign v1.4.3 // indirect
|
||||
github.com/Abirdcfly/dupword v0.1.8 // indirect
|
||||
github.com/Abirdcfly/dupword v0.1.7 // indirect
|
||||
github.com/AdminBenni/iota-mixing v1.0.0 // indirect
|
||||
github.com/AlwxSin/noinlineerr v1.0.6 // indirect
|
||||
github.com/Antonboom/errname v1.1.2 // indirect
|
||||
github.com/Antonboom/nilnil v1.1.2 // indirect
|
||||
github.com/AlwxSin/noinlineerr v1.0.5 // indirect
|
||||
github.com/Antonboom/errname v1.1.1 // indirect
|
||||
github.com/Antonboom/nilnil v1.1.1 // indirect
|
||||
github.com/Antonboom/testifylint v1.6.4 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect
|
||||
github.com/ClickHouse/clickhouse-go-linter v1.2.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/BurntSushi/toml v1.6.0 // indirect
|
||||
github.com/Djarvur/go-err113 v0.1.1 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.5.0 // indirect
|
||||
github.com/MirrexOne/unqueryvet v1.5.4 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.4.0 // indirect
|
||||
github.com/MirrexOne/unqueryvet v1.5.3 // indirect
|
||||
github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect
|
||||
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||
github.com/alecthomas/chroma/v2 v2.27.0 // indirect
|
||||
github.com/alecthomas/chroma/v2 v2.23.1 // indirect
|
||||
github.com/alecthomas/go-check-sumtype v0.3.1 // indirect
|
||||
github.com/alexkohler/nakedret/v2 v2.0.6 // indirect
|
||||
github.com/alexkohler/prealloc v1.1.0 // indirect
|
||||
github.com/alexkohler/prealloc v1.0.2 // indirect
|
||||
github.com/alfatraining/structtag v1.0.0 // indirect
|
||||
github.com/alingse/asasalint v0.0.11 // indirect
|
||||
github.com/alingse/nilnesserr v0.2.0 // indirect
|
||||
github.com/anchore/go-struct-converter v0.1.0 // indirect
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
|
||||
github.com/ashanbrown/forbidigo/v2 v2.3.1 // indirect
|
||||
github.com/ashanbrown/makezero/v2 v2.2.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 // indirect
|
||||
github.com/aws/smithy-go v1.26.0 // indirect
|
||||
github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect
|
||||
github.com/ashanbrown/makezero/v2 v2.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bkielbasa/cyclop v1.2.3 // indirect
|
||||
github.com/blizzy78/varnamelen v0.8.0 // indirect
|
||||
github.com/bombsimon/wsl/v4 v4.7.0 // indirect
|
||||
github.com/bombsimon/wsl/v5 v5.9.0 // indirect
|
||||
github.com/bombsimon/wsl/v5 v5.6.0 // indirect
|
||||
github.com/breml/bidichk v0.3.3 // indirect
|
||||
github.com/breml/errchkjson v0.4.1 // indirect
|
||||
github.com/butuzov/ireturn v0.4.1 // indirect
|
||||
github.com/butuzov/mirror v1.3.3 // indirect
|
||||
github.com/butuzov/ireturn v0.4.0 // indirect
|
||||
github.com/butuzov/mirror v1.3.0 // indirect
|
||||
github.com/catenacyber/perfsprint v0.10.1 // indirect
|
||||
github.com/ccojocar/zxcvbn-go v1.0.4 // indirect
|
||||
github.com/cenk/backoff v2.2.1+incompatible // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/charithe/durationcheck v0.0.11 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.4.3 // indirect
|
||||
github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.11.8 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/charmbracelet/x/termios v0.1.1 // indirect
|
||||
github.com/charmbracelet/x/windows v0.2.2 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
|
||||
github.com/charmbracelet/lipgloss v1.1.0 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.10.1 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/ckaznocha/intrange v0.3.1 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.11.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
|
||||
github.com/curioswitch/go-reassign v0.3.0 // indirect
|
||||
github.com/daixiang0/gci v0.13.7 // indirect
|
||||
github.com/dave/dst v0.27.3 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/denis-tingaikin/go-header v0.5.0 // indirect
|
||||
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ecosyste-ms/ecosystems-go v0.4.0 // indirect
|
||||
github.com/ecosyste-ms/ecosystems-go v0.1.1 // indirect
|
||||
github.com/ettle/strcase v0.2.0 // indirect
|
||||
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
|
||||
github.com/fatih/color v1.19.0 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/fatih/structtag v1.2.0 // indirect
|
||||
github.com/firefart/nonamedreturns v1.0.8 // indirect
|
||||
github.com/firefart/nonamedreturns v1.0.6 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/fzipp/gocyclo v0.6.0 // indirect
|
||||
github.com/ghostiam/protogetter v0.3.21 // indirect
|
||||
github.com/ghostiam/protogetter v0.3.20 // indirect
|
||||
github.com/git-pkgs/packageurl-go v0.3.1 // indirect
|
||||
github.com/git-pkgs/pom v0.1.5 // indirect
|
||||
github.com/github/go-spdx/v2 v2.7.0 // indirect
|
||||
github.com/go-critic/go-critic v0.14.4 // indirect
|
||||
github.com/git-pkgs/pom v0.1.4 // indirect
|
||||
github.com/github/go-spdx/v2 v2.6.0 // indirect
|
||||
github.com/go-critic/go-critic v0.14.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
|
|
@ -154,110 +148,109 @@ require (
|
|||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
github.com/godoc-lint/godoc-lint v0.11.2 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
|
||||
github.com/golangci/asciicheck v0.5.0 // indirect
|
||||
github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202 // indirect
|
||||
github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect
|
||||
github.com/golangci/go-printf-func-name v0.1.1 // indirect
|
||||
github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792 // indirect
|
||||
github.com/golangci/golangci-lint/v2 v2.13.1 // indirect
|
||||
github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d // indirect
|
||||
github.com/golangci/golangci-lint/v2 v2.10.1 // indirect
|
||||
github.com/golangci/golines v0.15.0 // indirect
|
||||
github.com/golangci/misspell v0.8.0 // indirect
|
||||
github.com/golangci/plugin-module-register v0.1.2 // indirect
|
||||
github.com/golangci/revgrep v0.8.0 // indirect
|
||||
github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba // indirect
|
||||
github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect
|
||||
github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.23.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.12 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/gordonklaus/ineffassign v0.2.0 // indirect
|
||||
github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
|
||||
github.com/gostaticanalysis/comment v1.5.0 // indirect
|
||||
github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect
|
||||
github.com/gostaticanalysis/nilerr v0.1.2 // indirect
|
||||
github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect
|
||||
github.com/hashicorp/go-version v1.9.0 // indirect
|
||||
github.com/hashicorp/go-version v1.8.0 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/hexops/gotextdiff v1.0.3 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jgautheron/goconst v1.11.0 // indirect
|
||||
github.com/jgautheron/goconst v1.8.2 // indirect
|
||||
github.com/jingyugao/rowserrcheck v1.1.1 // indirect
|
||||
github.com/jjti/go-spancheck v0.6.5 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/julz/importas v0.2.0 // indirect
|
||||
github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect
|
||||
github.com/kisielk/errcheck v1.20.0 // indirect
|
||||
github.com/kisielk/errcheck v1.9.0 // indirect
|
||||
github.com/kkHAIKE/contextcheck v1.1.6 // indirect
|
||||
github.com/kulti/thelper v0.7.1 // indirect
|
||||
github.com/kunwardeep/paralleltest v1.0.15 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lasiar/canonicalheader v1.1.2 // indirect
|
||||
github.com/ldez/exptostd v0.4.5 // indirect
|
||||
github.com/ldez/gomoddirectives v0.9.0 // indirect
|
||||
github.com/ldez/gomoddirectives v0.8.0 // indirect
|
||||
github.com/ldez/grignotin v0.10.1 // indirect
|
||||
github.com/ldez/structtags v0.6.1 // indirect
|
||||
github.com/ldez/tagliatelle v0.7.2 // indirect
|
||||
github.com/ldez/usetesting v0.5.0 // indirect
|
||||
github.com/leonklingele/grouper v1.1.2 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.1 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/macabu/inamedparam v0.2.0 // indirect
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect
|
||||
github.com/manuelarte/funcorder v0.6.0 // indirect
|
||||
github.com/manuelarte/funcorder v0.5.0 // indirect
|
||||
github.com/maratori/testableexamples v1.0.1 // indirect
|
||||
github.com/maratori/testpackage v1.1.2 // indirect
|
||||
github.com/matoous/godox v1.1.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.24 // indirect
|
||||
github.com/mgechev/revive v1.15.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mgechev/revive v1.14.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/moricho/tparallel v0.3.2 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nakabonne/nestif v0.3.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/nishanths/exhaustive v0.12.0 // indirect
|
||||
github.com/nishanths/predeclared v0.2.2 // indirect
|
||||
github.com/nunnatsa/ginkgolinter v0.24.0 // indirect
|
||||
github.com/oapi-codegen/nullable v1.2.0 // indirect
|
||||
github.com/oapi-codegen/runtime v1.6.0 // indirect
|
||||
github.com/nunnatsa/ginkgolinter v0.23.0 // indirect
|
||||
github.com/oapi-codegen/runtime v1.2.0 // indirect
|
||||
github.com/package-url/packageurl-go v0.1.6 // indirect
|
||||
github.com/pandatix/go-cvss v0.6.2 // indirect
|
||||
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/quasilyte/go-ruleguard v0.4.5 // indirect
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect
|
||||
github.com/quasilyte/gogrep v0.5.0 // indirect
|
||||
github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
|
||||
github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
|
||||
github.com/raeperd/recvcheck v0.3.0 // indirect
|
||||
github.com/raeperd/recvcheck v0.2.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rogpeppe/go-internal v1.16.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect
|
||||
github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/ryancurrah/gomodguard v1.4.1 // indirect
|
||||
github.com/ryancurrah/gomodguard/v2 v2.1.3 // indirect
|
||||
github.com/ryanrolds/sqlclosecheck v0.6.0 // indirect
|
||||
github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
|
||||
github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/sashamelentyev/interfacebloat v1.1.0 // indirect
|
||||
github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect
|
||||
github.com/securego/gosec/v2 v2.28.0 // indirect
|
||||
github.com/sirupsen/logrus v1.10.1 // indirect
|
||||
github.com/securego/gosec/v2 v2.23.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/sivchari/containedctx v1.0.3 // indirect
|
||||
github.com/sonatard/noctx v0.5.1 // indirect
|
||||
github.com/sourcegraph/go-diff v0.8.0 // indirect
|
||||
github.com/sonatard/noctx v0.4.0 // indirect
|
||||
github.com/sourcegraph/go-diff v0.7.0 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/cobra v1.10.2 // indirect
|
||||
|
|
@ -266,20 +259,20 @@ require (
|
|||
github.com/spf13/viper v1.12.0 // indirect
|
||||
github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
|
||||
github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect
|
||||
github.com/stretchr/objx v0.5.3 // indirect
|
||||
github.com/stretchr/testify v1.12.1 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/subosito/gotenv v1.4.1 // indirect
|
||||
github.com/tetafro/godot v1.5.6 // indirect
|
||||
github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4 // indirect
|
||||
github.com/tetafro/godot v1.5.4 // indirect
|
||||
github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 // indirect
|
||||
github.com/timonwong/loggercheck v0.11.0 // indirect
|
||||
github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect
|
||||
github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.16 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/ultraware/funlen v0.2.0 // indirect
|
||||
github.com/ultraware/whitespace v0.2.0 // indirect
|
||||
github.com/urfave/cli/v2 v2.3.0 // indirect
|
||||
github.com/uudashr/gocognit v1.2.1 // indirect
|
||||
github.com/uudashr/iface v1.5.0 // indirect
|
||||
github.com/uudashr/gocognit v1.2.0 // indirect
|
||||
github.com/uudashr/iface v1.4.1 // indirect
|
||||
github.com/xen0n/gosmopolitan v1.3.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yagipy/maintidx v1.0.0 // indirect
|
||||
|
|
@ -287,40 +280,40 @@ require (
|
|||
github.com/ykadowak/zerologlint v0.1.5 // indirect
|
||||
gitlab.com/bosi/decorder v0.4.2 // indirect
|
||||
go-simpler.org/musttag v0.14.0 // indirect
|
||||
go-simpler.org/sloglint v0.12.0 // indirect
|
||||
go-simpler.org/sloglint v0.11.1 // indirect
|
||||
go.augendre.info/arangolint v0.4.0 // indirect
|
||||
go.augendre.info/fatcontext v0.10.0 // indirect
|
||||
go.augendre.info/fatcontext v0.9.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f // indirect
|
||||
golang.org/x/mod v0.40.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
golang.org/x/tools v0.49.0 // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/net v0.51.0 // indirect
|
||||
golang.org/x/oauth2 v0.35.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.288.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect
|
||||
google.golang.org/grpc v1.82.1 // indirect
|
||||
google.golang.org/api v0.269.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
|
||||
google.golang.org/grpc v1.79.1 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
honnef.co/go/tools v0.8.0 // indirect
|
||||
modernc.org/libc v1.74.4 // indirect
|
||||
honnef.co/go/tools v0.7.0 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
mvdan.cc/gofumpt v0.11.0 // indirect
|
||||
mvdan.cc/unparam v0.0.0-20260818115549-3f964bcb5673 // indirect
|
||||
mvdan.cc/gofumpt v0.9.2 // indirect
|
||||
mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,109 +0,0 @@
|
|||
// Package accesslog writes proxy activity as JSON Lines.
|
||||
package accesslog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
accessLogFileMode os.FileMode = 0o600
|
||||
|
||||
// EventRequest identifies the response sent by the proxy to a client.
|
||||
EventRequest = "request"
|
||||
// EventUpstream identifies one HTTP exchange with an upstream service.
|
||||
EventUpstream = "upstream"
|
||||
)
|
||||
|
||||
type requestIDKey struct{}
|
||||
|
||||
// Entry is one proxy activity record.
|
||||
type Entry struct {
|
||||
Time time.Time `json:"time"`
|
||||
Event string `json:"event"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
Method string `json:"method"`
|
||||
Path string `json:"path,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
StatusCode int `json:"status_code,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
RemoteAddr string `json:"remote_addr,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Logger appends complete JSON objects to a file, one per line.
|
||||
type Logger struct {
|
||||
mu sync.Mutex
|
||||
file *os.File
|
||||
encoder *json.Encoder
|
||||
}
|
||||
|
||||
// Open opens path for append, creating it with owner-only permissions when needed.
|
||||
func Open(path string) (*Logger, error) {
|
||||
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, accessLogFileMode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening access log: %w", err)
|
||||
}
|
||||
|
||||
return &Logger{
|
||||
file: file,
|
||||
encoder: json.NewEncoder(file),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Write appends an entry to the log.
|
||||
func (l *Logger) Write(entry Entry) error {
|
||||
if entry.Time.IsZero() {
|
||||
entry.Time = time.Now().UTC()
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if err := l.encoder.Encode(entry); err != nil {
|
||||
return fmt.Errorf("writing access log: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the log file after any active writer finishes.
|
||||
func (l *Logger) Close() error {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if err := l.file.Close(); err != nil {
|
||||
return fmt.Errorf("closing access log: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WithRequestID stores a proxy request ID in ctx.
|
||||
func WithRequestID(ctx context.Context, requestID string) context.Context {
|
||||
return context.WithValue(ctx, requestIDKey{}, requestID)
|
||||
}
|
||||
|
||||
// RequestID returns the proxy request ID stored in ctx.
|
||||
func RequestID(ctx context.Context) string {
|
||||
requestID, _ := ctx.Value(requestIDKey{}).(string)
|
||||
return requestID
|
||||
}
|
||||
|
||||
// URLWithoutSecrets returns a URL without user information, query values, or fragments.
|
||||
func URLWithoutSecrets(value *url.URL) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
clean := *value
|
||||
clean.User = nil
|
||||
clean.RawQuery = ""
|
||||
clean.ForceQuery = false
|
||||
clean.Fragment = ""
|
||||
clean.RawFragment = ""
|
||||
return clean.String()
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
package accesslog
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoggerWritesJSONLines(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "access.jsonl")
|
||||
logger, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const entries = 20
|
||||
var wg sync.WaitGroup
|
||||
for range entries {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := logger.Write(Entry{
|
||||
Event: EventUpstream,
|
||||
RequestID: "request-id",
|
||||
Method: "GET",
|
||||
URL: "https://registry.example/packages/example",
|
||||
StatusCode: 429,
|
||||
}); err != nil {
|
||||
t.Errorf("Write: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if err := logger.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
count := 0
|
||||
for scanner.Scan() {
|
||||
var entry Entry
|
||||
if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
|
||||
t.Fatalf("line %d is not JSON: %v", count+1, err)
|
||||
}
|
||||
if entry.Time.IsZero() {
|
||||
t.Errorf("line %d has no time", count+1)
|
||||
}
|
||||
if entry.StatusCode != 429 {
|
||||
t.Errorf("line %d status_code = %d, want 429", count+1, entry.StatusCode)
|
||||
}
|
||||
count++
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != entries {
|
||||
t.Errorf("lines = %d, want %d", count, entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestID(t *testing.T) {
|
||||
ctx := WithRequestID(context.Background(), "abc-123")
|
||||
if got := RequestID(ctx); got != "abc-123" {
|
||||
t.Errorf("RequestID = %q, want %q", got, "abc-123")
|
||||
}
|
||||
}
|
||||
|
||||
func TestURLWithoutSecrets(t *testing.T) {
|
||||
value, err := url.Parse("https://user:password@registry.example/package.tgz?token=secret#fragment")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := URLWithoutSecrets(value)
|
||||
want := "https://registry.example/package.tgz"
|
||||
if got != want {
|
||||
t.Errorf("URLWithoutSecrets = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
|
@ -54,12 +54,10 @@ import (
|
|||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
|
|
@ -68,20 +66,11 @@ type Config struct {
|
|||
// Listen is the address to listen on (e.g., ":8080", "127.0.0.1:8080").
|
||||
Listen string `json:"listen" yaml:"listen"`
|
||||
|
||||
// BaseURL is the public URL where package endpoints are reachable.
|
||||
// Used for rewriting package metadata URLs and shown to humans on the
|
||||
// install guide so they know what to point their package manager at.
|
||||
// BaseURL is the public URL where this proxy is accessible.
|
||||
// Used for rewriting package metadata URLs.
|
||||
// Example: "https://proxy.example.com" or "http://localhost:8080"
|
||||
BaseURL string `json:"base_url" yaml:"base_url"`
|
||||
|
||||
// UIBaseURL is the public URL where the web UI is reachable. Defaults to
|
||||
// BaseURL when unset. Set this separately when the UI is served on a
|
||||
// different hostname than the package endpoints — for example, the UI on a
|
||||
// public domain behind auth while build machines hit a Docker network alias
|
||||
// for the package endpoints.
|
||||
// Example: "https://proxy.example.com/ui"
|
||||
UIBaseURL string `json:"ui_base_url" yaml:"ui_base_url"`
|
||||
|
||||
// Storage configures artifact storage.
|
||||
Storage StorageConfig `json:"storage" yaml:"storage"`
|
||||
|
||||
|
|
@ -91,9 +80,6 @@ type Config struct {
|
|||
// Log configures logging.
|
||||
Log LogConfig `json:"log" yaml:"log"`
|
||||
|
||||
// AccessLog configures the JSONL activity log.
|
||||
AccessLog AccessLogConfig `json:"access_log" yaml:"access_log"`
|
||||
|
||||
// Upstream configures upstream registry URLs (optional overrides).
|
||||
Upstream UpstreamConfig `json:"upstream" yaml:"upstream"`
|
||||
|
||||
|
|
@ -110,26 +96,9 @@ type Config struct {
|
|||
// Default: "5m". Set to "0" to always revalidate.
|
||||
MetadataTTL string `json:"metadata_ttl" yaml:"metadata_ttl"`
|
||||
|
||||
// MetadataMaxSize is the maximum size of an upstream metadata response
|
||||
// the proxy will buffer (e.g. "100MB", "250MB"). Responses over this
|
||||
// size return ErrMetadataTooLarge. Default: "100MB".
|
||||
MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
|
||||
|
||||
// HTTPTimeout is the timeout for individual upstream HTTP requests made
|
||||
// by protocol handlers (metadata fetches, pass-through file requests).
|
||||
// Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s".
|
||||
// Set to "0" to disable the timeout entirely.
|
||||
HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"`
|
||||
|
||||
// MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP.
|
||||
// Disabled by default to prevent unauthenticated users from triggering downloads.
|
||||
MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
|
||||
|
||||
// Gradle configures Gradle HttpBuildCache behavior.
|
||||
Gradle GradleConfig `json:"gradle" yaml:"gradle"`
|
||||
|
||||
// Health configures the /health endpoint behavior.
|
||||
Health HealthConfig `json:"health" yaml:"health"`
|
||||
}
|
||||
|
||||
// CooldownConfig configures version cooldown periods.
|
||||
|
|
@ -142,38 +111,9 @@ type CooldownConfig struct {
|
|||
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
|
||||
|
||||
// Packages overrides the cooldown for specific packages (keyed by PURL).
|
||||
// Valid PURL keys are normalized to canonical form before use.
|
||||
Packages map[string]string `json:"packages" yaml:"packages"`
|
||||
}
|
||||
|
||||
// NormalizedPackages returns a copy of the package overrides with valid PURL
|
||||
// keys in canonical form. An explicitly canonical key wins over an equivalent
|
||||
// noncanonical key, and invalid keys are preserved unchanged.
|
||||
func (c *CooldownConfig) NormalizedPackages() map[string]string {
|
||||
if c == nil || c.Packages == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(c.Packages))
|
||||
for key := range c.Packages {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
normalized := make(map[string]string, len(c.Packages))
|
||||
for _, key := range keys {
|
||||
canonical := key
|
||||
if parsed, err := purl.Parse(key); err == nil {
|
||||
canonical = parsed.String()
|
||||
}
|
||||
if _, exists := normalized[canonical]; exists && key != canonical {
|
||||
continue
|
||||
}
|
||||
normalized[canonical] = c.Packages[key]
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
// StorageConfig configures artifact storage.
|
||||
type StorageConfig struct {
|
||||
// URL is the storage backend URL.
|
||||
|
|
@ -211,42 +151,6 @@ type StorageConfig struct {
|
|||
DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"`
|
||||
}
|
||||
|
||||
// GradleConfig configures Gradle-specific features.
|
||||
type GradleConfig struct {
|
||||
// BuildCache configures the /gradle HttpBuildCache endpoint.
|
||||
BuildCache GradleBuildCacheConfig `json:"build_cache" yaml:"build_cache"`
|
||||
}
|
||||
|
||||
// GradleBuildCacheConfig configures Gradle HttpBuildCache safeguards.
|
||||
type GradleBuildCacheConfig struct {
|
||||
// ReadOnly disables PUT uploads and keeps cache reads (GET/HEAD) enabled.
|
||||
ReadOnly bool `json:"read_only" yaml:"read_only"`
|
||||
|
||||
// MaxUploadSize caps a single PUT body size (e.g., "100MB"). Must be > 0.
|
||||
// Default: "100MB".
|
||||
MaxUploadSize string `json:"max_upload_size" yaml:"max_upload_size"`
|
||||
|
||||
// MaxAge evicts entries older than this duration (e.g., "24h", "7d").
|
||||
// Empty or "0" disables age-based eviction.
|
||||
MaxAge string `json:"max_age" yaml:"max_age"`
|
||||
|
||||
// MaxSize evicts oldest entries until total Gradle cache size is <= MaxSize.
|
||||
// Empty or "0" disables size-based eviction.
|
||||
MaxSize string `json:"max_size" yaml:"max_size"`
|
||||
|
||||
// SweepInterval controls periodic eviction frequency.
|
||||
// Default: "10m".
|
||||
SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"`
|
||||
}
|
||||
|
||||
// HealthConfig configures the /health endpoint.
|
||||
type HealthConfig struct {
|
||||
// StorageProbeInterval is the minimum time between storage backend probes.
|
||||
// Uses Go duration syntax (e.g. "30s", "1m"). Default: "30s".
|
||||
// Set to "0" to probe on every /health request (useful for low-traffic deployments).
|
||||
StorageProbeInterval string `json:"storage_probe_interval" yaml:"storage_probe_interval"`
|
||||
}
|
||||
|
||||
// DatabaseConfig configures the cache database.
|
||||
type DatabaseConfig struct {
|
||||
// Driver is the database driver: "sqlite" or "postgres".
|
||||
|
|
@ -259,21 +163,6 @@ type DatabaseConfig struct {
|
|||
URL string `json:"url" yaml:"url"`
|
||||
}
|
||||
|
||||
// String returns a human-readable description of the configured database
|
||||
// suitable for logging. For postgres the password in the connection URL is
|
||||
// redacted; if the URL cannot be parsed only the driver name is returned to
|
||||
// avoid leaking credentials.
|
||||
func (d DatabaseConfig) String() string {
|
||||
if d.Driver == "postgres" {
|
||||
u, err := url.Parse(d.URL)
|
||||
if err != nil || u.Host == "" {
|
||||
return "postgres"
|
||||
}
|
||||
return u.Redacted()
|
||||
}
|
||||
return d.Path
|
||||
}
|
||||
|
||||
// LogConfig configures logging.
|
||||
type LogConfig struct {
|
||||
// Level is the minimum log level: "debug", "info", "warn", "error".
|
||||
|
|
@ -283,12 +172,6 @@ type LogConfig struct {
|
|||
Format string `json:"format" yaml:"format"`
|
||||
}
|
||||
|
||||
// AccessLogConfig configures the JSONL activity log.
|
||||
type AccessLogConfig struct {
|
||||
// Path is the file to append activity records to. Empty disables the access log.
|
||||
Path string `json:"path" yaml:"path"`
|
||||
}
|
||||
|
||||
// UpstreamConfig configures upstream registry URLs and authentication.
|
||||
// Leave empty to use defaults.
|
||||
type UpstreamConfig struct {
|
||||
|
|
@ -296,15 +179,6 @@ type UpstreamConfig struct {
|
|||
// Default: https://registry.npmjs.org
|
||||
NPM string `json:"npm" yaml:"npm"`
|
||||
|
||||
// Maven is the upstream Maven repository URL.
|
||||
// Default: https://repo1.maven.org/maven2
|
||||
Maven string `json:"maven" yaml:"maven"`
|
||||
|
||||
// GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL.
|
||||
// Used to resolve Gradle plugin marker artifacts.
|
||||
// Default: https://plugins.gradle.org/m2
|
||||
GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"`
|
||||
|
||||
// Cargo is the upstream cargo index URL.
|
||||
// Default: https://index.crates.io
|
||||
Cargo string `json:"cargo" yaml:"cargo"`
|
||||
|
|
@ -313,45 +187,24 @@ type UpstreamConfig struct {
|
|||
// Default: https://static.crates.io/crates
|
||||
CargoDownload string `json:"cargo_download" yaml:"cargo_download"`
|
||||
|
||||
// Debian is the upstream APT repository base URL.
|
||||
// Example: http://archive.ubuntu.com/ubuntu would get Ubuntu.
|
||||
// Default: http://deb.debian.org/debian
|
||||
Debian string `json:"debian" yaml:"debian"`
|
||||
|
||||
// Helm maps repository names to HTTP Helm chart repository URLs.
|
||||
// Requests use /helm/{name}/index.yaml and chart URLs in the index are
|
||||
// rewritten to the same named proxy endpoint.
|
||||
Helm map[string]string `json:"helm" yaml:"helm"`
|
||||
|
||||
// OCI maps names to OCI registry URLs. Requests to a named registry use
|
||||
// the repository prefix upstream/{name}/, for example
|
||||
// oci://proxy.example.com/upstream/ghcr/owner/chart.
|
||||
OCI map[string]string `json:"oci" yaml:"oci"`
|
||||
|
||||
// Auth configures authentication for upstream registries.
|
||||
// Keys are absolute URL scopes matched by scheme, host, effective port,
|
||||
// and path-segment prefix.
|
||||
// Keys are URL prefixes that are matched against request URLs.
|
||||
// Example: "https://npm.pkg.github.com" matches all requests to that host.
|
||||
Auth map[string]AuthConfig `json:"auth" yaml:"auth"`
|
||||
}
|
||||
|
||||
// AuthForURL returns the auth config that matches the given URL.
|
||||
// The longest matching URL scope wins.
|
||||
// Matches are based on URL prefix - the longest matching prefix wins.
|
||||
func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig {
|
||||
if u.Auth == nil {
|
||||
return nil
|
||||
}
|
||||
target, err := parseAuthURL(url)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var bestMatch *AuthConfig
|
||||
var bestLen int
|
||||
|
||||
for pattern, auth := range u.Auth {
|
||||
configured, err := parseAuthURL(pattern)
|
||||
if err == nil && authURLMatches(configured, target) && len(pattern) > bestLen {
|
||||
if strings.HasPrefix(url, pattern) && len(pattern) > bestLen {
|
||||
a := auth // copy to avoid loop variable capture
|
||||
bestMatch = &a
|
||||
bestLen = len(pattern)
|
||||
|
|
@ -361,73 +214,6 @@ func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig {
|
|||
return bestMatch
|
||||
}
|
||||
|
||||
// Validate checks upstream authentication URL scopes.
|
||||
func (u *UpstreamConfig) Validate() error {
|
||||
for pattern := range u.Auth {
|
||||
if _, err := parseAuthURL(pattern); err != nil {
|
||||
return fmt.Errorf("invalid upstream.auth URL %q: %w", pattern, err)
|
||||
}
|
||||
}
|
||||
if err := validateNamedUpstreams("upstream.helm", u.Helm); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateNamedUpstreams(field string, upstreams map[string]string) error {
|
||||
for name, upstreamURL := range upstreams {
|
||||
if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\\`) {
|
||||
return fmt.Errorf("invalid %s name %q", field, name)
|
||||
}
|
||||
if err := validateAbsoluteURL(field+"."+name, upstreamURL); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseAuthURL(value string) (*url.URL, error) {
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" {
|
||||
return nil, fmt.Errorf("invalid authentication URL")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func authURLMatches(configured, target *url.URL) bool {
|
||||
if !strings.EqualFold(configured.Scheme, target.Scheme) ||
|
||||
!strings.EqualFold(configured.Hostname(), target.Hostname()) ||
|
||||
authURLPort(configured) != authURLPort(target) {
|
||||
return false
|
||||
}
|
||||
if configured.RawQuery != "" && configured.RawQuery != target.RawQuery {
|
||||
return false
|
||||
}
|
||||
|
||||
configuredPath := strings.TrimSuffix(configured.EscapedPath(), "/")
|
||||
if configuredPath == "" {
|
||||
return true
|
||||
}
|
||||
targetPath := strings.TrimSuffix(target.EscapedPath(), "/")
|
||||
return targetPath == configuredPath || strings.HasPrefix(targetPath, configuredPath+"/")
|
||||
}
|
||||
|
||||
func authURLPort(value *url.URL) string {
|
||||
if port := value.Port(); port != "" {
|
||||
return port
|
||||
}
|
||||
if strings.EqualFold(value.Scheme, "https") {
|
||||
return "443"
|
||||
}
|
||||
if strings.EqualFold(value.Scheme, "http") {
|
||||
return "80"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// AuthConfig configures authentication for an upstream registry.
|
||||
type AuthConfig struct {
|
||||
// Type is the authentication type: "bearer", "basic", or "header".
|
||||
|
|
@ -471,20 +257,8 @@ func Default() *Config {
|
|||
},
|
||||
Upstream: UpstreamConfig{
|
||||
NPM: "https://registry.npmjs.org",
|
||||
Maven: "https://repo1.maven.org/maven2",
|
||||
GradlePluginPortal: "https://plugins.gradle.org/m2",
|
||||
Cargo: "https://index.crates.io",
|
||||
CargoDownload: "https://static.crates.io/crates",
|
||||
Debian: "http://deb.debian.org/debian",
|
||||
},
|
||||
Gradle: GradleConfig{
|
||||
BuildCache: GradleBuildCacheConfig{
|
||||
ReadOnly: false,
|
||||
MaxUploadSize: defaultGradleMaxUploadSizeStr,
|
||||
MaxAge: "168h",
|
||||
MaxSize: "",
|
||||
SweepInterval: defaultGradleSweepIntervalStr,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -520,74 +294,67 @@ func Load(path string) (*Config, error) {
|
|||
return cfg, nil
|
||||
}
|
||||
|
||||
// setEnvString sets *dst from the named environment variable, leaving it
|
||||
// untouched if the variable is unset or empty.
|
||||
func setEnvString(dst *string, key string) {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
*dst = v
|
||||
}
|
||||
}
|
||||
|
||||
// setEnvBool is setEnvString for boolean fields, parsed via envBool.
|
||||
func setEnvBool(dst *bool, key string) {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
*dst = envBool(v)
|
||||
}
|
||||
}
|
||||
|
||||
// LoadFromEnv applies environment variable overrides to a Config.
|
||||
// Environment variables use the PROXY_ prefix:
|
||||
// - PROXY_LISTEN
|
||||
// - PROXY_BASE_URL
|
||||
// - PROXY_UI_URL
|
||||
// - PROXY_STORAGE_PATH
|
||||
// - PROXY_STORAGE_MAX_SIZE
|
||||
// - PROXY_DATABASE_PATH
|
||||
// - PROXY_LOG_LEVEL
|
||||
// - PROXY_LOG_FORMAT
|
||||
// - PROXY_ACCESS_LOG_PATH
|
||||
// - PROXY_HEALTH_STORAGE_PROBE_INTERVAL
|
||||
func (c *Config) LoadFromEnv() {
|
||||
setEnvString(&c.Listen, "PROXY_LISTEN")
|
||||
setEnvString(&c.BaseURL, "PROXY_BASE_URL")
|
||||
setEnvString(&c.UIBaseURL, "PROXY_UI_URL")
|
||||
setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL")
|
||||
setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH")
|
||||
setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE")
|
||||
setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE")
|
||||
setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL")
|
||||
setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL")
|
||||
setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER")
|
||||
setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH")
|
||||
setEnvString(&c.Database.URL, "PROXY_DATABASE_URL")
|
||||
setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL")
|
||||
setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT")
|
||||
setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH")
|
||||
setEnvString(&c.Upstream.Maven, "PROXY_UPSTREAM_MAVEN")
|
||||
setEnvString(&c.Upstream.GradlePluginPortal, "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL")
|
||||
setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN")
|
||||
setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT")
|
||||
setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA")
|
||||
setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API")
|
||||
setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL")
|
||||
setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE")
|
||||
setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT")
|
||||
setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY")
|
||||
setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE")
|
||||
setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE")
|
||||
setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE")
|
||||
setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL")
|
||||
setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL")
|
||||
if v := os.Getenv("PROXY_LISTEN"); v != "" {
|
||||
c.Listen = v
|
||||
}
|
||||
|
||||
// validateAbsoluteURL returns an error if value is not a parseable URL with
|
||||
// both a scheme and host. fieldName is used in the error message.
|
||||
func validateAbsoluteURL(fieldName, value string) error {
|
||||
u, err := url.Parse(value)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return fmt.Errorf("invalid %s %q: must be an absolute URL", fieldName, value)
|
||||
if v := os.Getenv("PROXY_BASE_URL"); v != "" {
|
||||
c.BaseURL = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_STORAGE_URL"); v != "" {
|
||||
c.Storage.URL = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_STORAGE_PATH"); v != "" {
|
||||
c.Storage.Path = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_STORAGE_MAX_SIZE"); v != "" {
|
||||
c.Storage.MaxSize = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE"); v != "" {
|
||||
c.Storage.DirectServe = envBool(v)
|
||||
}
|
||||
if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_TTL"); v != "" {
|
||||
c.Storage.DirectServeTTL = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL"); v != "" {
|
||||
c.Storage.DirectServeBaseURL = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_DATABASE_DRIVER"); v != "" {
|
||||
c.Database.Driver = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_DATABASE_PATH"); v != "" {
|
||||
c.Database.Path = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_DATABASE_URL"); v != "" {
|
||||
c.Database.URL = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_LOG_LEVEL"); v != "" {
|
||||
c.Log.Level = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_LOG_FORMAT"); v != "" {
|
||||
c.Log.Format = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_COOLDOWN_DEFAULT"); v != "" {
|
||||
c.Cooldown.Default = v
|
||||
}
|
||||
if v := os.Getenv("PROXY_CACHE_METADATA"); v != "" {
|
||||
c.CacheMetadata = envBool(v)
|
||||
}
|
||||
if v := os.Getenv("PROXY_MIRROR_API"); v != "" {
|
||||
c.MirrorAPI = envBool(v)
|
||||
}
|
||||
if v := os.Getenv("PROXY_METADATA_TTL"); v != "" {
|
||||
c.MetadataTTL = v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate checks the configuration for errors.
|
||||
|
|
@ -598,11 +365,6 @@ func (c *Config) Validate() error {
|
|||
if c.BaseURL == "" {
|
||||
return fmt.Errorf("base_url is required")
|
||||
}
|
||||
if c.UIBaseURL == "" {
|
||||
c.UIBaseURL = c.BaseURL
|
||||
} else if err := validateAbsoluteURL("ui_base_url", c.UIBaseURL); err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Storage.URL == "" && c.Storage.Path == "" {
|
||||
return fmt.Errorf("storage.url or storage.path is required")
|
||||
}
|
||||
|
|
@ -651,8 +413,9 @@ func (c *Config) Validate() error {
|
|||
|
||||
// Validate direct serve base URL if specified
|
||||
if c.Storage.DirectServeBaseURL != "" {
|
||||
if err := validateAbsoluteURL("storage.direct_serve_base_url", c.Storage.DirectServeBaseURL); err != nil {
|
||||
return err
|
||||
u, err := url.Parse(c.Storage.DirectServeBaseURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return fmt.Errorf("invalid storage.direct_serve_base_url %q: must be an absolute URL", c.Storage.DirectServeBaseURL)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -663,93 +426,12 @@ func (c *Config) Validate() error {
|
|||
}
|
||||
}
|
||||
|
||||
if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateHTTPTimeout(c.HTTPTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return c.validateComponents()
|
||||
}
|
||||
|
||||
func (c *Config) validateComponents() error {
|
||||
if err := c.Upstream.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := c.Health.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return c.Gradle.BuildCache.Validate()
|
||||
}
|
||||
|
||||
// Validate checks the /health configuration. An unset interval is allowed
|
||||
// (the cache uses its default); explicit values must parse and be non-negative.
|
||||
func (h *HealthConfig) Validate() error {
|
||||
if h.StorageProbeInterval == "" || h.StorageProbeInterval == "0" {
|
||||
return nil
|
||||
}
|
||||
d, err := time.ParseDuration(h.StorageProbeInterval)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid health.storage_probe_interval %q: %w", h.StorageProbeInterval, err)
|
||||
}
|
||||
if d < 0 {
|
||||
return fmt.Errorf("invalid health.storage_probe_interval %q: must be non-negative", h.StorageProbeInterval)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate checks Gradle build cache settings, applying the default upload
|
||||
// size if unset.
|
||||
func (g *GradleBuildCacheConfig) Validate() error {
|
||||
if g.MaxUploadSize == "" {
|
||||
g.MaxUploadSize = defaultGradleMaxUploadSizeStr
|
||||
}
|
||||
uploadSize, err := ParseSize(g.MaxUploadSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid gradle.build_cache.max_upload_size: %w", err)
|
||||
}
|
||||
if uploadSize <= 0 {
|
||||
return fmt.Errorf("invalid gradle.build_cache.max_upload_size %q: must be > 0", g.MaxUploadSize)
|
||||
}
|
||||
|
||||
if g.MaxAge != "" && g.MaxAge != "0" {
|
||||
if _, err := time.ParseDuration(g.MaxAge); err != nil {
|
||||
return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", g.MaxAge, err)
|
||||
}
|
||||
}
|
||||
|
||||
if g.MaxSize != "" {
|
||||
if _, err := ParseSize(g.MaxSize); err != nil {
|
||||
return fmt.Errorf("invalid gradle.build_cache.max_size: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if g.SweepInterval != "" {
|
||||
d, err := time.ParseDuration(g.SweepInterval)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: %w", g.SweepInterval, err)
|
||||
}
|
||||
if d <= 0 {
|
||||
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: must be > 0", g.SweepInterval)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default
|
||||
defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default
|
||||
defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default
|
||||
defaultMetadataMaxSize = 100 << 20
|
||||
defaultGradleBuildCacheMaxUploadSize = 100 << 20
|
||||
defaultGradleBuildCacheSweepInterval = 10 * time.Minute
|
||||
defaultGradleMaxUploadSizeStr = "100MB"
|
||||
defaultGradleSweepIntervalStr = "10m"
|
||||
)
|
||||
|
||||
// ParseMaxSize returns the maximum cache size in bytes.
|
||||
|
|
@ -765,63 +447,6 @@ func (c *Config) ParseMaxSize() int64 {
|
|||
return size
|
||||
}
|
||||
|
||||
func validateHTTPTimeout(s string) error {
|
||||
if s == "" || s == "0" {
|
||||
return nil
|
||||
}
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid http_timeout %q: %w", s, err)
|
||||
}
|
||||
if d < 0 {
|
||||
return fmt.Errorf("invalid http_timeout %q: must be non-negative", s)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateMetadataMaxSize(s string) error {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
size, err := ParseSize(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid metadata_max_size: %w", err)
|
||||
}
|
||||
if size <= 0 {
|
||||
return fmt.Errorf("invalid metadata_max_size %q: must be positive", s)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseMetadataMaxSize returns the maximum metadata response size in bytes.
|
||||
// Returns 100MB if unset or invalid.
|
||||
func (c *Config) ParseMetadataMaxSize() int64 {
|
||||
if c.MetadataMaxSize == "" {
|
||||
return defaultMetadataMaxSize
|
||||
}
|
||||
size, err := ParseSize(c.MetadataMaxSize)
|
||||
if err != nil || size <= 0 {
|
||||
return defaultMetadataMaxSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// ParseHTTPTimeout returns the upstream HTTP client timeout.
|
||||
// Returns 30s if unset, 0 (no timeout) if explicitly set to "0".
|
||||
func (c *Config) ParseHTTPTimeout() time.Duration {
|
||||
if c.HTTPTimeout == "" {
|
||||
return defaultHTTPTimeout
|
||||
}
|
||||
if c.HTTPTimeout == "0" {
|
||||
return 0
|
||||
}
|
||||
d, err := time.ParseDuration(c.HTTPTimeout)
|
||||
if err != nil || d < 0 {
|
||||
return defaultHTTPTimeout
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// ParseMetadataTTL returns the metadata TTL duration.
|
||||
// Returns 5 minutes if unset, 0 if explicitly disabled.
|
||||
func (c *Config) ParseMetadataTTL() time.Duration {
|
||||
|
|
@ -838,58 +463,6 @@ func (c *Config) ParseMetadataTTL() time.Duration {
|
|||
return d
|
||||
}
|
||||
|
||||
// ParseGradleBuildCacheMaxUploadSize returns the max accepted PUT body size.
|
||||
// Defaults to 100MB if unset or invalid.
|
||||
func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 {
|
||||
if c.Gradle.BuildCache.MaxUploadSize == "" {
|
||||
return defaultGradleBuildCacheMaxUploadSize
|
||||
}
|
||||
size, err := ParseSize(c.Gradle.BuildCache.MaxUploadSize)
|
||||
if err != nil || size <= 0 {
|
||||
return defaultGradleBuildCacheMaxUploadSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// ParseGradleBuildCacheMaxAge returns age-based eviction threshold.
|
||||
// Returns 0 when disabled or invalid.
|
||||
func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration {
|
||||
if c.Gradle.BuildCache.MaxAge == "" || c.Gradle.BuildCache.MaxAge == "0" {
|
||||
return 0
|
||||
}
|
||||
d, err := time.ParseDuration(c.Gradle.BuildCache.MaxAge)
|
||||
if err != nil || d <= 0 {
|
||||
return 0
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// ParseGradleBuildCacheMaxSize returns total-size cap in bytes.
|
||||
// Returns 0 when disabled or invalid.
|
||||
func (c *Config) ParseGradleBuildCacheMaxSize() int64 {
|
||||
if c.Gradle.BuildCache.MaxSize == "" || c.Gradle.BuildCache.MaxSize == "0" {
|
||||
return 0
|
||||
}
|
||||
size, err := ParseSize(c.Gradle.BuildCache.MaxSize)
|
||||
if err != nil || size <= 0 {
|
||||
return 0
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// ParseGradleBuildCacheSweepInterval returns eviction sweep cadence.
|
||||
// Defaults to 10m if unset or invalid.
|
||||
func (c *Config) ParseGradleBuildCacheSweepInterval() time.Duration {
|
||||
if c.Gradle.BuildCache.SweepInterval == "" {
|
||||
return defaultGradleBuildCacheSweepInterval
|
||||
}
|
||||
d, err := time.ParseDuration(c.Gradle.BuildCache.SweepInterval)
|
||||
if err != nil || d <= 0 {
|
||||
return defaultGradleBuildCacheSweepInterval
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// ParseDirectServeTTL returns the presigned URL expiry duration.
|
||||
// Returns 15 minutes if unset.
|
||||
func (c *Config) ParseDirectServeTTL() time.Duration {
|
||||
|
|
@ -928,7 +501,8 @@ func ParseSize(s string) (int64, error) {
|
|||
}
|
||||
|
||||
for _, s2 := range suffixes {
|
||||
if numStr, ok := strings.CutSuffix(s, s2.suffix); ok {
|
||||
if strings.HasSuffix(s, s2.suffix) {
|
||||
numStr := strings.TrimSuffix(s, s2.suffix)
|
||||
num, err := strconv.ParseFloat(numStr, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid number %q", numStr)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package config
|
|||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -26,24 +25,6 @@ func TestDefault(t *testing.T) {
|
|||
if cfg.Database.Path == "" {
|
||||
t.Error("Database.Path should not be empty")
|
||||
}
|
||||
if cfg.AccessLog.Path != "" {
|
||||
t.Errorf("AccessLog.Path = %q, want disabled by default", cfg.AccessLog.Path)
|
||||
}
|
||||
if cfg.Gradle.BuildCache.MaxUploadSize != "100MB" {
|
||||
t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "100MB")
|
||||
}
|
||||
if cfg.Gradle.BuildCache.MaxAge != "168h" {
|
||||
t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "168h")
|
||||
}
|
||||
if cfg.Upstream.Maven != "https://repo1.maven.org/maven2" {
|
||||
t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://repo1.maven.org/maven2")
|
||||
}
|
||||
if cfg.Upstream.GradlePluginPortal != "https://plugins.gradle.org/m2" {
|
||||
t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.gradle.org/m2")
|
||||
}
|
||||
if cfg.Upstream.Debian != "http://deb.debian.org/debian" {
|
||||
t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://deb.debian.org/debian")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
|
|
@ -117,41 +98,6 @@ func TestValidate(t *testing.T) {
|
|||
modify: func(c *Config) { c.Storage.MaxSize = "10GB" },
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid gradle upload size",
|
||||
modify: func(c *Config) { c.Gradle.BuildCache.MaxUploadSize = testInvalid },
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "zero gradle upload size",
|
||||
modify: func(c *Config) { c.Gradle.BuildCache.MaxUploadSize = "0" },
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid gradle max age",
|
||||
modify: func(c *Config) { c.Gradle.BuildCache.MaxAge = testInvalid },
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "valid gradle max age",
|
||||
modify: func(c *Config) { c.Gradle.BuildCache.MaxAge = "24h" },
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid gradle max size",
|
||||
modify: func(c *Config) { c.Gradle.BuildCache.MaxSize = testInvalid },
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid gradle sweep interval",
|
||||
modify: func(c *Config) { c.Gradle.BuildCache.SweepInterval = "0" },
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "valid gradle sweep interval",
|
||||
modify: func(c *Config) { c.Gradle.BuildCache.SweepInterval = "30m" },
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
@ -215,8 +161,6 @@ database:
|
|||
log:
|
||||
level: "debug"
|
||||
format: "json"
|
||||
access_log:
|
||||
path: "/var/log/proxy/access.jsonl"
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("writing config file: %v", err)
|
||||
|
|
@ -245,9 +189,6 @@ access_log:
|
|||
if cfg.Log.Format != "json" {
|
||||
t.Errorf("Log.Format = %q, want %q", cfg.Log.Format, "json")
|
||||
}
|
||||
if cfg.AccessLog.Path != "/var/log/proxy/access.jsonl" {
|
||||
t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/var/log/proxy/access.jsonl")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadJSON(t *testing.T) {
|
||||
|
|
@ -280,18 +221,8 @@ func TestLoadFromEnv(t *testing.T) {
|
|||
|
||||
t.Setenv("PROXY_LISTEN", ":9000")
|
||||
t.Setenv("PROXY_BASE_URL", "https://env.example.com")
|
||||
t.Setenv("PROXY_UI_URL", "https://ui.env.example.com/ui")
|
||||
t.Setenv("PROXY_STORAGE_PATH", "/env/cache")
|
||||
t.Setenv("PROXY_LOG_LEVEL", testLevelDebug)
|
||||
t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl")
|
||||
t.Setenv("PROXY_UPSTREAM_MAVEN", "https://maven.example.com/repository/maven-public")
|
||||
t.Setenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", "https://plugins.example.com/m2")
|
||||
t.Setenv("PROXY_UPSTREAM_DEBIAN", "http://archive.ubuntu.com/ubuntu")
|
||||
t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true")
|
||||
t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB")
|
||||
t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h")
|
||||
t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE", "10GB")
|
||||
t.Setenv("PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL", "15m")
|
||||
|
||||
cfg.LoadFromEnv()
|
||||
|
||||
|
|
@ -301,42 +232,12 @@ func TestLoadFromEnv(t *testing.T) {
|
|||
if cfg.BaseURL != "https://env.example.com" {
|
||||
t.Errorf("BaseURL = %q, want %q", cfg.BaseURL, "https://env.example.com")
|
||||
}
|
||||
if cfg.UIBaseURL != "https://ui.env.example.com/ui" {
|
||||
t.Errorf("UIBaseURL = %q, want %q", cfg.UIBaseURL, "https://ui.env.example.com/ui")
|
||||
}
|
||||
if cfg.Storage.Path != "/env/cache" {
|
||||
t.Errorf("Storage.Path = %q, want %q", cfg.Storage.Path, "/env/cache")
|
||||
}
|
||||
if cfg.Log.Level != testLevelDebug {
|
||||
t.Errorf("Log.Level = %q, want %q", cfg.Log.Level, testLevelDebug)
|
||||
}
|
||||
if cfg.AccessLog.Path != "/tmp/proxy-access.jsonl" {
|
||||
t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/tmp/proxy-access.jsonl")
|
||||
}
|
||||
if cfg.Upstream.Maven != "https://maven.example.com/repository/maven-public" {
|
||||
t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://maven.example.com/repository/maven-public")
|
||||
}
|
||||
if cfg.Upstream.GradlePluginPortal != "https://plugins.example.com/m2" {
|
||||
t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.example.com/m2")
|
||||
}
|
||||
if cfg.Upstream.Debian != "http://archive.ubuntu.com/ubuntu" {
|
||||
t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://archive.ubuntu.com/ubuntu")
|
||||
}
|
||||
if !cfg.Gradle.BuildCache.ReadOnly {
|
||||
t.Error("Gradle.BuildCache.ReadOnly = false, want true")
|
||||
}
|
||||
if cfg.Gradle.BuildCache.MaxUploadSize != "32MB" {
|
||||
t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "32MB")
|
||||
}
|
||||
if cfg.Gradle.BuildCache.MaxAge != "12h" {
|
||||
t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "12h")
|
||||
}
|
||||
if cfg.Gradle.BuildCache.MaxSize != "10GB" {
|
||||
t.Errorf("Gradle.BuildCache.MaxSize = %q, want %q", cfg.Gradle.BuildCache.MaxSize, "10GB")
|
||||
}
|
||||
if cfg.Gradle.BuildCache.SweepInterval != "15m" {
|
||||
t.Errorf("Gradle.BuildCache.SweepInterval = %q, want %q", cfg.Gradle.BuildCache.SweepInterval, "15m")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCooldownConfig(t *testing.T) {
|
||||
|
|
@ -383,34 +284,6 @@ cooldown:
|
|||
if cfg.Cooldown.Packages["pkg:npm/@babel/core"] != "14d" {
|
||||
t.Errorf("Cooldown.Packages[@babel/core] = %q, want %q", cfg.Cooldown.Packages["pkg:npm/@babel/core"], "14d")
|
||||
}
|
||||
if got := cfg.Cooldown.NormalizedPackages()["pkg:npm/%40babel/core"]; got != "14d" {
|
||||
t.Errorf("normalized Cooldown.Packages[@babel/core] = %q, want %q", got, "14d")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCooldownConfigNormalizedPackages(t *testing.T) {
|
||||
rawScoped := "pkg:npm/@typescript/typescript-darwin-arm64"
|
||||
canonicalScoped := "pkg:npm/%40typescript/typescript-darwin-arm64"
|
||||
cfg := CooldownConfig{Packages: map[string]string{
|
||||
rawScoped: "2d",
|
||||
canonicalScoped: "3d",
|
||||
"not-a-purl": "4d",
|
||||
}}
|
||||
|
||||
got := cfg.NormalizedPackages()
|
||||
|
||||
if got[canonicalScoped] != "3d" {
|
||||
t.Errorf("canonical scoped package duration = %q, want %q", got[canonicalScoped], "3d")
|
||||
}
|
||||
if _, exists := got[rawScoped]; exists {
|
||||
t.Errorf("raw scoped package key %q was not canonicalized", rawScoped)
|
||||
}
|
||||
if got["not-a-purl"] != "4d" {
|
||||
t.Errorf("invalid PURL duration = %q, want preserved value %q", got["not-a-purl"], "4d")
|
||||
}
|
||||
if cfg.Packages[rawScoped] != "2d" {
|
||||
t.Error("NormalizedPackages mutated the source map")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCooldownFromEnv(t *testing.T) {
|
||||
|
|
@ -480,52 +353,6 @@ func TestParseMetadataTTL(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestParseMetadataMaxSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
size string
|
||||
want int64
|
||||
}{
|
||||
{"unset uses default", "", defaultMetadataMaxSize},
|
||||
{"explicit value", "250MB", 250 << 20},
|
||||
{"bytes", "1024", 1024},
|
||||
{"invalid uses default", "lots", defaultMetadataMaxSize},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.MetadataMaxSize = tt.size
|
||||
got := cfg.ParseMetadataMaxSize()
|
||||
if got != tt.want {
|
||||
t.Errorf("ParseMetadataMaxSize() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMetadataMaxSize(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.MetadataMaxSize = "not-a-size"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected validation error for invalid metadata_max_size")
|
||||
}
|
||||
|
||||
cfg.MetadataMaxSize = "0"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected validation error for zero metadata_max_size")
|
||||
}
|
||||
|
||||
cfg.MetadataMaxSize = "250MB"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for valid metadata_max_size: %v", err)
|
||||
}
|
||||
|
||||
cfg.MetadataMaxSize = ""
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for unset metadata_max_size: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMetadataTTL(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.MetadataTTL = "invalid"
|
||||
|
|
@ -544,97 +371,6 @@ func TestValidateMetadataTTL(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestValidateHealthStorageProbeInterval(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Health.StorageProbeInterval = "not-a-duration"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected validation error for invalid health.storage_probe_interval")
|
||||
}
|
||||
|
||||
cfg.Health.StorageProbeInterval = "30s"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for valid health.storage_probe_interval: %v", err)
|
||||
}
|
||||
|
||||
cfg.Health.StorageProbeInterval = "0"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for zero health.storage_probe_interval: %v", err)
|
||||
}
|
||||
|
||||
cfg.Health.StorageProbeInterval = ""
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for empty health.storage_probe_interval: %v", err)
|
||||
}
|
||||
|
||||
cfg.Health.StorageProbeInterval = "-5s"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected validation error for negative health.storage_probe_interval")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHTTPTimeout(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
timeout string
|
||||
want time.Duration
|
||||
}{
|
||||
{"empty defaults to 30s", "", 30 * time.Second},
|
||||
{"explicit zero disables", "0", 0},
|
||||
{"2 minutes", "2m", 2 * time.Minute},
|
||||
{"90 seconds", "90s", 90 * time.Second},
|
||||
{"invalid defaults to 30s", "not-a-duration", 30 * time.Second},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.HTTPTimeout = tt.timeout
|
||||
got := cfg.ParseHTTPTimeout()
|
||||
if got != tt.want {
|
||||
t.Errorf("ParseHTTPTimeout() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateHTTPTimeout(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.HTTPTimeout = "not-a-duration"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected validation error for invalid http_timeout")
|
||||
}
|
||||
|
||||
cfg.HTTPTimeout = "-5s"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected validation error for negative http_timeout")
|
||||
}
|
||||
|
||||
cfg.HTTPTimeout = "2m"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for valid http_timeout: %v", err)
|
||||
}
|
||||
|
||||
cfg.HTTPTimeout = "0"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for zero http_timeout: %v", err)
|
||||
}
|
||||
|
||||
cfg.HTTPTimeout = ""
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for empty http_timeout: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadHTTPTimeoutFromEnv(t *testing.T) {
|
||||
cfg := Default()
|
||||
t.Setenv("PROXY_HTTP_TIMEOUT", "90s")
|
||||
cfg.LoadFromEnv()
|
||||
|
||||
if cfg.HTTPTimeout != "90s" {
|
||||
t.Errorf("HTTPTimeout = %q, want %q", cfg.HTTPTimeout, "90s")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMetadataTTLFromEnv(t *testing.T) {
|
||||
cfg := Default()
|
||||
t.Setenv("PROXY_METADATA_TTL", "10m")
|
||||
|
|
@ -645,41 +381,6 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestParseGradleBuildCacheConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
if got := cfg.ParseGradleBuildCacheMaxUploadSize(); got != 100*1024*1024 {
|
||||
t.Errorf("ParseGradleBuildCacheMaxUploadSize() = %d, want %d", got, 100*1024*1024)
|
||||
}
|
||||
if got := cfg.ParseGradleBuildCacheMaxAge(); got != 168*time.Hour {
|
||||
t.Errorf("ParseGradleBuildCacheMaxAge() = %v, want %v", got, 168*time.Hour)
|
||||
}
|
||||
if got := cfg.ParseGradleBuildCacheMaxSize(); got != 0 {
|
||||
t.Errorf("ParseGradleBuildCacheMaxSize() = %d, want 0", got)
|
||||
}
|
||||
if got := cfg.ParseGradleBuildCacheSweepInterval(); got != 10*time.Minute {
|
||||
t.Errorf("ParseGradleBuildCacheSweepInterval() = %v, want %v", got, 10*time.Minute)
|
||||
}
|
||||
|
||||
cfg.Gradle.BuildCache.MaxUploadSize = "64MB"
|
||||
cfg.Gradle.BuildCache.MaxAge = "48h"
|
||||
cfg.Gradle.BuildCache.MaxSize = "2GB"
|
||||
cfg.Gradle.BuildCache.SweepInterval = "20m"
|
||||
|
||||
if got := cfg.ParseGradleBuildCacheMaxUploadSize(); got != 64*1024*1024 {
|
||||
t.Errorf("ParseGradleBuildCacheMaxUploadSize() = %d, want %d", got, 64*1024*1024)
|
||||
}
|
||||
if got := cfg.ParseGradleBuildCacheMaxAge(); got != 48*time.Hour {
|
||||
t.Errorf("ParseGradleBuildCacheMaxAge() = %v, want %v", got, 48*time.Hour)
|
||||
}
|
||||
if got := cfg.ParseGradleBuildCacheMaxSize(); got != 2*1024*1024*1024 {
|
||||
t.Errorf("ParseGradleBuildCacheMaxSize() = %d, want %d", got, 2*1024*1024*1024)
|
||||
}
|
||||
if got := cfg.ParseGradleBuildCacheSweepInterval(); got != 20*time.Minute {
|
||||
t.Errorf("ParseGradleBuildCacheSweepInterval() = %v, want %v", got, 20*time.Minute)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDirectServeTTL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
|
@ -735,40 +436,6 @@ func TestLoadDirectServeFromEnv(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestValidateUIBaseURLDefaultsToBaseURL(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.BaseURL = "https://proxy.example.com"
|
||||
cfg.UIBaseURL = ""
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("unexpected validation error: %v", err)
|
||||
}
|
||||
if cfg.UIBaseURL != "https://proxy.example.com" {
|
||||
t.Errorf("UIBaseURL = %q, want it to default to BaseURL %q", cfg.UIBaseURL, "https://proxy.example.com")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUIBaseURL(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
cfg.UIBaseURL = "not a url"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected validation error for relative ui_base_url")
|
||||
}
|
||||
|
||||
cfg = Default()
|
||||
cfg.UIBaseURL = "://bad"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected validation error for unparseable ui_base_url")
|
||||
}
|
||||
|
||||
cfg = Default()
|
||||
cfg.UIBaseURL = "https://ui.example.com/ui"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("unexpected error for valid ui_base_url: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectServeBaseURL(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
|
|
@ -787,135 +454,3 @@ func TestValidateDirectServeBaseURL(t *testing.T) {
|
|||
t.Errorf("unexpected error for valid direct_serve_base_url: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseConfigString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg DatabaseConfig
|
||||
want string
|
||||
}{
|
||||
{"sqlite", DatabaseConfig{Driver: "sqlite", Path: "./cache/proxy.db"}, "./cache/proxy.db"},
|
||||
{"default driver", DatabaseConfig{Path: "/var/lib/proxy.db"}, "/var/lib/proxy.db"},
|
||||
{"postgres no password", DatabaseConfig{Driver: "postgres", URL: "postgres://user@localhost:5432/proxy"}, "postgres://user@localhost:5432/proxy"},
|
||||
{"postgres redacts password", DatabaseConfig{Driver: "postgres", URL: "postgres://user:secret@localhost:5432/proxy?sslmode=disable"}, "postgres://user:xxxxx@localhost:5432/proxy?sslmode=disable"},
|
||||
{"postgres unparseable url", DatabaseConfig{Driver: "postgres", URL: "host=localhost user=foo password=bar"}, "postgres"},
|
||||
{"postgres ignores sqlite path", DatabaseConfig{Driver: "postgres", URL: "postgres://localhost/db", Path: "./cache/proxy.db"}, "postgres://localhost/db"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := tt.cfg.String(); got != tt.want {
|
||||
t.Errorf("%s: String() = %q, want %q", tt.name, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamAuthForURLMatchesURLComponents(t *testing.T) {
|
||||
registryAuth := AuthConfig{Type: "bearer", Token: "registry-token"}
|
||||
privateAuth := AuthConfig{Type: "bearer", Token: "private-token"}
|
||||
config := UpstreamConfig{Auth: map[string]AuthConfig{
|
||||
"https://registry.example.com": registryAuth,
|
||||
"https://registry.example.com/private": privateAuth,
|
||||
}}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
wantToken string
|
||||
}{
|
||||
{name: "registry root", url: "https://registry.example.com/package", wantToken: "registry-token"},
|
||||
{name: "host is case insensitive", url: "https://REGISTRY.EXAMPLE.COM/package", wantToken: "registry-token"},
|
||||
{name: "longest path match", url: "https://registry.example.com/private/package", wantToken: "private-token"},
|
||||
{name: "exact path match", url: "https://registry.example.com/private", wantToken: "private-token"},
|
||||
{name: "path segment boundary", url: "https://registry.example.com/private-other/package", wantToken: "registry-token"},
|
||||
{name: "lookalike host rejected", url: "https://registry.example.com.evil.test/package"},
|
||||
{name: "different scheme rejected", url: "http://registry.example.com/package"},
|
||||
{name: "different port rejected", url: "https://registry.example.com:8443/package"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
auth := config.AuthForURL(tt.url)
|
||||
if tt.wantToken == "" {
|
||||
if auth != nil {
|
||||
t.Fatalf("AuthForURL() = %+v, want nil", auth)
|
||||
}
|
||||
return
|
||||
}
|
||||
if auth == nil {
|
||||
t.Fatal("AuthForURL() = nil, want authentication")
|
||||
}
|
||||
if auth.Token != tt.wantToken {
|
||||
t.Errorf("token = %q, want %q", auth.Token, tt.wantToken)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUpstreamAuthURLs(t *testing.T) {
|
||||
t.Run("valid absolute URL", func(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Upstream.Auth = map[string]AuthConfig{
|
||||
"https://registry.example.com/private": {Type: "bearer", Token: "token"},
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid URL", func(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Upstream.Auth = map[string]AuthConfig{
|
||||
"registry.example.com": {Type: "bearer", Token: "token"},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want invalid upstream.auth URL error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "upstream.auth") || !strings.Contains(err.Error(), "registry.example.com") {
|
||||
t.Errorf("Validate() error = %q, want field and URL", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateNamedUpstreams(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
modify func(*Config)
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid Helm and OCI upstreams",
|
||||
modify: func(cfg *Config) {
|
||||
cfg.Upstream.Helm = map[string]string{"bitnami": "https://charts.bitnami.com/bitnami"}
|
||||
cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Helm upstream name contains path separator",
|
||||
modify: func(cfg *Config) {
|
||||
cfg.Upstream.Helm = map[string]string{"team/charts": "https://charts.example.com"}
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "OCI upstream URL is not absolute",
|
||||
modify: func(cfg *Config) {
|
||||
cfg.Upstream.OCI = map[string]string{"private": "registry.example.com"}
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
tt.modify(cfg)
|
||||
err := cfg.Validate()
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
125
internal/cooldown/cooldown.go
Normal file
125
internal/cooldown/cooldown.go
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
package cooldown
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const hoursPerDay = 24
|
||||
|
||||
// Config holds cooldown settings for version filtering.
|
||||
// Cooldown hides package versions published too recently, giving the community
|
||||
// time to spot malicious releases before they're pulled into projects.
|
||||
type Config struct {
|
||||
// Default is the global default cooldown duration (e.g., "3d", "48h").
|
||||
Default string `json:"default" yaml:"default"`
|
||||
|
||||
// Ecosystems overrides the default for specific ecosystems.
|
||||
// Keys are ecosystem names (e.g., "npm", "pypi").
|
||||
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
|
||||
|
||||
// Packages overrides the cooldown for specific packages.
|
||||
// Keys are PURLs (e.g., "pkg:npm/lodash", "pkg:npm/@babel/core").
|
||||
Packages map[string]string `json:"packages" yaml:"packages"`
|
||||
|
||||
defaultDuration time.Duration
|
||||
ecosystemDurations map[string]time.Duration
|
||||
packageDurations map[string]time.Duration
|
||||
parsed bool
|
||||
}
|
||||
|
||||
// parse resolves all string durations into time.Duration values.
|
||||
// Called lazily on first use.
|
||||
func (c *Config) parse() {
|
||||
if c.parsed {
|
||||
return
|
||||
}
|
||||
c.parsed = true
|
||||
|
||||
c.defaultDuration, _ = ParseDuration(c.Default)
|
||||
|
||||
c.ecosystemDurations = make(map[string]time.Duration, len(c.Ecosystems))
|
||||
for k, v := range c.Ecosystems {
|
||||
d, _ := ParseDuration(v)
|
||||
c.ecosystemDurations[k] = d
|
||||
}
|
||||
|
||||
c.packageDurations = make(map[string]time.Duration, len(c.Packages))
|
||||
for k, v := range c.Packages {
|
||||
d, _ := ParseDuration(v)
|
||||
c.packageDurations[k] = d
|
||||
}
|
||||
}
|
||||
|
||||
// For returns the effective cooldown duration for a given ecosystem and package PURL.
|
||||
// Resolution order: package override > ecosystem override > global default.
|
||||
func (c *Config) For(ecosystem, packagePURL string) time.Duration {
|
||||
c.parse()
|
||||
|
||||
if d, ok := c.packageDurations[packagePURL]; ok {
|
||||
return d
|
||||
}
|
||||
if d, ok := c.ecosystemDurations[ecosystem]; ok {
|
||||
return d
|
||||
}
|
||||
return c.defaultDuration
|
||||
}
|
||||
|
||||
// IsAllowed returns true if a version with the given publish time has passed
|
||||
// the cooldown period for this ecosystem/package.
|
||||
func (c *Config) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool {
|
||||
d := c.For(ecosystem, packagePURL)
|
||||
if d == 0 {
|
||||
return true
|
||||
}
|
||||
if publishedAt.IsZero() {
|
||||
return true
|
||||
}
|
||||
return time.Since(publishedAt) >= d
|
||||
}
|
||||
|
||||
// Enabled returns true if any cooldown is configured.
|
||||
func (c *Config) Enabled() bool {
|
||||
c.parse()
|
||||
if c.defaultDuration > 0 {
|
||||
return true
|
||||
}
|
||||
for _, d := range c.ecosystemDurations {
|
||||
if d > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, d := range c.packageDurations {
|
||||
if d > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ParseDuration parses a duration string supporting days (e.g., "3d"),
|
||||
// in addition to Go's standard time.ParseDuration formats ("48h", "30m").
|
||||
// "0" means disabled (returns 0).
|
||||
func ParseDuration(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "0" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Handle day suffix
|
||||
if numStr, ok := strings.CutSuffix(s, "d"); ok {
|
||||
days, err := strconv.ParseFloat(numStr, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid duration %q: %w", s, err)
|
||||
}
|
||||
return time.Duration(days * float64(hoursPerDay*time.Hour)), nil
|
||||
}
|
||||
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid duration %q: %w", s, err)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
133
internal/cooldown/cooldown_test.go
Normal file
133
internal/cooldown/cooldown_test.go
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
package cooldown
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want time.Duration
|
||||
wantErr bool
|
||||
}{
|
||||
{"", 0, false},
|
||||
{"0", 0, false},
|
||||
{"3d", 3 * 24 * time.Hour, false},
|
||||
{"7d", 7 * 24 * time.Hour, false},
|
||||
{"14d", 14 * 24 * time.Hour, false},
|
||||
{"1.5d", 36 * time.Hour, false},
|
||||
{"48h", 48 * time.Hour, false},
|
||||
{"30m", 30 * time.Minute, false},
|
||||
{"1h30m", 90 * time.Minute, false},
|
||||
{"invalid", 0, true},
|
||||
{"d", 0, true},
|
||||
{"xd", 0, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got, err := ParseDuration(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ParseDuration(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
continue
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("ParseDuration(%q) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigFor(t *testing.T) {
|
||||
c := &Config{
|
||||
Default: "3d",
|
||||
Ecosystems: map[string]string{
|
||||
"npm": "7d",
|
||||
"cargo": "0",
|
||||
},
|
||||
Packages: map[string]string{
|
||||
"pkg:npm/lodash": "0",
|
||||
"pkg:npm/@babel/core": "14d",
|
||||
},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
ecosystem string
|
||||
packagePURL string
|
||||
want time.Duration
|
||||
}{
|
||||
// Package override takes priority
|
||||
{"npm", "pkg:npm/lodash", 0},
|
||||
{"npm", "pkg:npm/@babel/core", 14 * 24 * time.Hour},
|
||||
// Ecosystem override
|
||||
{"npm", "pkg:npm/express", 7 * 24 * time.Hour},
|
||||
{"cargo", "pkg:cargo/serde", 0},
|
||||
// Global default
|
||||
{"pypi", "pkg:pypi/requests", 3 * 24 * time.Hour},
|
||||
{"pub", "pkg:pub/flutter", 3 * 24 * time.Hour},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := c.For(tt.ecosystem, tt.packagePURL)
|
||||
if got != tt.want {
|
||||
t.Errorf("For(%q, %q) = %v, want %v", tt.ecosystem, tt.packagePURL, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigIsAllowed(t *testing.T) {
|
||||
c := &Config{
|
||||
Default: "3d",
|
||||
Packages: map[string]string{
|
||||
"pkg:npm/lodash": "0",
|
||||
},
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ecosystem string
|
||||
packagePURL string
|
||||
publishedAt time.Time
|
||||
want bool
|
||||
}{
|
||||
{"old enough", "npm", "pkg:npm/express", now.Add(-4 * 24 * time.Hour), true},
|
||||
{"too recent", "npm", "pkg:npm/express", now.Add(-1 * 24 * time.Hour), false},
|
||||
{"exactly at boundary", "npm", "pkg:npm/express", now.Add(-3 * 24 * time.Hour), true},
|
||||
{"exempt package", "npm", "pkg:npm/lodash", now.Add(-1 * time.Minute), true},
|
||||
{"zero time", "npm", "pkg:npm/express", time.Time{}, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := c.IsAllowed(tt.ecosystem, tt.packagePURL, tt.publishedAt)
|
||||
if got != tt.want {
|
||||
t.Errorf("IsAllowed(%q, %q, %v) = %v, want %v",
|
||||
tt.ecosystem, tt.packagePURL, tt.publishedAt, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigEnabled(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg Config
|
||||
want bool
|
||||
}{
|
||||
{"empty config", Config{}, false},
|
||||
{"default only", Config{Default: "3d"}, true},
|
||||
{"ecosystem only", Config{Ecosystems: map[string]string{"npm": "7d"}}, true},
|
||||
{"package only", Config{Packages: map[string]string{"pkg:npm/x": "1d"}}, true},
|
||||
{"all zero", Config{Default: "0", Ecosystems: map[string]string{"npm": "0"}}, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.cfg.Enabled()
|
||||
if got != tt.want {
|
||||
t.Errorf("Enabled() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -8,11 +8,6 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
testContentHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
testIntegrity = "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="
|
||||
)
|
||||
|
||||
func TestCreateAndOpen(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
|
@ -137,7 +132,7 @@ func TestVersionCRUD(t *testing.T) {
|
|||
v := &Version{
|
||||
PURL: "pkg:npm/lodash@4.17.21",
|
||||
PackagePURL: "pkg:npm/lodash",
|
||||
Integrity: sql.NullString{String: testIntegrity, Valid: true},
|
||||
Integrity: sql.NullString{String: "sha512-abc123", Valid: true},
|
||||
}
|
||||
|
||||
err = db.UpsertVersion(v)
|
||||
|
|
@ -205,7 +200,7 @@ func TestArtifactCRUD(t *testing.T) {
|
|||
t.Error("expected artifact to not be cached yet")
|
||||
}
|
||||
|
||||
err = db.MarkArtifactCached(versionPURL, "lodash-4.17.21.tgz", "/cache/npm/lodash-4.17.21.tgz", testContentHash, 12345, "application/gzip")
|
||||
err = db.MarkArtifactCached(versionPURL, "lodash-4.17.21.tgz", "/cache/npm/lodash-4.17.21.tgz", "sha256-abc", 12345, "application/gzip")
|
||||
if err != nil {
|
||||
t.Fatalf("MarkArtifactCached failed: %v", err)
|
||||
}
|
||||
|
|
@ -244,86 +239,6 @@ func TestArtifactCRUD(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestGetCachedArtifact(t *testing.T) {
|
||||
runWithBothDatabases(t, func(t *testing.T, db *DB) {
|
||||
const (
|
||||
packagePURL = "pkg:npm/lodash"
|
||||
versionPURL = "pkg:npm/lodash@4.17.21"
|
||||
filename = "lodash-4.17.21.tgz"
|
||||
)
|
||||
seedCachedArtifactTestData(t, db, packagePURL, versionPURL, filename)
|
||||
|
||||
cached, err := db.GetCachedArtifact(packagePURL, versionPURL, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("GetCachedArtifact before cache failed: %v", err)
|
||||
}
|
||||
if cached != nil {
|
||||
t.Fatalf("expected no cached artifact, got %+v", cached)
|
||||
}
|
||||
|
||||
if err := db.MarkArtifactCached(versionPURL, filename, "/cache/npm/"+filename,
|
||||
testContentHash, 12345, "application/gzip"); err != nil {
|
||||
t.Fatalf("MarkArtifactCached failed: %v", err)
|
||||
}
|
||||
|
||||
cached, err = db.GetCachedArtifact(packagePURL, versionPURL, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("GetCachedArtifact failed: %v", err)
|
||||
}
|
||||
if cached == nil {
|
||||
t.Fatal("expected cached artifact, got nil")
|
||||
}
|
||||
if cached.Ecosystem != "npm" {
|
||||
t.Errorf("expected npm ecosystem, got %q", cached.Ecosystem)
|
||||
}
|
||||
if cached.StoragePath != "/cache/npm/"+filename {
|
||||
t.Errorf("expected cached storage path, got %q", cached.StoragePath)
|
||||
}
|
||||
if cached.ContentHash.String != testContentHash {
|
||||
t.Errorf("expected cached content hash, got %q", cached.ContentHash.String)
|
||||
}
|
||||
if cached.Size.Int64 != 12345 {
|
||||
t.Errorf("expected cached size 12345, got %d", cached.Size.Int64)
|
||||
}
|
||||
if cached.ContentType.String != "application/gzip" {
|
||||
t.Errorf("expected cached content type, got %q", cached.ContentType.String)
|
||||
}
|
||||
if cached.Integrity.String != testIntegrity {
|
||||
t.Errorf("expected cached integrity, got %q", cached.Integrity.String)
|
||||
}
|
||||
|
||||
cached, err = db.GetCachedArtifact("pkg:npm/other", versionPURL, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("GetCachedArtifact with wrong package failed: %v", err)
|
||||
}
|
||||
if cached != nil {
|
||||
t.Fatalf("expected package mismatch to miss cache, got %+v", cached)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func seedCachedArtifactTestData(t *testing.T, db *DB, packagePURL, versionPURL, filename string) {
|
||||
t.Helper()
|
||||
|
||||
if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: "npm", Name: "lodash"}); err != nil {
|
||||
t.Fatalf("UpsertPackage failed: %v", err)
|
||||
}
|
||||
if err := db.UpsertVersion(&Version{
|
||||
PURL: versionPURL,
|
||||
PackagePURL: packagePURL,
|
||||
Integrity: sql.NullString{String: testIntegrity, Valid: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("UpsertVersion failed: %v", err)
|
||||
}
|
||||
if err := db.UpsertArtifact(&Artifact{
|
||||
VersionPURL: versionPURL,
|
||||
Filename: filename,
|
||||
UpstreamURL: "https://registry.npmjs.org/lodash/-/" + filename,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpsertArtifact failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheManagement(t *testing.T) {
|
||||
runWithBothDatabases(t, func(t *testing.T, db *DB) {
|
||||
pkg := &Package{
|
||||
|
|
|
|||
|
|
@ -30,10 +30,6 @@ func TestUpsertAndGetMetadataCache(t *testing.T) {
|
|||
StoragePath: "_metadata/npm/lodash/metadata",
|
||||
ETag: sql.NullString{String: `"abc123"`, Valid: true},
|
||||
ContentType: sql.NullString{String: "application/json", Valid: true},
|
||||
ContentDigest: sql.NullString{
|
||||
String: "sha256:0123456789abcdef",
|
||||
Valid: true,
|
||||
},
|
||||
Size: sql.NullInt64{Int64: 1024, Valid: true},
|
||||
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
}
|
||||
|
|
@ -66,9 +62,6 @@ func TestUpsertAndGetMetadataCache(t *testing.T) {
|
|||
if !got.ContentType.Valid || got.ContentType.String != "application/json" {
|
||||
t.Errorf("content_type = %v, want %q", got.ContentType, "application/json")
|
||||
}
|
||||
if !got.ContentDigest.Valid || got.ContentDigest.String != "sha256:0123456789abcdef" {
|
||||
t.Errorf("content_digest = %v, want %q", got.ContentDigest, "sha256:0123456789abcdef")
|
||||
}
|
||||
if !got.Size.Valid || got.Size.Int64 != 1024 {
|
||||
t.Errorf("size = %v, want 1024", got.Size)
|
||||
}
|
||||
|
|
@ -185,47 +178,3 @@ func TestMetadataCacheTableCreatedByMigration(t *testing.T) {
|
|||
t.Error("metadata_cache table should exist after migration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataCacheContentDigestMigrationPreservesExistingRows(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
db, err := Create(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
if _, err := db.Exec("ALTER TABLE metadata_cache DROP COLUMN content_digest"); err != nil {
|
||||
t.Fatalf("dropping content_digest: %v", err)
|
||||
}
|
||||
if _, err := db.Exec("DELETE FROM migrations WHERE name = ?", "006_add_metadata_content_digest"); err != nil {
|
||||
t.Fatalf("resetting digest migration: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO metadata_cache (ecosystem, name, storage_path, content_type, size, fetched_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, "oci-manifest", "cache-key", "_metadata/oci-manifest/cache-key/metadata", "application/json", 2, time.Now(), time.Now(), time.Now()); err != nil {
|
||||
t.Fatalf("inserting legacy cache row: %v", err)
|
||||
}
|
||||
|
||||
if err := db.MigrateSchema(); err != nil {
|
||||
t.Fatalf("MigrateSchema() error = %v", err)
|
||||
}
|
||||
hasDigest, err := db.HasColumn("metadata_cache", "content_digest")
|
||||
if err != nil {
|
||||
t.Fatalf("HasColumn() error = %v", err)
|
||||
}
|
||||
if !hasDigest {
|
||||
t.Fatal("metadata_cache.content_digest was not added")
|
||||
}
|
||||
|
||||
entry, err := db.GetMetadataCache("oci-manifest", "cache-key")
|
||||
if err != nil {
|
||||
t.Fatalf("GetMetadataCache() error = %v", err)
|
||||
}
|
||||
if entry == nil || entry.StoragePath != "_metadata/oci-manifest/cache-key/metadata" {
|
||||
t.Fatalf("existing metadata cache row was not preserved: %#v", entry)
|
||||
}
|
||||
if entry.ContentDigest.Valid {
|
||||
t.Errorf("legacy content digest = %q, want NULL", entry.ContentDigest.String)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -191,28 +191,6 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) {
|
|||
return &a, nil
|
||||
}
|
||||
|
||||
// GetCachedArtifact returns the fields needed to serve a cached artifact.
|
||||
func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*CachedArtifact, error) {
|
||||
var artifact CachedArtifact
|
||||
query := db.Rebind(`
|
||||
SELECT packages.ecosystem, artifacts.storage_path, artifacts.content_hash, artifacts.size,
|
||||
artifacts.content_type, versions.integrity
|
||||
FROM artifacts
|
||||
JOIN versions ON versions.purl = artifacts.version_purl
|
||||
JOIN packages ON packages.purl = versions.package_purl
|
||||
WHERE packages.purl = ? AND artifacts.version_purl = ? AND artifacts.filename = ?
|
||||
AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL
|
||||
`)
|
||||
err := db.Get(&artifact, query, packagePURL, versionPURL, filename)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &artifact, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) {
|
||||
var a Artifact
|
||||
query := db.Rebind(`
|
||||
|
|
@ -467,12 +445,9 @@ func (db *DB) GetMostPopularPackages(limit int) ([]PopularPackage, error) {
|
|||
type RecentPackage struct {
|
||||
Ecosystem string `db:"ecosystem"`
|
||||
Name string `db:"name"`
|
||||
VersionPURL string `db:"version_purl"`
|
||||
Version string `db:"version"`
|
||||
CachedAt time.Time `db:"fetched_at"`
|
||||
Size int64 `db:"size"`
|
||||
// Version is derived from VersionPURL rather than selected, so that the
|
||||
// PURL percent-encoding is decoded (e.g. "%2B" back to "+").
|
||||
Version string `db:"-"`
|
||||
}
|
||||
|
||||
func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
|
||||
|
|
@ -486,10 +461,10 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
|
|||
}
|
||||
|
||||
var packages []RecentPackage
|
||||
// There is no separate version column, so the full version PURL is selected
|
||||
// and the version is decoded from it in Go.
|
||||
// We need to extract version from the purl since there's no separate version column
|
||||
query := db.Rebind(`
|
||||
SELECT p.ecosystem, p.name, v.purl as version_purl,
|
||||
SELECT p.ecosystem, p.name,
|
||||
SUBSTR(v.purl, INSTR(v.purl, '@') + 1) as version,
|
||||
a.fetched_at, COALESCE(a.size, 0) as size
|
||||
FROM artifacts a
|
||||
JOIN versions v ON v.purl = a.version_purl
|
||||
|
|
@ -499,13 +474,25 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
|
|||
LIMIT ?
|
||||
`)
|
||||
|
||||
// For postgres, use different string function
|
||||
if db.dialect == DialectPostgres {
|
||||
query = db.Rebind(`
|
||||
SELECT p.ecosystem, p.name,
|
||||
SUBSTRING(v.purl FROM POSITION('@' IN v.purl) + 1) as version,
|
||||
a.fetched_at, COALESCE(a.size, 0) as size
|
||||
FROM artifacts a
|
||||
JOIN versions v ON v.purl = a.version_purl
|
||||
JOIN packages p ON p.purl = v.package_purl
|
||||
WHERE a.storage_path IS NOT NULL AND a.fetched_at IS NOT NULL
|
||||
ORDER BY a.fetched_at DESC
|
||||
LIMIT ?
|
||||
`)
|
||||
}
|
||||
|
||||
err = db.Select(&packages, query, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range packages {
|
||||
packages[i].Version = VersionFromPURL(packages[i].VersionPURL)
|
||||
}
|
||||
return packages, nil
|
||||
}
|
||||
|
||||
|
|
@ -907,7 +894,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err
|
|||
var entry MetadataCacheEntry
|
||||
query := db.Rebind(`
|
||||
SELECT id, ecosystem, name, storage_path, etag, content_type,
|
||||
content_digest, size, last_modified, fetched_at, created_at, updated_at
|
||||
size, last_modified, fetched_at, created_at, updated_at
|
||||
FROM metadata_cache WHERE ecosystem = ? AND name = ?
|
||||
`)
|
||||
err := db.Get(&entry, query, ecosystem, name)
|
||||
|
|
@ -927,13 +914,12 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
|
|||
if db.dialect == DialectPostgres {
|
||||
query = `
|
||||
INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type,
|
||||
content_digest, size, last_modified, fetched_at, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
size, last_modified, fetched_at, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT(ecosystem, name) DO UPDATE SET
|
||||
storage_path = EXCLUDED.storage_path,
|
||||
etag = EXCLUDED.etag,
|
||||
content_type = EXCLUDED.content_type,
|
||||
content_digest = EXCLUDED.content_digest,
|
||||
size = EXCLUDED.size,
|
||||
last_modified = EXCLUDED.last_modified,
|
||||
fetched_at = EXCLUDED.fetched_at,
|
||||
|
|
@ -942,13 +928,12 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
|
|||
} else {
|
||||
query = `
|
||||
INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type,
|
||||
content_digest, size, last_modified, fetched_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
size, last_modified, fetched_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(ecosystem, name) DO UPDATE SET
|
||||
storage_path = excluded.storage_path,
|
||||
etag = excluded.etag,
|
||||
content_type = excluded.content_type,
|
||||
content_digest = excluded.content_digest,
|
||||
size = excluded.size,
|
||||
last_modified = excluded.last_modified,
|
||||
fetched_at = excluded.fetched_at,
|
||||
|
|
@ -958,7 +943,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
|
|||
|
||||
_, err := db.Exec(query,
|
||||
entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag,
|
||||
entry.ContentType, entry.ContentDigest, entry.Size, entry.LastModified, entry.FetchedAt, now, now,
|
||||
entry.ContentType, entry.Size, entry.LastModified, entry.FetchedAt, now, now,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upserting metadata cache: %w", err)
|
||||
|
|
|
|||
|
|
@ -6,11 +6,7 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
postgresTimestamp = "TIMESTAMP"
|
||||
sqliteDatetime = "DATETIME"
|
||||
colTypeText = "TEXT"
|
||||
)
|
||||
const postgresTimestamp = "TIMESTAMP"
|
||||
|
||||
// Schema for proxy-specific tables. The packages and versions tables
|
||||
// are compatible with git-pkgs, allowing the proxy to use an existing
|
||||
|
|
@ -102,7 +98,6 @@ CREATE TABLE IF NOT EXISTS metadata_cache (
|
|||
storage_path TEXT NOT NULL,
|
||||
etag TEXT,
|
||||
content_type TEXT,
|
||||
content_digest TEXT,
|
||||
size INTEGER,
|
||||
last_modified DATETIME,
|
||||
fetched_at DATETIME,
|
||||
|
|
@ -203,7 +198,6 @@ CREATE TABLE IF NOT EXISTS metadata_cache (
|
|||
storage_path TEXT NOT NULL,
|
||||
etag TEXT,
|
||||
content_type TEXT,
|
||||
content_digest TEXT,
|
||||
size BIGINT,
|
||||
last_modified TIMESTAMP,
|
||||
fetched_at TIMESTAMP,
|
||||
|
|
@ -361,7 +355,6 @@ var migrations = []migration{
|
|||
{"003_ensure_artifacts_table", migrateEnsureArtifactsTable},
|
||||
{"004_ensure_vulnerabilities_table", migrateEnsureVulnerabilitiesTable},
|
||||
{"005_ensure_metadata_cache_table", migrateEnsureMetadataCacheTable},
|
||||
{"006_add_metadata_content_digest", migrateAddMetadataContentDigest},
|
||||
}
|
||||
|
||||
// isTableNotFound returns true if the error indicates a missing table.
|
||||
|
|
@ -376,9 +369,9 @@ func isTableNotFound(err error) bool {
|
|||
func (db *DB) createMigrationsTable() error {
|
||||
var ts string
|
||||
if db.dialect == DialectPostgres {
|
||||
ts = postgresTimestamp
|
||||
ts = "TIMESTAMP"
|
||||
} else {
|
||||
ts = sqliteDatetime
|
||||
ts = "DATETIME"
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS migrations (
|
||||
|
|
@ -464,12 +457,12 @@ func (db *DB) MigrateSchema() error {
|
|||
|
||||
func migrateAddPackagesEnrichmentColumns(db *DB) error {
|
||||
columns := map[string]string{
|
||||
"registry_url": colTypeText,
|
||||
"supplier_name": colTypeText,
|
||||
"supplier_type": colTypeText,
|
||||
"source": colTypeText,
|
||||
"enriched_at": sqliteDatetime,
|
||||
"vulns_synced_at": sqliteDatetime,
|
||||
"registry_url": "TEXT",
|
||||
"supplier_name": "TEXT",
|
||||
"supplier_type": "TEXT",
|
||||
"source": "TEXT",
|
||||
"enriched_at": "DATETIME",
|
||||
"vulns_synced_at": "DATETIME",
|
||||
}
|
||||
|
||||
if db.dialect == DialectPostgres {
|
||||
|
|
@ -494,10 +487,10 @@ func migrateAddPackagesEnrichmentColumns(db *DB) error {
|
|||
|
||||
func migrateAddVersionsEnrichmentColumns(db *DB) error {
|
||||
columns := map[string]string{
|
||||
"integrity": colTypeText,
|
||||
"integrity": "TEXT",
|
||||
"yanked": "INTEGER DEFAULT 0",
|
||||
"source": colTypeText,
|
||||
"enriched_at": sqliteDatetime,
|
||||
"source": "TEXT",
|
||||
"enriched_at": "DATETIME",
|
||||
}
|
||||
|
||||
if db.dialect == DialectPostgres {
|
||||
|
|
@ -584,20 +577,6 @@ func migrateEnsureMetadataCacheTable(db *DB) error {
|
|||
return db.EnsureMetadataCacheTable()
|
||||
}
|
||||
|
||||
func migrateAddMetadataContentDigest(db *DB) error {
|
||||
hasColumn, err := db.HasColumn("metadata_cache", "content_digest")
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking metadata_cache content_digest column: %w", err)
|
||||
}
|
||||
if hasColumn {
|
||||
return nil
|
||||
}
|
||||
if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN content_digest TEXT"); err != nil {
|
||||
return fmt.Errorf("adding metadata_cache content_digest column: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist.
|
||||
func (db *DB) EnsureMetadataCacheTable() error {
|
||||
has, err := db.HasTable("metadata_cache")
|
||||
|
|
@ -618,7 +597,6 @@ func (db *DB) EnsureMetadataCacheTable() error {
|
|||
storage_path TEXT NOT NULL,
|
||||
etag TEXT,
|
||||
content_type TEXT,
|
||||
content_digest TEXT,
|
||||
size BIGINT,
|
||||
last_modified TIMESTAMP,
|
||||
fetched_at TIMESTAMP,
|
||||
|
|
@ -636,7 +614,6 @@ func (db *DB) EnsureMetadataCacheTable() error {
|
|||
storage_path TEXT NOT NULL,
|
||||
etag TEXT,
|
||||
content_type TEXT,
|
||||
content_digest TEXT,
|
||||
size INTEGER,
|
||||
last_modified DATETIME,
|
||||
fetched_at DATETIME,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package database
|
|||
|
||||
import (
|
||||
"database/sql"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -48,80 +47,11 @@ type Version struct {
|
|||
// Version extracts the version string from the PURL.
|
||||
// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21"
|
||||
func (v *Version) Version() string {
|
||||
return VersionFromPURL(v.PURL)
|
||||
if idx := strings.LastIndex(v.PURL, "@"); idx >= 0 {
|
||||
return v.PURL[idx+1:]
|
||||
}
|
||||
|
||||
// EscapedVersion returns the version escaped for use as a single URL path
|
||||
// segment.
|
||||
//
|
||||
// Version returns decoded text, which is what should be shown to a user but is
|
||||
// not safe to drop into a link: html/template preserves reserved characters and
|
||||
// existing escapes in a URL, so "release/1" would split into two path segments,
|
||||
// "v1?build" would start a query string, and a literal "%2B" would be read back
|
||||
// as "+". Escaping here and decoding in splitWildcardPath round-trips the value,
|
||||
// so the link resolves to the version that was stored.
|
||||
func (v *Version) EscapedVersion() string {
|
||||
return url.PathEscape(v.Version())
|
||||
}
|
||||
|
||||
// DisplayPURL returns the PURL with its path components percent-decoded, for
|
||||
// showing in the UI. The stored PURL keeps the canonical encoding (which is
|
||||
// what the API and all lookups use); this is only a readable rendering, so that
|
||||
// a version like "7.91+dfsg1-2ubuntu0.1" is not shown as "7.91%2Bdfsg1-2ubuntu0.1"
|
||||
// and an npm scope is shown as "@babel" rather than "%40babel". Qualifiers and
|
||||
// subpath keep their encoding, since decoding those would be ambiguous.
|
||||
func (v *Version) DisplayPURL() string {
|
||||
base, suffix := v.PURL, ""
|
||||
if i := strings.IndexAny(base, "?#"); i >= 0 {
|
||||
base, suffix = base[:i], base[i:]
|
||||
}
|
||||
|
||||
name, version := base, ""
|
||||
if idx := strings.LastIndex(base, "@"); idx >= 0 {
|
||||
name, version = base[:idx], "@"+decodePURLComponent(base[idx+1:])
|
||||
}
|
||||
|
||||
parts := strings.Split(name, "/")
|
||||
for i, part := range parts {
|
||||
parts[i] = decodePURLComponent(part)
|
||||
}
|
||||
return strings.Join(parts, "/") + version + suffix
|
||||
}
|
||||
|
||||
// VersionFromPURL extracts the decoded version string from a PURL.
|
||||
//
|
||||
// PURL percent-encodes characters that are not safe in a path component, so a
|
||||
// Debian version like "7.91+dfsg1-2ubuntu0.1" is stored as
|
||||
// "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1". The raw substring after "@" is
|
||||
// therefore not the version: it must be percent-decoded before being displayed
|
||||
// or used to build a URL, otherwise "%2B" leaks into the UI and round-tripping
|
||||
// the value back into a PURL double-encodes it.
|
||||
//
|
||||
// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21"
|
||||
func VersionFromPURL(p string) string {
|
||||
// Qualifiers ("?key=value") and subpath ("#path") follow the version.
|
||||
if i := strings.IndexAny(p, "?#"); i >= 0 {
|
||||
p = p[:i]
|
||||
}
|
||||
idx := strings.LastIndex(p, "@")
|
||||
if idx < 0 {
|
||||
return ""
|
||||
}
|
||||
return decodePURLComponent(p[idx+1:])
|
||||
}
|
||||
|
||||
// decodePURLComponent percent-decodes a single PURL path component, returning
|
||||
// the input unchanged if it is not valid percent-encoding.
|
||||
func decodePURLComponent(s string) string {
|
||||
if !strings.Contains(s, "%") {
|
||||
return s
|
||||
}
|
||||
decoded, err := url.PathUnescape(s)
|
||||
if err != nil {
|
||||
return s
|
||||
}
|
||||
return decoded
|
||||
}
|
||||
|
||||
// Artifact represents a cached artifact in the database.
|
||||
// This table is proxy-specific and not part of git-pkgs.
|
||||
|
|
@ -146,16 +76,6 @@ func (a *Artifact) IsCached() bool {
|
|||
return a.StoragePath.Valid && a.FetchedAt.Valid
|
||||
}
|
||||
|
||||
// CachedArtifact contains the fields needed to serve a cached artifact.
|
||||
type CachedArtifact struct {
|
||||
Ecosystem string `db:"ecosystem"`
|
||||
StoragePath string `db:"storage_path"`
|
||||
ContentHash sql.NullString `db:"content_hash"`
|
||||
Size sql.NullInt64 `db:"size"`
|
||||
ContentType sql.NullString `db:"content_type"`
|
||||
Integrity sql.NullString `db:"integrity"`
|
||||
}
|
||||
|
||||
// MetadataCacheEntry represents a cached metadata blob for offline serving.
|
||||
type MetadataCacheEntry struct {
|
||||
ID int64 `db:"id" json:"id"`
|
||||
|
|
@ -164,7 +84,6 @@ type MetadataCacheEntry struct {
|
|||
StoragePath string `db:"storage_path" json:"storage_path"`
|
||||
ETag sql.NullString `db:"etag" json:"etag,omitempty"`
|
||||
ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"`
|
||||
ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"`
|
||||
Size sql.NullInt64 `db:"size" json:"size,omitempty"`
|
||||
LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"`
|
||||
FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"`
|
||||
|
|
|
|||
|
|
@ -1,159 +0,0 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestVersionFromPURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
purl string
|
||||
want string
|
||||
}{
|
||||
{"simple", "pkg:npm/lodash@4.17.21", "4.17.21"},
|
||||
{"namespaced", "pkg:composer/symfony/console@6.0.0", "6.0.0"},
|
||||
// Debian/Ubuntu versions routinely contain "+", which PURL encodes.
|
||||
{"encoded plus", "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"},
|
||||
{"encoded epoch", "pkg:deb/curl@1%3A7.81.0-1", "1:7.81.0-1"},
|
||||
{"encoded plus with qualifier", "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", "7.91+dfsg1"},
|
||||
{"tilde is not encoded", "pkg:deb/foo@1.0~rc1", "1.0~rc1"},
|
||||
{"no version", "pkg:npm/lodash", ""},
|
||||
{"invalid escape passed through", "pkg:npm/lodash@1.0%zz", "1.0%zz"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := VersionFromPURL(tt.purl); got != tt.want {
|
||||
t.Errorf("VersionFromPURL(%q) = %q, want %q", tt.purl, got, tt.want)
|
||||
}
|
||||
v := &Version{PURL: tt.purl}
|
||||
if got := v.Version(); got != tt.want {
|
||||
t.Errorf("Version.Version() for %q = %q, want %q", tt.purl, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestVersionEscapedVersion checks the value the templates put in a URL. It
|
||||
// must survive the round trip back through the router: escaping here and
|
||||
// decoding per path segment on the way in has to yield the original version.
|
||||
func TestVersionEscapedVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
purl string
|
||||
want string
|
||||
}{
|
||||
{"simple", "pkg:npm/lodash@4.17.21", "4.17.21"},
|
||||
// "+" is legal in a path segment, so it stays literal and the UI keeps
|
||||
// showing the version the way Debian writes it.
|
||||
{"plus stays literal", "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1-2ubuntu0.1"},
|
||||
// A slash would otherwise split the version into two path segments.
|
||||
{"slash", "pkg:golang/example@release%2F1", "release%2F1"},
|
||||
// A question mark would otherwise start the query string.
|
||||
{"question mark", "pkg:npm/example@v1%3Fbuild", "v1%3Fbuild"},
|
||||
// A version containing a literal "%2B" is stored double-encoded; the
|
||||
// link must re-encode it or it decodes back to "+" instead.
|
||||
{"literal percent escape", "pkg:npm/example@1.0%252B", "1.0%252B"},
|
||||
{"space", "pkg:npm/example@1.0%20beta", "1.0%20beta"},
|
||||
{"no version", "pkg:npm/lodash", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
v := &Version{PURL: tt.purl}
|
||||
got := v.EscapedVersion()
|
||||
if got != tt.want {
|
||||
t.Errorf("EscapedVersion() for %q = %q, want %q", tt.purl, got, tt.want)
|
||||
}
|
||||
// The router decodes each path segment, which must give back the
|
||||
// version the page displays.
|
||||
decoded, err := url.PathUnescape(got)
|
||||
if err != nil {
|
||||
t.Fatalf("PathUnescape(%q) failed: %v", got, err)
|
||||
}
|
||||
if decoded != v.Version() {
|
||||
t.Errorf("round trip for %q = %q, want %q", tt.purl, decoded, v.Version())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionDisplayPURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
purl string
|
||||
want string
|
||||
}{
|
||||
{"simple", "pkg:npm/lodash@4.17.21", "pkg:npm/lodash@4.17.21"},
|
||||
{
|
||||
"encoded plus",
|
||||
"pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1",
|
||||
"pkg:deb/nmap@7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1",
|
||||
},
|
||||
{
|
||||
"qualifier preserved",
|
||||
"pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com",
|
||||
"pkg:deb/nmap@7.91+dfsg1?repository_url=http%3A%2F%2Fexample.com",
|
||||
},
|
||||
// The namespace is encoded too: MakePURLString("npm", "@babel/core", …)
|
||||
// produces "pkg:npm/%40babel/core@…".
|
||||
{"encoded npm scope", "pkg:npm/%40babel/core@7.0.0", "pkg:npm/@babel/core@7.0.0"},
|
||||
{"encoded scope without version", "pkg:npm/%40babel/core", "pkg:npm/@babel/core"},
|
||||
{"no version", "pkg:npm/lodash", "pkg:npm/lodash"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
v := &Version{PURL: tt.purl}
|
||||
if got := v.DisplayPURL(); got != tt.want {
|
||||
t.Errorf("DisplayPURL() for %q = %q, want %q", tt.purl, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetRecentlyCachedPackagesDecodesVersion guards the dashboard's "recently
|
||||
// cached" list, which derives the version from the version PURL.
|
||||
func TestGetRecentlyCachedPackagesDecodesVersion(t *testing.T) {
|
||||
runWithBothDatabases(t, func(t *testing.T, db *DB) {
|
||||
const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1"
|
||||
|
||||
if err := db.UpsertPackage(&Package{
|
||||
PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap",
|
||||
}); err != nil {
|
||||
t.Fatalf("UpsertPackage failed: %v", err)
|
||||
}
|
||||
if err := db.UpsertVersion(&Version{
|
||||
PURL: versionPURL, PackagePURL: "pkg:deb/nmap",
|
||||
}); err != nil {
|
||||
t.Fatalf("UpsertVersion failed: %v", err)
|
||||
}
|
||||
if err := db.UpsertArtifact(&Artifact{
|
||||
VersionPURL: versionPURL,
|
||||
Filename: "nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb",
|
||||
UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb",
|
||||
StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true},
|
||||
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("UpsertArtifact failed: %v", err)
|
||||
}
|
||||
|
||||
recent, err := db.GetRecentlyCachedPackages(10)
|
||||
if err != nil {
|
||||
t.Fatalf("GetRecentlyCachedPackages failed: %v", err)
|
||||
}
|
||||
if len(recent) != 1 {
|
||||
t.Fatalf("expected 1 recent package, got %d", len(recent))
|
||||
}
|
||||
const want = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"
|
||||
if recent[0].Version != want {
|
||||
t.Errorf("Version = %q, want %q", recent[0].Version, want)
|
||||
}
|
||||
if recent[0].VersionPURL != versionPURL {
|
||||
t.Errorf("VersionPURL = %q, want %q", recent[0].VersionPURL, versionPURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -201,6 +201,43 @@ func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, ver
|
|||
return results, nil
|
||||
}
|
||||
|
||||
// BulkCheckVulnerabilities queries vulnerabilities for multiple package versions.
|
||||
func (s *Service) BulkCheckVulnerabilities(ctx context.Context, packages []struct{ Ecosystem, Name, Version string }) (map[string][]VulnInfo, error) {
|
||||
purls := make([]*purl.PURL, len(packages))
|
||||
for i, pkg := range packages {
|
||||
purls[i] = purl.MakePURL(pkg.Ecosystem, pkg.Name, pkg.Version)
|
||||
}
|
||||
|
||||
vulnResults, err := s.vulnSource.QueryBatch(ctx, purls)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make(map[string][]VulnInfo, len(packages))
|
||||
for i, vulnList := range vulnResults {
|
||||
pkg := packages[i]
|
||||
key := purl.MakePURLString(pkg.Ecosystem, pkg.Name, pkg.Version)
|
||||
|
||||
var infos []VulnInfo
|
||||
for _, v := range vulnList {
|
||||
info := VulnInfo{
|
||||
ID: v.ID,
|
||||
Summary: v.Summary,
|
||||
Severity: v.SeverityLevel(),
|
||||
CVSSScore: v.CVSSScore(),
|
||||
FixedVersion: v.FixedVersion(pkg.Ecosystem, pkg.Name),
|
||||
}
|
||||
for _, ref := range v.References {
|
||||
info.References = append(info.References, ref.URL)
|
||||
}
|
||||
infos = append(infos, info)
|
||||
}
|
||||
result[key] = infos
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// IsOutdated checks if a version is older than the latest version.
|
||||
func (s *Service) IsOutdated(currentVersion, latestVersion string) bool {
|
||||
if latestVersion == "" || currentVersion == "" {
|
||||
|
|
@ -251,6 +288,19 @@ func (s *Service) CategorizeLicense(license string) LicenseCategory {
|
|||
return LicenseUnknown
|
||||
}
|
||||
|
||||
// NormalizeLicense normalizes a license string to SPDX format.
|
||||
func (s *Service) NormalizeLicense(license string) string {
|
||||
if license == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
if normalized, err := spdx.NormalizeExpressionLax(license); err == nil {
|
||||
return normalized
|
||||
}
|
||||
|
||||
return license
|
||||
}
|
||||
|
||||
// EnrichmentResult contains all enrichment data for a package version.
|
||||
type EnrichmentResult struct {
|
||||
Package *PackageInfo
|
||||
|
|
|
|||
|
|
@ -74,3 +74,25 @@ func TestCategorizeLicense(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLicense(t *testing.T) {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
svc := New(logger)
|
||||
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{"MIT", "MIT"},
|
||||
{"Apache 2", "Apache-2.0"},
|
||||
{"Apache-2.0", "Apache-2.0"},
|
||||
{"", ""},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
result := svc.NormalizeLicense(tc.input)
|
||||
if result != tc.expected {
|
||||
t.Errorf("NormalizeLicense(%q) = %q, want %q", tc.input, result, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,9 +6,10 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -29,18 +30,11 @@ type CargoHandler struct {
|
|||
}
|
||||
|
||||
// NewCargoHandler creates a new cargo protocol handler.
|
||||
func NewCargoHandler(proxy *Proxy, proxyURL, indexURL, downloadURL string) *CargoHandler {
|
||||
if strings.TrimSpace(indexURL) == "" {
|
||||
indexURL = cargoUpstream
|
||||
}
|
||||
if strings.TrimSpace(downloadURL) == "" {
|
||||
downloadURL = cargoDownloadBase
|
||||
}
|
||||
|
||||
func NewCargoHandler(proxy *Proxy, proxyURL string) *CargoHandler {
|
||||
return &CargoHandler{
|
||||
proxy: proxy,
|
||||
indexURL: strings.TrimSuffix(indexURL, "/"),
|
||||
downloadURL: strings.TrimSuffix(downloadURL, "/"),
|
||||
indexURL: cargoUpstream,
|
||||
downloadURL: cargoDownloadBase,
|
||||
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
||||
}
|
||||
}
|
||||
|
|
@ -149,7 +143,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr
|
|||
continue
|
||||
}
|
||||
|
||||
cratePURL := canonicalPackagePURL("cargo", crate.Name)
|
||||
cratePURL := purl.MakePURLString("cargo", crate.Name, "")
|
||||
|
||||
if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) {
|
||||
h.proxy.Logger.Info("cooldown: filtering cargo version",
|
||||
|
|
@ -197,17 +191,10 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|||
h.proxy.Logger.Info("cargo download request",
|
||||
"crate", name, "version", version, "filename", filename)
|
||||
|
||||
downloadURL := fmt.Sprintf(
|
||||
"%s/%s/%s",
|
||||
h.downloadURL,
|
||||
url.PathEscape(name),
|
||||
url.PathEscape(filename),
|
||||
)
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(
|
||||
r.Context(), "cargo", name, version, filename, downloadURL,
|
||||
)
|
||||
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "cargo", name, version, filename)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch crate")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch crate", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package handler
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -10,8 +9,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
)
|
||||
|
||||
func cargoTestProxy() *Proxy {
|
||||
|
|
@ -72,75 +70,6 @@ func TestCargoConfigEndpoint(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCargoHandlerUsesConfiguredUpstreams(t *testing.T) {
|
||||
t.Run("index", func(t *testing.T) {
|
||||
var requestPath, authHeader string
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestPath = r.URL.Path
|
||||
authHeader = r.Header.Get("Authorization")
|
||||
if authHeader != "Bearer cargo-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = io.WriteString(w, `{"name":"serde","vers":"1.0.0"}`)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
proxy.AuthForURL = func(string) (string, string) {
|
||||
return "Authorization", "Bearer cargo-token"
|
||||
}
|
||||
h := NewCargoHandler(
|
||||
proxy,
|
||||
"http://proxy.test",
|
||||
upstream.URL+"/index/",
|
||||
"https://crates.example.test/files/",
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/se/rd/serde", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
}
|
||||
if requestPath != "/index/se/rd/serde" {
|
||||
t.Errorf("upstream path = %q, want %q", requestPath, "/index/se/rd/serde")
|
||||
}
|
||||
if authHeader != "Bearer cargo-token" {
|
||||
t.Errorf("Authorization = %q, want %q", authHeader, "Bearer cargo-token")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("download", func(t *testing.T) {
|
||||
proxy, _, _, artifactFetcher := setupTestProxy(t)
|
||||
artifactFetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("crate")),
|
||||
ContentType: "application/gzip",
|
||||
}
|
||||
h := NewCargoHandler(
|
||||
proxy,
|
||||
"http://proxy.test",
|
||||
"https://index.example.test/root/",
|
||||
"https://crates.example.test/files/",
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/crates/serde/1.0.0/download", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
}
|
||||
want := "https://crates.example.test/files/serde/serde-1.0.0.crate"
|
||||
if artifactFetcher.fetchedURL != want {
|
||||
t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCargoIndexProxy(t *testing.T) {
|
||||
// Create a mock upstream index server
|
||||
indexContent := `{"name":"serde","vers":"1.0.0","deps":[],"cksum":"abc123"}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
|
@ -10,12 +9,13 @@ import (
|
|||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
)
|
||||
|
||||
const (
|
||||
composerUpstream = "https://packagist.org"
|
||||
composerRepo = "https://repo.packagist.org"
|
||||
composerUnset = "__unset"
|
||||
vendorPackageParts = 2
|
||||
)
|
||||
|
||||
|
|
@ -150,8 +150,7 @@ func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) {
|
|||
|
||||
// expandMinifiedVersions expands the Composer v2 minified format where each
|
||||
// version entry only contains fields that differ from the previous entry.
|
||||
// The "~dev" sentinel string resets the inheritance chain, and the "__unset"
|
||||
// value removes a field from the inherited state.
|
||||
// The "~dev" sentinel string resets the inheritance chain.
|
||||
func expandMinifiedVersions(versionList []any) []any {
|
||||
expanded := make([]any, 0, len(versionList))
|
||||
inherited := map[string]any{}
|
||||
|
|
@ -175,10 +174,6 @@ func expandMinifiedVersions(versionList []any) []any {
|
|||
merged[k] = deepCopyValue(val)
|
||||
}
|
||||
for k, val := range vmap {
|
||||
if val == composerUnset {
|
||||
delete(merged, k)
|
||||
continue
|
||||
}
|
||||
merged[k] = val
|
||||
}
|
||||
|
||||
|
|
@ -214,7 +209,7 @@ func deepCopyValue(v any) any {
|
|||
// filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs
|
||||
// for a single package's version list.
|
||||
func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any {
|
||||
packagePURL := canonicalPackagePURL("composer", packageName)
|
||||
packagePURL := purl.MakePURLString("composer", packageName, "")
|
||||
|
||||
filtered := versionList[:0]
|
||||
for _, v := range versionList {
|
||||
|
|
@ -306,127 +301,50 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request)
|
|||
h.proxy.Logger.Info("composer download request",
|
||||
"package", packageName, "version", version, "filename", filename)
|
||||
|
||||
// We need to fetch the metadata to get the actual download URL since
|
||||
// Packagist URLs include a hash. Packagist serves dev versions (e.g.
|
||||
// "3.x-dev", "dev-master") from a separate "~dev" metadata file, while
|
||||
// tagged releases live in the regular file. Try the file most likely to
|
||||
// contain this version first, then fall back to the other so that both
|
||||
// stable and dev versions resolve correctly.
|
||||
metaURLs := h.metadataURLsForVersion(vendor, pkg, version)
|
||||
// We need to fetch the metadata to get the actual download URL
|
||||
// since Packagist URLs include a hash
|
||||
metaURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg)
|
||||
|
||||
h.proxy.Logger.Debug("resolving download URL",
|
||||
"package", packageName, "version", version,
|
||||
"metadata_urls", metaURLs)
|
||||
|
||||
var downloadURL string
|
||||
for _, metaURL := range metaURLs {
|
||||
url, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version)
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, metaURL, nil)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL)
|
||||
http.Error(w, "failed to fetch metadata", http.StatusBadGateway)
|
||||
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if url != "" {
|
||||
downloadURL = url
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if downloadURL == "" {
|
||||
h.proxy.Logger.Debug("version not found in any metadata source",
|
||||
"package", packageName, "version", version,
|
||||
"tried_urls", metaURLs)
|
||||
http.Error(w, "version not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.Logger.Debug("resolved download URL",
|
||||
"package", packageName, "version", version,
|
||||
"download_url", downloadURL)
|
||||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
return
|
||||
}
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// isDevVersion reports whether a Composer version string refers to a
|
||||
// development (unstable, branch) version rather than a tagged release.
|
||||
// Composer formats these as either "dev-<branch>" (e.g. "dev-master") or
|
||||
// "<alias>-dev" (e.g. "3.x-dev").
|
||||
func isDevVersion(version string) bool {
|
||||
return strings.HasPrefix(version, "dev-") || strings.HasSuffix(version, "-dev")
|
||||
}
|
||||
|
||||
// metadataURLsForVersion returns the upstream metadata URLs to consult for a
|
||||
// given version, in priority order. Dev versions are served from the "~dev"
|
||||
// file, tagged releases from the regular file; the other file is included as a
|
||||
// fallback so an unexpected classification still resolves.
|
||||
func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string {
|
||||
stable := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg)
|
||||
dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", h.repoURL, vendor, pkg)
|
||||
|
||||
if isDevVersion(version) {
|
||||
return []string{dev, stable}
|
||||
}
|
||||
return []string{stable, dev}
|
||||
}
|
||||
|
||||
// findDownloadURLFromMetadata fetches a metadata document and returns the dist
|
||||
// URL for the given version, or an empty string if the version is not present.
|
||||
// An error is returned only on transport failure; a missing document (non-200)
|
||||
// or a missing version both yield an empty string so the caller can fall back.
|
||||
func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, error) {
|
||||
h.proxy.Logger.Debug("fetching upstream metadata for download lookup",
|
||||
"url", metaURL, "package", packageName, "version", version)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
h.proxy.Logger.Error("failed to fetch metadata", "error", err)
|
||||
http.Error(w, "failed to fetch metadata", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
h.proxy.Logger.Debug("upstream metadata response",
|
||||
"url", metaURL, "status", resp.StatusCode)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", nil
|
||||
http.Error(w, "package not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
var metadata map[string]any
|
||||
if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
|
||||
return "", err
|
||||
http.Error(w, "failed to parse metadata", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Expand minified Composer v2 format so that inherited fields (including
|
||||
// dist) are present on every version entry. Without this, versions that
|
||||
// inherit dist from a previous entry will appear to have no download URL.
|
||||
if metadata["minified"] == "composer/2.0" {
|
||||
h.proxy.Logger.Debug("expanding minified metadata", "url", metaURL)
|
||||
if packages, ok := metadata["packages"].(map[string]any); ok {
|
||||
for pkgName, versions := range packages {
|
||||
versionList, ok := versions.([]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
packages[pkgName] = expandMinifiedVersions(versionList)
|
||||
}
|
||||
}
|
||||
// Find the download URL for this version
|
||||
downloadURL := h.findDownloadURL(metadata, packageName, version)
|
||||
if downloadURL == "" {
|
||||
http.Error(w, "version not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
url := h.findDownloadURL(metadata, packageName, version)
|
||||
h.proxy.Logger.Debug("download URL lookup result",
|
||||
"url", metaURL, "package", packageName, "version", version,
|
||||
"download_url", url)
|
||||
return url, nil
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// findDownloadURL finds the dist URL for a specific version in metadata.
|
||||
|
|
|
|||
|
|
@ -1,16 +1,13 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
)
|
||||
|
||||
func TestComposerRewriteMetadata(t *testing.T) {
|
||||
|
|
@ -180,80 +177,6 @@ func TestComposerRewriteMetadataMinifiedDevReset(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestComposerRewriteMetadataUnset(t *testing.T) {
|
||||
h := &ComposerHandler{
|
||||
proxy: &Proxy{Logger: slog.Default()},
|
||||
proxyURL: "http://localhost:8080",
|
||||
}
|
||||
|
||||
// In the minified format, "__unset" removes a field from the inherited
|
||||
// state. v1.29.0 has require-dev, v1.28.0 unsets it, v1.27.0 inherits the
|
||||
// unset state. Composer rejects metadata where require-dev (or any link
|
||||
// field) is the literal string "__unset" rather than an object.
|
||||
input := `{
|
||||
"minified": "composer/2.0",
|
||||
"packages": {
|
||||
"venturecraft/revisionable": [
|
||||
{
|
||||
"name": "venturecraft/revisionable",
|
||||
"version": "1.29.0",
|
||||
"require": {"php": ">=5.4"},
|
||||
"require-dev": {"orchestra/testbench": "~3.0"},
|
||||
"dist": {"url": "https://example.com/a.zip", "type": "zip"}
|
||||
},
|
||||
{
|
||||
"version": "1.28.0",
|
||||
"require-dev": "__unset"
|
||||
},
|
||||
{
|
||||
"version": "1.27.0"
|
||||
},
|
||||
{
|
||||
"version": "1.26.0",
|
||||
"require-dev": {"foo/bar": "1.0"}
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
output, err := h.rewriteMetadata([]byte(input))
|
||||
if err != nil {
|
||||
t.Fatalf("rewriteMetadata failed: %v", err)
|
||||
}
|
||||
|
||||
var result map[string]any
|
||||
if err := json.Unmarshal(output, &result); err != nil {
|
||||
t.Fatalf("failed to parse output: %v", err)
|
||||
}
|
||||
|
||||
versions := result["packages"].(map[string]any)["venturecraft/revisionable"].([]any)
|
||||
if len(versions) != 4 {
|
||||
t.Fatalf("expected 4 versions, got %d", len(versions))
|
||||
}
|
||||
|
||||
byVersion := map[string]map[string]any{}
|
||||
for _, v := range versions {
|
||||
vmap := v.(map[string]any)
|
||||
byVersion[vmap["version"].(string)] = vmap
|
||||
}
|
||||
|
||||
if _, ok := byVersion["1.29.0"]["require-dev"].(map[string]any); !ok {
|
||||
t.Errorf("1.29.0 require-dev should be an object, got %T", byVersion["1.29.0"]["require-dev"])
|
||||
}
|
||||
if rd, ok := byVersion["1.28.0"]["require-dev"]; ok {
|
||||
t.Errorf("1.28.0 require-dev should be absent, got %v", rd)
|
||||
}
|
||||
if rd, ok := byVersion["1.27.0"]["require-dev"]; ok {
|
||||
t.Errorf("1.27.0 require-dev should be absent (inherited unset), got %v", rd)
|
||||
}
|
||||
if _, ok := byVersion["1.26.0"]["require-dev"].(map[string]any); !ok {
|
||||
t.Errorf("1.26.0 require-dev should be an object, got %T", byVersion["1.26.0"]["require-dev"])
|
||||
}
|
||||
if _, ok := byVersion["1.27.0"]["require"].(map[string]any); !ok {
|
||||
t.Error("1.27.0 should still inherit require from 1.29.0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) {
|
||||
now := time.Now()
|
||||
old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339)
|
||||
|
|
@ -468,100 +391,6 @@ func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestComposerDownloadDevVersionUsesDevMetadata is a regression test for the
|
||||
// bug that made it impossible to install a *-dev dependency from dist.
|
||||
//
|
||||
// Packagist serves development versions (e.g. "3.x-dev", "dev-master") from a
|
||||
// separate "{package}~dev.json" metadata file; the regular "{package}.json"
|
||||
// file contains only tagged releases. The download handler used to fetch only
|
||||
// the regular file, so it could never find the dist URL for a dev version and
|
||||
// returned 404 — causing Composer to silently fall back to a git clone.
|
||||
//
|
||||
// This test serves both files from a mock upstream and asserts that:
|
||||
// - the OLD behavior (regular file only) cannot resolve the dev version, and
|
||||
// - the FIXED behavior (consulting the ~dev file) does.
|
||||
func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) {
|
||||
const (
|
||||
pkg = "phpmd/phpmd"
|
||||
vendor = "phpmd"
|
||||
name = "phpmd"
|
||||
version = "3.x-dev"
|
||||
distURL = "https://api.github.com/repos/phpmd/phpmd/zipball/2a9217f60aaf27bf6ddad9188f254d020ab70745"
|
||||
)
|
||||
|
||||
// Regular metadata: tagged releases only — no dev versions.
|
||||
stableBody := `{
|
||||
"packages": {
|
||||
"phpmd/phpmd": [
|
||||
{"version": "2.15.0", "dist": {"url": "https://example.com/2.15.0.zip", "type": "zip"}}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
// ~dev metadata: where the 3.x-dev version actually lives.
|
||||
devBody := `{
|
||||
"packages": {
|
||||
"phpmd/phpmd": [
|
||||
{"version": "3.x-dev", "dist": {"url": "` + distURL + `", "type": "zip"}}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/p2/phpmd/phpmd.json":
|
||||
_, _ = w.Write([]byte(stableBody))
|
||||
case "/p2/phpmd/phpmd~dev.json":
|
||||
_, _ = w.Write([]byte(devBody))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
h := &ComposerHandler{
|
||||
proxy: testProxy(),
|
||||
repoURL: srv.URL,
|
||||
proxyURL: "http://localhost:8080",
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// OLD behavior: fetching only the regular file fails to resolve the dev
|
||||
// version, which is what produced the 404 before the fix.
|
||||
stableURL := srv.URL + "/p2/phpmd/phpmd.json"
|
||||
got, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error fetching regular metadata: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("regular metadata unexpectedly contained dev version %q (got %q); "+
|
||||
"the test no longer reproduces the original bug", version, got)
|
||||
}
|
||||
|
||||
// FIXED behavior: the handler consults the ~dev file (it is first in the
|
||||
// candidate list for dev versions) and resolves the dist URL.
|
||||
urls := h.metadataURLsForVersion(vendor, name, version)
|
||||
if len(urls) == 0 || !strings.HasSuffix(urls[0], "/p2/phpmd/phpmd~dev.json") {
|
||||
t.Fatalf("dev version should consult the ~dev metadata file first, got %v", urls)
|
||||
}
|
||||
|
||||
var resolved string
|
||||
for _, u := range urls {
|
||||
resolved, err = h.findDownloadURLFromMetadata(ctx, u, pkg, version)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error fetching metadata %q: %v", u, err)
|
||||
}
|
||||
if resolved != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if resolved != distURL {
|
||||
t.Errorf("dev version dist URL = %q, want %q", resolved, distURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposerRewriteMetadataCooldown(t *testing.T) {
|
||||
now := time.Now()
|
||||
old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339)
|
||||
|
|
|
|||
|
|
@ -84,7 +84,8 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request)
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch file")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch file", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -121,7 +122,8 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request)
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch file")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch file", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import (
|
|||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -72,7 +74,8 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, filename, upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -137,7 +140,7 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) {
|
|||
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
req.Header.Set(headerAcceptEncoding, "gzip")
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
|
|
@ -158,7 +161,7 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
body, err := h.proxy.ReadMetadata(resp.Body)
|
||||
body, err := ReadMetadata(resp.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to read response", http.StatusInternalServerError)
|
||||
return
|
||||
|
|
@ -215,7 +218,7 @@ func (h *CondaHandler) applyCooldownFiltering(body []byte) ([]byte, error) {
|
|||
continue
|
||||
}
|
||||
|
||||
packagePURL := canonicalPackagePURL("conda", name)
|
||||
packagePURL := purl.MakePURLString("conda", name, "")
|
||||
|
||||
if !h.proxy.Cooldown.IsAllowed("conda", packagePURL, publishedAt) {
|
||||
version, _ := entryMap["version"].(string)
|
||||
|
|
@ -238,5 +241,5 @@ func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
// proxyUpstream forwards a request to Anaconda without caching.
|
||||
func (h *CondaHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{headerAcceptEncoding})
|
||||
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept-Encoding"})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
)
|
||||
|
||||
func TestCondaParseFilename(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package handler
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
|
@ -12,10 +11,10 @@ import (
|
|||
|
||||
const (
|
||||
dockerHubRegistry = "https://registry-1.docker.io"
|
||||
dockerHubAuth = "https://auth.docker.io"
|
||||
blobMatchCount = 3 // full match + name + digest
|
||||
manifestMatchCount = 3 // full match + name + reference
|
||||
tagsListMatchCount = 2 // full match + name
|
||||
registrySelectorParts = 3 // upstream + name + repository
|
||||
)
|
||||
|
||||
// ContainerHandler handles OCI/Docker container registry protocol requests.
|
||||
|
|
@ -24,27 +23,18 @@ const (
|
|||
type ContainerHandler struct {
|
||||
proxy *Proxy
|
||||
registryURL string
|
||||
authURL string
|
||||
proxyURL string
|
||||
namedRegistries map[string]string
|
||||
}
|
||||
|
||||
// NewContainerHandler creates a new container registry protocol handler.
|
||||
// Named registries are selected with the repository prefix
|
||||
// upstream/{name}/, leaving unprefixed requests compatible with the Docker Hub
|
||||
// mirror behavior.
|
||||
func NewContainerHandler(proxy *Proxy, proxyURL string, namedRegistries ...map[string]string) *ContainerHandler {
|
||||
h := &ContainerHandler{
|
||||
func NewContainerHandler(proxy *Proxy, proxyURL string) *ContainerHandler {
|
||||
return &ContainerHandler{
|
||||
proxy: proxy,
|
||||
registryURL: dockerHubRegistry,
|
||||
authURL: dockerHubAuth,
|
||||
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
||||
}
|
||||
if len(namedRegistries) > 0 {
|
||||
h.namedRegistries = make(map[string]string, len(namedRegistries[0]))
|
||||
for name, registryURL := range namedRegistries[0] {
|
||||
h.namedRegistries[name] = strings.TrimSuffix(registryURL, "/")
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Routes returns the HTTP handler for container registry requests.
|
||||
|
|
@ -97,68 +87,48 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req
|
|||
return
|
||||
}
|
||||
|
||||
registryURL, upstreamName, cacheName, ok := h.registryForName(name)
|
||||
if !ok {
|
||||
h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry")
|
||||
return
|
||||
}
|
||||
h.proxy.Logger.Info("container blob request", "name", name, "digest", digest)
|
||||
|
||||
h.proxy.Logger.Info("container blob request", "name", upstreamName, "digest", digest)
|
||||
|
||||
filename := digest
|
||||
cached, err := h.proxy.GetCachedArtifact(r.Context(), "oci", cacheName, digest, filename)
|
||||
// Get auth token for upstream
|
||||
token, err := h.getAuthToken(r.Context(), name, "pull")
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("failed to check blob cache", "error", err)
|
||||
h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to check blob cache")
|
||||
return
|
||||
}
|
||||
if cached != nil {
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
if cached.ContentType != "" {
|
||||
w.Header().Set("Content-Type", cached.ContentType)
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
}
|
||||
serveArtifact(w, r.Method, cached)
|
||||
h.proxy.Logger.Error("failed to get auth token", "error", err)
|
||||
h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate")
|
||||
return
|
||||
}
|
||||
|
||||
// For HEAD requests, just proxy to upstream
|
||||
if r.Method == http.MethodHead {
|
||||
h.proxyBlobHead(w, r, registryURL, upstreamName, digest)
|
||||
h.proxyBlobHead(w, r, name, digest, token)
|
||||
return
|
||||
}
|
||||
|
||||
// Try to get from cache, or fetch from the authentication-aware upstream client.
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(
|
||||
// Try to get from cache, or fetch from upstream with auth
|
||||
filename := digest
|
||||
headers := http.Header{"Authorization": {"Bearer " + token}}
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURLWithHeaders(
|
||||
r.Context(),
|
||||
"oci",
|
||||
cacheName,
|
||||
name,
|
||||
digest, // use digest as version
|
||||
filename,
|
||||
fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, upstreamName, digest),
|
||||
fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest),
|
||||
headers,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUpstreamNotFound) {
|
||||
h.containerError(w, http.StatusNotFound, "BLOB_UNKNOWN", "blob unknown to registry")
|
||||
return
|
||||
}
|
||||
h.proxy.Logger.Error("failed to fetch blob", "error", err)
|
||||
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch blob")
|
||||
h.containerError(w, http.StatusBadGateway, "BLOB_UNKNOWN", "failed to fetch blob")
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
if result.ContentType != "" {
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
}
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// handleManifest serves immutable manifests from cache and revalidates mutable tags.
|
||||
// handleManifest proxies manifest requests to upstream.
|
||||
// Manifests change when tags are updated, so we proxy these directly.
|
||||
// Path format: {name}/manifests/{reference}
|
||||
func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request, path string) {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
|
|
@ -172,14 +142,58 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
registryURL, upstreamName, _, ok := h.registryForName(name)
|
||||
if !ok {
|
||||
h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry")
|
||||
h.proxy.Logger.Info("container manifest request", "name", name, "reference", reference)
|
||||
|
||||
// Get auth token
|
||||
token, err := h.getAuthToken(r.Context(), name, "pull")
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("failed to get auth token", "error", err)
|
||||
h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate")
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.Logger.Info("container manifest request", "name", upstreamName, "reference", reference)
|
||||
h.serveManifest(w, r, registryURL, upstreamName, reference)
|
||||
// Proxy to upstream
|
||||
upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", h.registryURL, name, reference)
|
||||
|
||||
req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil)
|
||||
if err != nil {
|
||||
h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request")
|
||||
return
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
// Forward Accept header for content negotiation
|
||||
if accept := r.Header.Get("Accept"); accept != "" {
|
||||
req.Header.Set("Accept", accept)
|
||||
} else {
|
||||
// Default accept headers for manifests
|
||||
req.Header.Set("Accept", strings.Join([]string{
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.v1+prettyjws",
|
||||
}, ", "))
|
||||
}
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("failed to fetch manifest", "error", err)
|
||||
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
// Copy relevant headers
|
||||
for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag"} {
|
||||
if v := resp.Header.Get(header); v != "" {
|
||||
w.Header().Set(header, v)
|
||||
}
|
||||
}
|
||||
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// handleTagsList proxies tag list requests to upstream.
|
||||
|
|
@ -195,13 +209,14 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
registryURL, upstreamName, _, ok := h.registryForName(name)
|
||||
if !ok {
|
||||
h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry")
|
||||
// Get auth token
|
||||
token, err := h.getAuthToken(r.Context(), name, "pull")
|
||||
if err != nil {
|
||||
h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate")
|
||||
return
|
||||
}
|
||||
|
||||
upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", registryURL, upstreamName)
|
||||
upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", h.registryURL, name)
|
||||
if r.URL.RawQuery != "" {
|
||||
upstreamURL += "?" + r.URL.RawQuery
|
||||
}
|
||||
|
|
@ -212,6 +227,8 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
|
||||
|
|
@ -224,9 +241,46 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request
|
|||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// getAuthToken gets a bearer token for the specified repository.
|
||||
// Docker Hub requires auth even for public images.
|
||||
func (h *ContainerHandler) getAuthToken(_ interface{ Done() <-chan struct{} }, repository, action string) (string, error) {
|
||||
// For Docker Hub: https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull
|
||||
authURL := fmt.Sprintf("%s/token?service=registry.docker.io&scope=repository:%s:%s",
|
||||
h.authURL, repository, action)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, authURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("auth failed with status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var tokenResp struct {
|
||||
Token string `json:"token"`
|
||||
AccessToken string `json:"access_token"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if tokenResp.Token != "" {
|
||||
return tokenResp.Token, nil
|
||||
}
|
||||
return tokenResp.AccessToken, nil
|
||||
}
|
||||
|
||||
// proxyBlobHead handles HEAD requests for blobs.
|
||||
func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, registryURL, name, digest string) {
|
||||
upstreamURL := fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, name, digest)
|
||||
func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, name, digest, token string) {
|
||||
upstreamURL := fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest)
|
||||
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodHead, upstreamURL, nil)
|
||||
if err != nil {
|
||||
|
|
@ -234,6 +288,8 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request,
|
|||
return
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
|
||||
|
|
@ -250,24 +306,6 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request,
|
|||
w.WriteHeader(resp.StatusCode)
|
||||
}
|
||||
|
||||
// registryForName resolves a client-visible OCI repository name to an upstream
|
||||
// registry and its repository name. Named upstreams use upstream/{name}/ as a
|
||||
// reserved prefix; all other names continue to target Docker Hub.
|
||||
func (h *ContainerHandler) registryForName(name string) (registryURL, upstreamName, cacheName string, ok bool) {
|
||||
parts := strings.SplitN(name, "/", registrySelectorParts)
|
||||
if len(parts) >= 2 && parts[0] == "upstream" {
|
||||
if len(parts) != registrySelectorParts || parts[2] == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
registryURL, ok = h.namedRegistries[parts[1]]
|
||||
if !ok || registryURL == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
return registryURL, parts[2], name, true
|
||||
}
|
||||
return h.registryURL, name, name, true
|
||||
}
|
||||
|
||||
// containerError writes an OCI-compliant error response.
|
||||
func (h *ContainerHandler) containerError(w http.ResponseWriter, status int, code, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
|
|
|||
|
|
@ -1,251 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
)
|
||||
|
||||
const (
|
||||
containerManifestCacheEcosystem = "oci-manifest"
|
||||
containerStaleWarning = `110 - "Response is Stale"`
|
||||
)
|
||||
|
||||
var manifestDigestReferencePattern = regexp.MustCompile(`^[a-z0-9]+:[a-f0-9]+$`)
|
||||
|
||||
type cachedContainerManifest struct {
|
||||
body []byte
|
||||
contentType string
|
||||
contentDigest string
|
||||
etag string
|
||||
size int64
|
||||
fetchedAt time.Time
|
||||
}
|
||||
|
||||
func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, reference string) {
|
||||
accept := containerManifestAccept(r)
|
||||
cacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept)
|
||||
cached, err := h.loadContainerManifest(r.Context(), cacheKey)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Warn("failed to read cached container manifest", "error", err)
|
||||
cached = nil
|
||||
}
|
||||
|
||||
immutable := manifestDigestReferencePattern.MatchString(reference)
|
||||
if cached != nil && (immutable || h.containerManifestFresh(cached)) {
|
||||
writeContainerManifest(w, r.Method, cached, false)
|
||||
return
|
||||
}
|
||||
|
||||
upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", registryURL, name, reference)
|
||||
req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil)
|
||||
if err != nil {
|
||||
h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request")
|
||||
return
|
||||
}
|
||||
req.Header.Set("Accept", accept)
|
||||
if cached != nil && cached.etag != "" {
|
||||
req.Header.Set("If-None-Match", cached.etag)
|
||||
}
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
h.serveStaleManifestOrError(w, r, cached, err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode == http.StatusNotModified && cached != nil {
|
||||
cached.fetchedAt = time.Now()
|
||||
if err := h.storeContainerManifest(r.Context(), cacheKey, cached); err != nil {
|
||||
h.proxy.Logger.Warn("failed to refresh cached container manifest", "error", err)
|
||||
}
|
||||
writeContainerManifest(w, r.Method, cached, false)
|
||||
return
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if cached != nil && shouldServeStaleManifest(resp.StatusCode) {
|
||||
writeContainerManifest(w, r.Method, cached, true)
|
||||
return
|
||||
}
|
||||
copyContainerManifestHeaders(w.Header(), resp.Header)
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodHead {
|
||||
copyContainerManifestHeaders(w.Header(), resp.Header)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
body, err := h.proxy.ReadMetadata(resp.Body)
|
||||
if err != nil {
|
||||
h.serveStaleManifestOrError(w, r, cached, fmt.Errorf("reading manifest: %w", err))
|
||||
return
|
||||
}
|
||||
manifest := &cachedContainerManifest{
|
||||
body: body,
|
||||
contentType: resp.Header.Get("Content-Type"),
|
||||
contentDigest: resp.Header.Get("Docker-Content-Digest"),
|
||||
etag: resp.Header.Get("ETag"),
|
||||
size: int64(len(body)),
|
||||
fetchedAt: time.Now(),
|
||||
}
|
||||
if manifest.contentDigest == "" {
|
||||
manifest.contentDigest = sha256Digest(body)
|
||||
}
|
||||
if err := h.storeContainerManifest(r.Context(), cacheKey, manifest); err != nil {
|
||||
h.proxy.Logger.Warn("failed to cache container manifest", "error", err)
|
||||
}
|
||||
if manifest.contentDigest != reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) {
|
||||
digestKey := h.containerManifestCacheKey(registryURL, name, manifest.contentDigest, accept)
|
||||
if err := h.storeContainerManifest(r.Context(), digestKey, manifest); err != nil {
|
||||
h.proxy.Logger.Warn("failed to cache container manifest by digest", "error", err)
|
||||
}
|
||||
}
|
||||
writeContainerManifest(w, r.Method, manifest, false)
|
||||
}
|
||||
|
||||
func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, cached *cachedContainerManifest, err error) {
|
||||
if cached != nil {
|
||||
h.proxy.Logger.Warn("upstream manifest fetch failed, serving stale cache", "error", err)
|
||||
writeContainerManifest(w, r.Method, cached, true)
|
||||
return
|
||||
}
|
||||
h.proxy.Logger.Error("failed to fetch manifest", "error", err)
|
||||
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
|
||||
}
|
||||
|
||||
func (h *ContainerHandler) containerManifestFresh(manifest *cachedContainerManifest) bool {
|
||||
return h.proxy.MetadataTTL > 0 && !manifest.fetchedAt.IsZero() && time.Since(manifest.fetchedAt) < h.proxy.MetadataTTL
|
||||
}
|
||||
|
||||
func (h *ContainerHandler) containerManifestCacheKey(registryURL, name, reference, accept string) string {
|
||||
identity := strings.Join([]string{registryURL, name, reference, accept}, "\x00")
|
||||
sum := sha256.Sum256([]byte(identity))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, error) {
|
||||
if h.proxy.DB == nil || h.proxy.Storage == nil {
|
||||
return nil, nil
|
||||
}
|
||||
entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, cacheKey)
|
||||
if err != nil || entry == nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
body, err := h.proxy.ReadMetadata(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
manifest := &cachedContainerManifest{body: body, size: int64(len(body))}
|
||||
if entry.ContentType.Valid {
|
||||
manifest.contentType = entry.ContentType.String
|
||||
}
|
||||
if entry.ContentDigest.Valid {
|
||||
manifest.contentDigest = entry.ContentDigest.String
|
||||
} else {
|
||||
manifest.contentDigest = sha256Digest(body)
|
||||
}
|
||||
if entry.ETag.Valid {
|
||||
manifest.etag = entry.ETag.String
|
||||
}
|
||||
if entry.Size.Valid {
|
||||
manifest.size = entry.Size.Int64
|
||||
}
|
||||
if entry.FetchedAt.Valid {
|
||||
manifest.fetchedAt = entry.FetchedAt.Time
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func (h *ContainerHandler) storeContainerManifest(ctx context.Context, cacheKey string, manifest *cachedContainerManifest) error {
|
||||
if h.proxy.DB == nil || h.proxy.Storage == nil {
|
||||
return nil
|
||||
}
|
||||
storagePath := metadataStoragePath(containerManifestCacheEcosystem, cacheKey)
|
||||
size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(manifest.body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("storing manifest: %w", err)
|
||||
}
|
||||
manifest.size = size
|
||||
return h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{
|
||||
Ecosystem: containerManifestCacheEcosystem,
|
||||
Name: cacheKey,
|
||||
StoragePath: storagePath,
|
||||
ETag: sql.NullString{String: manifest.etag, Valid: manifest.etag != ""},
|
||||
ContentType: sql.NullString{String: manifest.contentType, Valid: manifest.contentType != ""},
|
||||
ContentDigest: sql.NullString{String: manifest.contentDigest, Valid: manifest.contentDigest != ""},
|
||||
Size: sql.NullInt64{Int64: size, Valid: true},
|
||||
FetchedAt: sql.NullTime{Time: manifest.fetchedAt, Valid: !manifest.fetchedAt.IsZero()},
|
||||
})
|
||||
}
|
||||
|
||||
func writeContainerManifest(w http.ResponseWriter, method string, manifest *cachedContainerManifest, stale bool) {
|
||||
if manifest.contentType != "" {
|
||||
w.Header().Set("Content-Type", manifest.contentType)
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(manifest.size, 10))
|
||||
if manifest.contentDigest != "" {
|
||||
w.Header().Set("Docker-Content-Digest", manifest.contentDigest)
|
||||
}
|
||||
if manifest.etag != "" {
|
||||
w.Header().Set("ETag", manifest.etag)
|
||||
}
|
||||
if stale {
|
||||
w.Header().Set("Warning", containerStaleWarning)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if method != http.MethodHead {
|
||||
_, _ = w.Write(manifest.body)
|
||||
}
|
||||
}
|
||||
|
||||
func containerManifestAccept(r *http.Request) string {
|
||||
if accept := r.Header.Get("Accept"); accept != "" {
|
||||
return accept
|
||||
}
|
||||
return strings.Join([]string{
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.v1+prettyjws",
|
||||
}, ", ")
|
||||
}
|
||||
|
||||
func copyContainerManifestHeaders(destination, source http.Header) {
|
||||
for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag", "WWW-Authenticate"} {
|
||||
if value := source.Get(header); value != "" {
|
||||
destination.Set(header, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func shouldServeStaleManifest(status int) bool {
|
||||
return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError
|
||||
}
|
||||
|
||||
func sha256Digest(body []byte) string {
|
||||
digest := sha256.Sum256(body)
|
||||
return "sha256:" + hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
|
@ -1,15 +1,16 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient"
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
|
|
@ -134,568 +135,90 @@ func TestContainerHandler_parseTagsListPath(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_NamedOCIRegistryServesHelmArtifacts(t *testing.T) {
|
||||
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
|
||||
manifest := `{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.helm.config.v1+json"},"layers":[{"mediaType":"application/vnd.cncf.helm.chart.content.v1.tar+gzip","digest":"` + digest + `"}]}`
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v2/owner/demo/manifests/1.0.0":
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
_, _ = io.WriteString(w, manifest)
|
||||
case "/v2/owner/demo/blobs/" + digest:
|
||||
w.Header().Set("Content-Type", "application/vnd.cncf.helm.chart.content.v1.tar+gzip")
|
||||
_, _ = io.WriteString(w, "chart archive")
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
|
||||
proxy.Fetcher = fetcher
|
||||
t.Cleanup(func() { _ = fetcher.Close() })
|
||||
h := NewContainerHandler(proxy, "http://proxy.example", map[string]string{"ghcr": upstream.URL})
|
||||
|
||||
manifestResponse := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(manifestResponse,
|
||||
httptest.NewRequest(http.MethodGet, "/upstream/ghcr/owner/demo/manifests/1.0.0", nil))
|
||||
if manifestResponse.Code != http.StatusOK {
|
||||
t.Fatalf("manifest status = %d, want 200: %s", manifestResponse.Code, manifestResponse.Body.String())
|
||||
}
|
||||
if got := manifestResponse.Header().Get("Content-Type"); got != "application/vnd.oci.image.manifest.v1+json" {
|
||||
t.Errorf("manifest Content-Type = %q", got)
|
||||
}
|
||||
|
||||
blobResponse := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(blobResponse,
|
||||
httptest.NewRequest(http.MethodGet, "/upstream/ghcr/owner/demo/blobs/"+digest, nil))
|
||||
if blobResponse.Code != http.StatusOK {
|
||||
t.Fatalf("blob status = %d, want 200: %s", blobResponse.Code, blobResponse.Body.String())
|
||||
}
|
||||
if got := blobResponse.Header().Get("Content-Type"); got != "application/vnd.cncf.helm.chart.content.v1.tar+gzip" {
|
||||
t.Errorf("blob Content-Type = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_BlobDownload_DiscoversBearerChallenge(t *testing.T) {
|
||||
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
|
||||
registryRequests := 0
|
||||
tokenRequests := 0
|
||||
var upstream *httptest.Server
|
||||
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/token":
|
||||
tokenRequests++
|
||||
func TestContainerHandler_BlobDownload_CachesWithAuth(t *testing.T) {
|
||||
// Set up a mock auth server that returns a token
|
||||
authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"token": "discovered-token",
|
||||
"expires_in": 3600,
|
||||
})
|
||||
case "/v2/library/nginx/blobs/" + digest:
|
||||
registryRequests++
|
||||
if r.Header.Get("Authorization") != "Bearer discovered-token" {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="registry.test",scope="repository:library/nginx:pull"`)
|
||||
http.Error(w, "authentication required", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = io.WriteString(w, "upstream blob")
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"token": "test-token-123"})
|
||||
}))
|
||||
defer upstream.Close()
|
||||
defer authServer.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
authTransport := upstreamhttp.NewTransport(http.DefaultTransport, nil)
|
||||
client := &http.Client{Transport: authTransport}
|
||||
artifactFetcher := fetch.NewFetcher(
|
||||
fetch.WithHTTPClient(client),
|
||||
fetch.WithMaxRetries(0),
|
||||
)
|
||||
t.Cleanup(func() { _ = artifactFetcher.Close() })
|
||||
proxy.Fetcher = artifactFetcher
|
||||
proxy.HTTPClient = client
|
||||
// Set up mock fetcher that captures headers
|
||||
var capturedHeaders http.Header
|
||||
mf := &mockFetcherWithHeaders{
|
||||
fetchFn: func(_ context.Context, _ string, headers http.Header) (*fetch.Artifact, error) {
|
||||
capturedHeaders = headers
|
||||
return &fetch.Artifact{
|
||||
Body: io.NopCloser(bytes.NewReader([]byte("blob-content"))),
|
||||
Size: 12,
|
||||
ContentType: "application/octet-stream",
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
db, err := database.Create(dir + "/test.db")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test database: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
store := newMockStorage()
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
proxy := &Proxy{
|
||||
DB: db,
|
||||
Storage: store,
|
||||
Fetcher: mf,
|
||||
Logger: logger,
|
||||
HTTPClient: &http.Client{},
|
||||
}
|
||||
|
||||
h := &ContainerHandler{
|
||||
proxy: proxy,
|
||||
registryURL: upstream.URL,
|
||||
registryURL: "https://registry-1.docker.io",
|
||||
authURL: authServer.URL,
|
||||
proxyURL: "http://localhost:8080",
|
||||
}
|
||||
|
||||
for range 2 {
|
||||
req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil)
|
||||
handler := h.Routes()
|
||||
req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
}
|
||||
if got := w.Body.String(); got != "upstream blob" {
|
||||
t.Errorf("body = %q, want %q", got, "upstream blob")
|
||||
}
|
||||
}
|
||||
|
||||
if tokenRequests != 1 {
|
||||
t.Errorf("token requests = %d, want 1", tokenRequests)
|
||||
}
|
||||
if registryRequests != 2 {
|
||||
t.Errorf("registry requests = %d, want 2", registryRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_CachedImagePullSurvivesRegistryAndTokenOutages(t *testing.T) {
|
||||
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
|
||||
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`
|
||||
blob := "cached image blob"
|
||||
registryAvailable := true
|
||||
tokenAvailable := true
|
||||
registryRequests := 0
|
||||
tokenRequests := 0
|
||||
|
||||
tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
tokenRequests++
|
||||
if !tokenAvailable {
|
||||
http.Error(w, "token service unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"token": "discovered-token",
|
||||
"expires_in": 3600,
|
||||
})
|
||||
}))
|
||||
defer tokenServer.Close()
|
||||
|
||||
registryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
registryRequests++
|
||||
if !registryAvailable {
|
||||
http.Error(w, "registry unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("Authorization") != "Bearer discovered-token" {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenServer.URL+`",service="registry.test",scope="repository:library/nginx:pull"`)
|
||||
http.Error(w, "authentication required", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
switch r.URL.Path {
|
||||
case "/v2/library/nginx/manifests/latest":
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
_, _ = io.WriteString(w, manifest)
|
||||
case "/v2/library/nginx/blobs/" + digest:
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = io.WriteString(w, blob)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer registryServer.Close()
|
||||
|
||||
warmProxy, db, store, _ := setupTestProxy(t)
|
||||
warmClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)}
|
||||
warmFetcher := fetch.NewFetcher(
|
||||
fetch.WithHTTPClient(warmClient),
|
||||
fetch.WithMaxRetries(0),
|
||||
)
|
||||
t.Cleanup(func() { _ = warmFetcher.Close() })
|
||||
warmProxy.Fetcher = warmFetcher
|
||||
warmProxy.HTTPClient = warmClient
|
||||
warmProxy.MetadataTTL = time.Hour
|
||||
warmHandler := (&ContainerHandler{
|
||||
proxy: warmProxy,
|
||||
registryURL: registryServer.URL,
|
||||
proxyURL: "http://localhost:8080",
|
||||
}).Routes()
|
||||
|
||||
for _, request := range []struct {
|
||||
path string
|
||||
body string
|
||||
}{
|
||||
{path: "/library/nginx/manifests/latest", body: manifest},
|
||||
{path: "/library/nginx/blobs/" + digest, body: blob},
|
||||
} {
|
||||
response := httptest.NewRecorder()
|
||||
warmHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil))
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("warming %s: status = %d, want %d; body: %s", request.path, response.Code, http.StatusOK, response.Body.String())
|
||||
}
|
||||
if got := response.Body.String(); got != request.body {
|
||||
t.Fatalf("warming %s: body = %q, want %q", request.path, got, request.body)
|
||||
}
|
||||
}
|
||||
|
||||
warmRegistryRequests := registryRequests
|
||||
warmTokenRequests := tokenRequests
|
||||
registryAvailable = false
|
||||
tokenAvailable = false
|
||||
|
||||
offlineClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)}
|
||||
offlineFetcher := fetch.NewFetcher(
|
||||
fetch.WithHTTPClient(offlineClient),
|
||||
fetch.WithMaxRetries(0),
|
||||
)
|
||||
t.Cleanup(func() { _ = offlineFetcher.Close() })
|
||||
offlineProxy := NewProxy(db, store, offlineFetcher, fetch.NewResolver(), warmProxy.Logger)
|
||||
offlineProxy.HTTPClient = offlineClient
|
||||
offlineProxy.MetadataTTL = time.Hour
|
||||
offlineHandler := (&ContainerHandler{
|
||||
proxy: offlineProxy,
|
||||
registryURL: registryServer.URL,
|
||||
proxyURL: "http://localhost:8080",
|
||||
}).Routes()
|
||||
|
||||
for _, request := range []struct {
|
||||
name string
|
||||
path string
|
||||
body string
|
||||
}{
|
||||
{name: "tag manifest", path: "/library/nginx/manifests/latest", body: manifest},
|
||||
{name: "digest manifest", path: "/library/nginx/manifests/" + digest, body: manifest},
|
||||
{name: "blob", path: "/library/nginx/blobs/" + digest, body: blob},
|
||||
} {
|
||||
t.Run(request.name, func(t *testing.T) {
|
||||
response := httptest.NewRecorder()
|
||||
offlineHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil))
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", response.Code, http.StatusOK, response.Body.String())
|
||||
}
|
||||
if got := response.Body.String(); got != request.body {
|
||||
t.Errorf("body = %q, want %q", got, request.body)
|
||||
}
|
||||
if got := response.Header().Get("Docker-Content-Digest"); got != digest {
|
||||
t.Errorf("Docker-Content-Digest = %q, want %q", got, digest)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if registryRequests != warmRegistryRequests {
|
||||
t.Errorf("offline registry requests = %d, want 0", registryRequests-warmRegistryRequests)
|
||||
}
|
||||
if tokenRequests != warmTokenRequests {
|
||||
t.Errorf("offline token requests = %d, want 0", tokenRequests-warmTokenRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) {
|
||||
proxy, db, store, fetcher := setupTestProxy(t)
|
||||
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
|
||||
seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob")
|
||||
|
||||
upstreamRequests := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
upstreamRequests++
|
||||
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
h := &ContainerHandler{
|
||||
proxy: proxy,
|
||||
registryURL: upstream.URL,
|
||||
proxyURL: "http://localhost:8080",
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
handler.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
t.Errorf("got status %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
}
|
||||
if got := w.Body.String(); got != "cached blob" {
|
||||
t.Errorf("body = %q, want %q", got, "cached blob")
|
||||
|
||||
// Verify auth header was passed to the fetcher
|
||||
if capturedHeaders == nil {
|
||||
t.Fatal("expected headers to be passed to fetcher, got nil")
|
||||
}
|
||||
if upstreamRequests != 0 {
|
||||
t.Errorf("upstream requests = %d, want 0", upstreamRequests)
|
||||
auth := capturedHeaders.Get("Authorization")
|
||||
if auth != "Bearer test-token-123" {
|
||||
t.Errorf("Authorization = %q, want %q", auth, "Bearer test-token-123")
|
||||
}
|
||||
if fetcher.fetchCalled {
|
||||
t.Error("fetcher should not be called on cache hit")
|
||||
|
||||
// Verify response headers
|
||||
if got := w.Header().Get("Docker-Content-Digest"); got != "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" {
|
||||
t.Errorf("Docker-Content-Digest = %q, want digest", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) {
|
||||
proxy, db, store, fetcher := setupTestProxy(t)
|
||||
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
|
||||
seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob")
|
||||
|
||||
upstreamRequests := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
upstreamRequests++
|
||||
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
|
||||
h := &ContainerHandler{
|
||||
proxy: proxy,
|
||||
registryURL: upstream.URL,
|
||||
proxyURL: "http://localhost:8080",
|
||||
// mockFetcherWithHeaders captures headers passed to FetchWithHeaders.
|
||||
type mockFetcherWithHeaders struct {
|
||||
fetchFn func(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
}
|
||||
if got := w.Header().Get("Docker-Content-Digest"); got != digest {
|
||||
t.Errorf("Docker-Content-Digest = %q, want %q", got, digest)
|
||||
}
|
||||
if got := w.Header().Get("Content-Length"); got != "11" {
|
||||
t.Errorf("Content-Length = %q, want %q", got, "11")
|
||||
}
|
||||
if w.Body.Len() != 0 {
|
||||
t.Errorf("HEAD response body length = %d, want 0", w.Body.Len())
|
||||
}
|
||||
if upstreamRequests != 0 {
|
||||
t.Errorf("upstream requests = %d, want 0", upstreamRequests)
|
||||
}
|
||||
if fetcher.fetchCalled {
|
||||
t.Error("fetcher should not be called on cache hit")
|
||||
}
|
||||
func (f *mockFetcherWithHeaders) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) {
|
||||
return f.FetchWithHeaders(ctx, url, nil)
|
||||
}
|
||||
|
||||
func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) {
|
||||
proxy, db, store, fetcher := setupTestProxy(t)
|
||||
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
|
||||
seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob")
|
||||
store.signedURL = "https://storage.example.test/cached-blob?signature=test"
|
||||
proxy.DirectServe = true
|
||||
|
||||
h := &ContainerHandler{
|
||||
proxy: proxy,
|
||||
registryURL: "https://registry.example.test",
|
||||
proxyURL: "http://localhost:8080",
|
||||
func (f *mockFetcherWithHeaders) FetchWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) {
|
||||
return f.fetchFn(ctx, url, headers)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusFound {
|
||||
t.Fatalf("status = %d, want %d", w.Code, http.StatusFound)
|
||||
}
|
||||
if got := w.Header().Get("Location"); got != store.signedURL {
|
||||
t.Errorf("Location = %q, want %q", got, store.signedURL)
|
||||
}
|
||||
wantETag := `"` + sha256Hex("cached blob") + `"`
|
||||
if got := w.Header().Get("ETag"); got != wantETag {
|
||||
t.Errorf("ETag = %q, want %q", got, wantETag)
|
||||
}
|
||||
if w.Body.Len() != 0 {
|
||||
t.Errorf("HEAD response body length = %d, want 0", w.Body.Len())
|
||||
}
|
||||
if fetcher.fetchCalled {
|
||||
t.Error("fetcher should not be called on cache hit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_ManifestByDigest_CacheHitSkipsUpstream(t *testing.T) {
|
||||
digest := "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`
|
||||
upstreamAvailable := true
|
||||
upstreamRequests := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
upstreamRequests++
|
||||
if !upstreamAvailable {
|
||||
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/v2/library/nginx/manifests/"+digest {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
w.Header().Set("ETag", `"manifest-etag"`)
|
||||
if r.Method != http.MethodHead {
|
||||
_, _ = io.WriteString(w, manifest)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"}
|
||||
|
||||
first := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil))
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String())
|
||||
}
|
||||
if first.Body.String() != manifest {
|
||||
t.Fatalf("initial body = %q, want %q", first.Body.String(), manifest)
|
||||
}
|
||||
|
||||
upstreamAvailable = false
|
||||
second := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil))
|
||||
if second.Code != http.StatusOK {
|
||||
t.Fatalf("cached status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String())
|
||||
}
|
||||
if second.Body.String() != manifest {
|
||||
t.Errorf("cached body = %q, want %q", second.Body.String(), manifest)
|
||||
}
|
||||
if got := second.Header().Get("Docker-Content-Digest"); got != digest {
|
||||
t.Errorf("cached Docker-Content-Digest = %q, want %q", got, digest)
|
||||
}
|
||||
|
||||
head := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/"+digest, nil))
|
||||
if head.Code != http.StatusOK {
|
||||
t.Fatalf("cached HEAD status = %d, want %d", head.Code, http.StatusOK)
|
||||
}
|
||||
wantLength := strconv.Itoa(len(manifest))
|
||||
if got := head.Header().Get("Content-Length"); got != wantLength {
|
||||
t.Errorf("cached HEAD Content-Length = %q, want %q", got, wantLength)
|
||||
}
|
||||
if head.Body.Len() != 0 {
|
||||
t.Errorf("cached HEAD body length = %d, want 0", head.Body.Len())
|
||||
}
|
||||
if upstreamRequests != 1 {
|
||||
t.Errorf("upstream requests = %d, want 1", upstreamRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_ManifestByTag_UsesStaleCacheOnUpstreamFailure(t *testing.T) {
|
||||
digest := "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json"}`
|
||||
upstreamAvailable := true
|
||||
upstreamRequests := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
upstreamRequests++
|
||||
if !upstreamAvailable {
|
||||
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
_, _ = io.WriteString(w, manifest)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
proxy.MetadataTTL = 0
|
||||
h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"}
|
||||
|
||||
first := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil))
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String())
|
||||
}
|
||||
|
||||
upstreamAvailable = false
|
||||
second := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil))
|
||||
if second.Code != http.StatusOK {
|
||||
t.Fatalf("stale status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String())
|
||||
}
|
||||
if second.Body.String() != manifest {
|
||||
t.Errorf("stale body = %q, want %q", second.Body.String(), manifest)
|
||||
}
|
||||
if got := second.Header().Get("Warning"); got != `110 - "Response is Stale"` {
|
||||
t.Errorf("Warning = %q, want stale warning", got)
|
||||
}
|
||||
if got := second.Header().Get("Docker-Content-Digest"); got != digest {
|
||||
t.Errorf("stale Docker-Content-Digest = %q, want %q", got, digest)
|
||||
}
|
||||
if upstreamRequests != 2 {
|
||||
t.Errorf("upstream requests = %d, want 2", upstreamRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_ManifestByTag_CachesDigestAlias(t *testing.T) {
|
||||
digest := "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
|
||||
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`
|
||||
upstreamAvailable := true
|
||||
upstreamRequests := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
upstreamRequests++
|
||||
if !upstreamAvailable {
|
||||
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/v2/library/nginx/manifests/latest" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
w.Header().Set("Docker-Content-Digest", digest)
|
||||
_, _ = io.WriteString(w, manifest)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"}
|
||||
|
||||
first := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil))
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("tag status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String())
|
||||
}
|
||||
|
||||
upstreamAvailable = false
|
||||
byDigest := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(byDigest, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil))
|
||||
if byDigest.Code != http.StatusOK {
|
||||
t.Fatalf("digest status = %d, want %d; body: %s", byDigest.Code, http.StatusOK, byDigest.Body.String())
|
||||
}
|
||||
if byDigest.Body.String() != manifest {
|
||||
t.Errorf("digest body = %q, want %q", byDigest.Body.String(), manifest)
|
||||
}
|
||||
if got := byDigest.Header().Get("Docker-Content-Digest"); got != digest {
|
||||
t.Errorf("Docker-Content-Digest = %q, want %q", got, digest)
|
||||
}
|
||||
if upstreamRequests != 1 {
|
||||
t.Errorf("upstream requests = %d, want 1", upstreamRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHandler_ManifestByTag_StaleHeadChecksUpstream(t *testing.T) {
|
||||
oldDigest := "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
||||
newDigest := "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
|
||||
currentDigest := oldDigest
|
||||
upstreamRequests := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
upstreamRequests++
|
||||
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
w.Header().Set("Docker-Content-Digest", currentDigest)
|
||||
w.Header().Set("ETag", `"`+currentDigest+`"`)
|
||||
if r.Method != http.MethodHead {
|
||||
_, _ = io.WriteString(w, `{"schemaVersion":2}`)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
proxy.MetadataTTL = 0
|
||||
h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"}
|
||||
|
||||
first := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil))
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("initial status = %d, want %d", first.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
currentDigest = newDigest
|
||||
head := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/latest", nil))
|
||||
if head.Code != http.StatusOK {
|
||||
t.Fatalf("HEAD status = %d, want %d", head.Code, http.StatusOK)
|
||||
}
|
||||
if got := head.Header().Get("Docker-Content-Digest"); got != newDigest {
|
||||
t.Errorf("Docker-Content-Digest = %q, want %q", got, newDigest)
|
||||
}
|
||||
if upstreamRequests != 2 {
|
||||
t.Errorf("upstream requests = %d, want 2", upstreamRequests)
|
||||
}
|
||||
func (f *mockFetcherWithHeaders) Head(_ context.Context, _ string) (int64, string, error) {
|
||||
return 0, "", nil
|
||||
}
|
||||
|
||||
func TestContainerHandler_Routes_VersionCheck(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -72,7 +72,8 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, version, filename, upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -106,7 +107,8 @@ func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Reques
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, storageVersion, filename, upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -157,5 +159,5 @@ func (h *CRANHandler) proxyCached(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
// proxyUpstream forwards a request to CRAN without caching.
|
||||
func (h *CRANHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{headerAcceptEncoding})
|
||||
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept-Encoding"})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,13 +21,10 @@ type DebianHandler struct {
|
|||
}
|
||||
|
||||
// NewDebianHandler creates a new Debian/APT protocol handler.
|
||||
func NewDebianHandler(proxy *Proxy, proxyURL string, upstreamURL string) *DebianHandler {
|
||||
if upstreamURL == "" {
|
||||
upstreamURL = debianUpstream
|
||||
}
|
||||
func NewDebianHandler(proxy *Proxy, proxyURL string) *DebianHandler {
|
||||
return &DebianHandler{
|
||||
proxy: proxy,
|
||||
upstreamURL: strings.TrimSuffix(upstreamURL, "/"),
|
||||
upstreamURL: debianUpstream,
|
||||
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
||||
}
|
||||
}
|
||||
|
|
@ -84,7 +81,8 @@ func (h *DebianHandler) handlePackageDownload(w http.ResponseWriter, r *http.Req
|
|||
result, err := h.proxy.GetOrFetchArtifactFromURL(
|
||||
r.Context(), "deb", name, version, filename, downloadURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
h.proxy.Logger.Error("failed to get debian package", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,17 +12,12 @@ func TestDebianHandler_parsePoolPath(t *testing.T) {
|
|||
{"pool/main/libn/libncurses/libncurses6_6.2-1_amd64.deb", "libncurses6", "6.2-1", "amd64"},
|
||||
{"pool/contrib/v/virtualbox/virtualbox_6.1.38-1_amd64.deb", "virtualbox", "6.1.38-1", "amd64"},
|
||||
{"pool/main/g/git/git_2.39.2-1_arm64.deb", "git", "2.39.2-1", "arm64"},
|
||||
{
|
||||
"pool/universe/n/nmap/nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb",
|
||||
"nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64",
|
||||
},
|
||||
{"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"},
|
||||
{"invalid/path", "", "", ""},
|
||||
{"pool/main/n/nginx/nginx.deb", "", "", ""},
|
||||
})
|
||||
}
|
||||
|
||||
func TestDebianHandler_Routes(t *testing.T) {
|
||||
h := NewDebianHandler(nil, "http://localhost:8080", "")
|
||||
h := NewDebianHandler(nil, "http://localhost:8080")
|
||||
assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ func seedPackageWithPURL(t *testing.T, db *database.DB, store *mockStorage, ecos
|
|||
Filename: filename,
|
||||
UpstreamURL: "https://example.com/" + filename,
|
||||
StoragePath: sql.NullString{String: storagePath, Valid: true},
|
||||
ContentHash: sql.NullString{String: sha256Hex(content), Valid: true},
|
||||
ContentHash: sql.NullString{String: "abc123", Valid: true},
|
||||
Size: sql.NullInt64{Int64: int64(len(content)), Valid: true},
|
||||
ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
|
||||
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
|
|
@ -673,7 +673,7 @@ func TestMavenHandler_DownloadCacheHit(t *testing.T) {
|
|||
proxy, db, store, _ := setupTestProxy(t)
|
||||
seedPackageWithPURL(t, db, store, "maven", "com.google.guava:guava", "32.1.3-jre", "guava-32.1.3-jre.jar", "jar content")
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", "", "")
|
||||
h := NewMavenHandler(proxy, "http://localhost")
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
|
|
@ -730,7 +730,7 @@ func TestMavenHandler_MetadataProxied(t *testing.T) {
|
|||
|
||||
func TestMavenHandler_EmptyPathNotFound(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewMavenHandler(proxy, "http://localhost", "", "")
|
||||
h := NewMavenHandler(proxy, "http://localhost")
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
|
|
@ -748,7 +748,7 @@ func TestMavenHandler_EmptyPathNotFound(t *testing.T) {
|
|||
func TestMavenHandler_ArtifactExtensions(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
|
||||
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"}
|
||||
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib"}
|
||||
for _, ext := range extensions {
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("artifact")),
|
||||
|
|
@ -756,7 +756,7 @@ func TestMavenHandler_ArtifactExtensions(t *testing.T) {
|
|||
}
|
||||
fetcher.fetchCalled = false
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", "", "")
|
||||
h := NewMavenHandler(proxy, "http://localhost")
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Errorf("should not proxy artifact file %s to upstream", ext)
|
||||
|
|
@ -789,7 +789,7 @@ func TestMavenHandler_CacheMiss(t *testing.T) {
|
|||
ContentType: "application/java-archive",
|
||||
}
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", "", "")
|
||||
h := NewMavenHandler(proxy, "http://localhost")
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
|
|
@ -809,274 +809,6 @@ func TestMavenHandler_CacheMiss(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestMavenHandler_GradlePluginMarkerFallbackAndCache(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
markerPath string
|
||||
}{
|
||||
{
|
||||
name: "Spotless",
|
||||
markerPath: "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom",
|
||||
},
|
||||
{
|
||||
name: "BenManes",
|
||||
markerPath: "/com/github/ben-manes/versions/com.github.ben-manes.versions.gradle.plugin/0.54.0/com.github.ben-manes.versions.gradle.plugin-0.54.0.pom",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
|
||||
primaryUpstream := "https://repo1.maven.org/maven2"
|
||||
pluginPortalUpstream := "https://plugins.gradle.org/m2"
|
||||
primaryURL := primaryUpstream + tt.markerPath
|
||||
|
||||
fetcher.fetchErrByURL = map[string]error{
|
||||
primaryURL: ErrUpstreamNotFound,
|
||||
}
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("<project/>")),
|
||||
ContentType: "application/xml",
|
||||
}
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + tt.markerPath)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
if string(body) != "<project/>" {
|
||||
t.Fatalf("body = %q, want %q", body, "<project/>")
|
||||
}
|
||||
|
||||
wantFallbackURL := pluginPortalUpstream + tt.markerPath
|
||||
if fetcher.fetchedURL != wantFallbackURL {
|
||||
t.Fatalf("fallback URL = %q, want %q", fetcher.fetchedURL, wantFallbackURL)
|
||||
}
|
||||
|
||||
fetcher.fetchCalled = false
|
||||
resp, err = http.Get(srv.URL + tt.markerPath)
|
||||
if err != nil {
|
||||
t.Fatalf("second request failed: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("second status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
if fetcher.fetchCalled {
|
||||
t.Fatal("expected plugin marker POM to be served from cache on second request")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) {
|
||||
paths := map[string]string{
|
||||
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1": "sha1",
|
||||
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha256": "sha256",
|
||||
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.md5": "md5",
|
||||
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "<metadata/>",
|
||||
}
|
||||
|
||||
primaryHits := map[string]int{}
|
||||
pluginHits := map[string]int{}
|
||||
|
||||
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
primaryHits[r.URL.Path]++
|
||||
if _, ok := paths[r.URL.Path]; ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path)
|
||||
}))
|
||||
defer primary.Close()
|
||||
|
||||
pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
pluginHits[r.URL.Path]++
|
||||
body, ok := paths[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, body)
|
||||
}))
|
||||
defer pluginPortal.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = primary.Client()
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
for reqPath, wantBody := range paths {
|
||||
resp, err := http.Get(srv.URL + reqPath)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s failed: %v", reqPath, err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
if string(body) != wantBody {
|
||||
t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody)
|
||||
}
|
||||
|
||||
if primaryHits[reqPath] == 0 {
|
||||
t.Fatalf("GET %s did not hit primary upstream", reqPath)
|
||||
}
|
||||
if pluginHits[reqPath] == 0 {
|
||||
t.Fatalf("GET %s did not hit plugin portal fallback", reqPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMavenHandler_GradlePluginImplementationMetadataFallback(t *testing.T) {
|
||||
paths := map[string]string{
|
||||
"/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha1": "impl-sha1",
|
||||
"/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha256": "impl-sha256",
|
||||
}
|
||||
|
||||
primaryHits := map[string]int{}
|
||||
pluginHits := map[string]int{}
|
||||
|
||||
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
primaryHits[r.URL.Path]++
|
||||
if _, ok := paths[r.URL.Path]; ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path)
|
||||
}))
|
||||
defer primary.Close()
|
||||
|
||||
pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
pluginHits[r.URL.Path]++
|
||||
body, ok := paths[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, body)
|
||||
}))
|
||||
defer pluginPortal.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = primary.Client()
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
for reqPath, wantBody := range paths {
|
||||
resp, err := http.Get(srv.URL + reqPath)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s failed: %v", reqPath, err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
if string(body) != wantBody {
|
||||
t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody)
|
||||
}
|
||||
|
||||
if primaryHits[reqPath] == 0 {
|
||||
t.Fatalf("GET %s did not hit primary upstream", reqPath)
|
||||
}
|
||||
if pluginHits[reqPath] == 0 {
|
||||
t.Fatalf("GET %s did not hit plugin portal fallback", reqPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMavenHandler_GradlePluginImplementation_FallbackToPluginPortal(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
|
||||
primaryUpstream := "https://repo1.maven.org/maven2"
|
||||
pluginPortalUpstream := "https://plugins.gradle.org/m2"
|
||||
implPath := "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar"
|
||||
primaryURL := primaryUpstream + implPath
|
||||
pluginPortalURL := pluginPortalUpstream + implPath
|
||||
|
||||
fetcher.fetchErrByURL = map[string]error{
|
||||
primaryURL: ErrUpstreamNotFound,
|
||||
}
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("plugin impl jar")),
|
||||
ContentType: "application/java-archive",
|
||||
}
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + implPath)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
if string(body) != "plugin impl jar" {
|
||||
t.Fatalf("body = %q, want %q", body, "plugin impl jar")
|
||||
}
|
||||
|
||||
if fetcher.fetchedURL != pluginPortalURL {
|
||||
t.Fatalf("implementation artifact should fallback to plugin portal; fetched URL = %q, want %q", fetcher.fetchedURL, pluginPortalURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMavenHandler_GradlePluginImplementation_NotFoundInBothUpstreams(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
|
||||
primaryUpstream := "https://repo1.maven.org/maven2"
|
||||
pluginPortalUpstream := "https://plugins.gradle.org/m2"
|
||||
implPath := "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar"
|
||||
primaryURL := primaryUpstream + implPath
|
||||
pluginPortalURL := pluginPortalUpstream + implPath
|
||||
|
||||
fetcher.fetchErrByURL = map[string]error{
|
||||
primaryURL: ErrUpstreamNotFound,
|
||||
pluginPortalURL: ErrUpstreamNotFound,
|
||||
}
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + implPath)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotFound)
|
||||
}
|
||||
|
||||
if fetcher.fetchedURL != pluginPortalURL {
|
||||
t.Fatalf("expected fallback attempt to plugin portal; fetched URL = %q, want %q", fetcher.fetchedURL, pluginPortalURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNuGetHandler_DownloadCacheMiss(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
|
|
@ -1165,7 +897,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) {
|
|||
ContentType: "application/vnd.debian.binary-package",
|
||||
}
|
||||
|
||||
h := NewDebianHandler(proxy, "http://localhost", "")
|
||||
h := NewDebianHandler(proxy, "http://localhost")
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type filenameDownload struct {
|
||||
ecosystem string
|
||||
suffix string
|
||||
parseErr string
|
||||
fetchErr string
|
||||
parse func(string) (name, version string)
|
||||
}
|
||||
|
||||
func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) {
|
||||
filename := r.PathValue("filename")
|
||||
if filename == "" || !strings.HasSuffix(filename, d.suffix) {
|
||||
http.Error(w, "invalid filename", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
name, version := d.parse(filename)
|
||||
if name == "" || version == "" {
|
||||
http.Error(w, d.parseErr, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
p.Logger.Info(d.ecosystem+" download request",
|
||||
"name", name, "version", version, "filename", filename)
|
||||
|
||||
result, err := p.GetOrFetchArtifact(r.Context(), d.ecosystem, name, version, filename)
|
||||
if err != nil {
|
||||
p.serveArtifactError(w, err, d.fetchErr)
|
||||
return
|
||||
}
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
|
@ -8,6 +8,8 @@ import (
|
|||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -58,13 +60,30 @@ func (h *GemHandler) Routes() http.Handler {
|
|||
|
||||
// handleDownload serves a gem file, fetching and caching from upstream if needed.
|
||||
func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
||||
h.proxy.handleFilenameDownload(w, r, filenameDownload{
|
||||
ecosystem: "gem",
|
||||
suffix: ".gem",
|
||||
parseErr: "could not parse gem filename",
|
||||
fetchErr: "failed to fetch gem",
|
||||
parse: h.parseGemFilename,
|
||||
})
|
||||
filename := r.PathValue("filename")
|
||||
if filename == "" || !strings.HasSuffix(filename, ".gem") {
|
||||
http.Error(w, "invalid filename", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract name and version from filename (e.g., "rails-7.1.0.gem")
|
||||
name, version := h.parseGemFilename(filename)
|
||||
if name == "" || version == "" {
|
||||
http.Error(w, "could not parse gem filename", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.Logger.Info("gem download request",
|
||||
"name", name, "version", version, "filename", filename)
|
||||
|
||||
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "gem", name, version, filename)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch gem", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// parseGemFilename extracts name and version from a gem filename.
|
||||
|
|
@ -163,7 +182,7 @@ func (h *GemHandler) fetchCompactIndex(r *http.Request, name string) (*http.Resp
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, hdr := range []string{"Accept", headerAcceptEncoding, "If-None-Match", "If-Modified-Since"} {
|
||||
for _, hdr := range []string{"Accept", "Accept-Encoding", "If-None-Match", "If-Modified-Since"} {
|
||||
if v := r.Header.Get(hdr); v != "" {
|
||||
req.Header.Set(hdr, v)
|
||||
}
|
||||
|
|
@ -247,7 +266,7 @@ func (h *GemHandler) fetchFilteredVersions(r *http.Request, name string) (map[st
|
|||
return nil, err
|
||||
}
|
||||
|
||||
packagePURL := canonicalPackagePURL("gem", name)
|
||||
packagePURL := purl.MakePURLString("gem", name, "")
|
||||
filtered := make(map[string]bool)
|
||||
|
||||
for _, v := range versions {
|
||||
|
|
@ -292,7 +311,7 @@ func (h *GemHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
// Copy relevant headers
|
||||
for _, h := range []string{"Accept", headerAcceptEncoding, "If-None-Match", "If-Modified-Since"} {
|
||||
for _, h := range []string{"Accept", "Accept-Encoding", "If-None-Match", "If-Modified-Since"} {
|
||||
if v := r.Header.Get(h); v != "" {
|
||||
req.Header.Set(h, v)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
)
|
||||
|
||||
func TestGemParseFilename(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -103,10 +100,6 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "golang", decodedModule, version, filename)
|
||||
if err != nil {
|
||||
if errors.Is(err, fetch.ErrNotFound) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch module", http.StatusBadGateway)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1,49 +1,9 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
func TestGoModuleDownloadUpstreamErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fetchErr error
|
||||
wantStatus int
|
||||
}{
|
||||
{
|
||||
name: "module not found",
|
||||
fetchErr: fetch.ErrNotFound,
|
||||
wantStatus: http.StatusNotFound,
|
||||
},
|
||||
{
|
||||
name: "upstream failure",
|
||||
fetchErr: errors.New("connection refused"),
|
||||
wantStatus: http.StatusBadGateway,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.fetchErr = tt.fetchErr
|
||||
handler := NewGoHandler(proxy, "http://localhost:8080")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/example.com/mod/@v/v1.0.0.zip", nil)
|
||||
resp := httptest.NewRecorder()
|
||||
handler.Routes().ServeHTTP(resp, req)
|
||||
|
||||
if resp.Code != tt.wantStatus {
|
||||
t.Fatalf("status = %d, want %d", resp.Code, tt.wantStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeGoModule(t *testing.T) {
|
||||
tests := []struct {
|
||||
encoded string
|
||||
|
|
|
|||
|
|
@ -1,178 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/metrics"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
gradleBuildCacheContentType = "application/vnd.gradle.build-cache-artifact.v2"
|
||||
gradleBuildCacheStorageRoot = "_gradle/http-build-cache"
|
||||
defaultGradleMaxUploadSize = 100 << 20
|
||||
)
|
||||
|
||||
var gradleBuildCacheKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
|
||||
// GradleBuildCacheHandler handles Gradle HttpBuildCache GET/HEAD/PUT requests.
|
||||
//
|
||||
// This handler accepts /{key} when mounted under a base URL.
|
||||
type GradleBuildCacheHandler struct {
|
||||
proxy *Proxy
|
||||
}
|
||||
|
||||
// NewGradleBuildCacheHandler creates a Gradle HttpBuildCache handler.
|
||||
func NewGradleBuildCacheHandler(proxy *Proxy) *GradleBuildCacheHandler {
|
||||
return &GradleBuildCacheHandler{proxy: proxy}
|
||||
}
|
||||
|
||||
// Routes returns the HTTP handler for Gradle HttpBuildCache requests.
|
||||
func (h *GradleBuildCacheHandler) Routes() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead, http.MethodPut:
|
||||
default:
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
key, statusCode := h.parseCacheKey(r.URL.Path)
|
||||
if statusCode != http.StatusOK {
|
||||
if statusCode == http.StatusNotFound {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
http.Error(w, "invalid cache key", statusCode)
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodPut {
|
||||
if h.proxy.GradleReadOnly {
|
||||
http.Error(w, "gradle build cache is read-only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
h.handlePut(w, r, key)
|
||||
return
|
||||
}
|
||||
|
||||
h.handleGetOrHead(w, r, key)
|
||||
})
|
||||
}
|
||||
|
||||
func (h *GradleBuildCacheHandler) parseCacheKey(urlPath string) (string, int) {
|
||||
keyPath := strings.TrimPrefix(urlPath, "/")
|
||||
if keyPath == "" {
|
||||
return "", http.StatusNotFound
|
||||
}
|
||||
|
||||
if containsPathTraversal(keyPath) {
|
||||
return "", http.StatusBadRequest
|
||||
}
|
||||
|
||||
if strings.Contains(keyPath, "/") {
|
||||
return "", http.StatusNotFound
|
||||
}
|
||||
|
||||
if !gradleBuildCacheKeyPattern.MatchString(keyPath) {
|
||||
return "", http.StatusBadRequest
|
||||
}
|
||||
|
||||
return keyPath, http.StatusOK
|
||||
}
|
||||
|
||||
func (h *GradleBuildCacheHandler) cacheStoragePath(key string) string {
|
||||
return gradleBuildCacheStorageRoot + "/" + key
|
||||
}
|
||||
|
||||
func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http.Request, key string) {
|
||||
storagePath := h.cacheStoragePath(key)
|
||||
w.Header().Set("Content-Type", gradleBuildCacheContentType)
|
||||
|
||||
if r.Method == http.MethodHead {
|
||||
existsStart := time.Now()
|
||||
exists, err := h.proxy.Storage.Exists(r.Context(), storagePath)
|
||||
metrics.RecordStorageOperation("read", time.Since(existsStart))
|
||||
if err != nil {
|
||||
metrics.RecordStorageError("read")
|
||||
h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err)
|
||||
http.Error(w, "failed to read cache entry", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !exists {
|
||||
metrics.RecordCacheMiss("gradle")
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
metrics.RecordCacheHit("gradle")
|
||||
|
||||
sizeStart := time.Now()
|
||||
size, err := h.proxy.Storage.Size(r.Context(), storagePath)
|
||||
metrics.RecordStorageOperation("read", time.Since(sizeStart))
|
||||
if err != nil {
|
||||
metrics.RecordStorageError("read")
|
||||
} else if size >= 0 {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
readStart := time.Now()
|
||||
reader, err := h.proxy.Storage.Open(r.Context(), storagePath)
|
||||
metrics.RecordStorageOperation("read", time.Since(readStart))
|
||||
if err != nil {
|
||||
if errors.Is(err, storage.ErrNotFound) {
|
||||
metrics.RecordCacheMiss("gradle")
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
metrics.RecordStorageError("read")
|
||||
h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err)
|
||||
http.Error(w, "failed to read cache entry", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
metrics.RecordCacheHit("gradle")
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Request, key string) {
|
||||
storagePath := h.cacheStoragePath(key)
|
||||
maxUploadSize := h.proxy.GradleMaxUploadSize
|
||||
if maxUploadSize <= 0 {
|
||||
maxUploadSize = defaultGradleMaxUploadSize
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize)
|
||||
|
||||
storeStart := time.Now()
|
||||
_, hash, err := h.proxy.Storage.Store(r.Context(), storagePath, r.Body)
|
||||
metrics.RecordStorageOperation("write", time.Since(storeStart))
|
||||
if err != nil {
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
if errors.As(err, &maxBytesErr) {
|
||||
http.Error(w, "cache entry too large", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
|
||||
metrics.RecordStorageError("write")
|
||||
h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err)
|
||||
http.Error(w, "failed to write cache entry", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Length", "0")
|
||||
w.Header().Set("ETag", `"`+hash+`"`)
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
|
|
@ -1,285 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/metrics"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
|
||||
func TestGradleBuildCacheHandler_PutGetHead(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
key := "a1b2c3d4e5f6"
|
||||
payload := "cache entry content"
|
||||
|
||||
putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create PUT request: %v", err)
|
||||
}
|
||||
putResp, err := http.DefaultClient.Do(putReq)
|
||||
if err != nil {
|
||||
t.Fatalf("PUT request failed: %v", err)
|
||||
}
|
||||
_ = putResp.Body.Close()
|
||||
|
||||
if putResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("PUT status = %d, want %d", putResp.StatusCode, http.StatusCreated)
|
||||
}
|
||||
|
||||
getResp, err := http.Get(srv.URL + "/" + key)
|
||||
if err != nil {
|
||||
t.Fatalf("GET request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = getResp.Body.Close() }()
|
||||
|
||||
if getResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET status = %d, want %d", getResp.StatusCode, http.StatusOK)
|
||||
}
|
||||
if getResp.Header.Get("Content-Type") != gradleBuildCacheContentType {
|
||||
t.Fatalf("GET Content-Type = %q, want %q", getResp.Header.Get("Content-Type"), gradleBuildCacheContentType)
|
||||
}
|
||||
|
||||
body, _ := io.ReadAll(getResp.Body)
|
||||
if string(body) != payload {
|
||||
t.Fatalf("GET body = %q, want %q", body, payload)
|
||||
}
|
||||
|
||||
headReq, err := http.NewRequest(http.MethodHead, srv.URL+"/"+key, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create HEAD request: %v", err)
|
||||
}
|
||||
headResp, err := http.DefaultClient.Do(headReq)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = headResp.Body.Close() }()
|
||||
|
||||
if headResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("HEAD status = %d, want %d", headResp.StatusCode, http.StatusOK)
|
||||
}
|
||||
body, _ = io.ReadAll(headResp.Body)
|
||||
if len(body) != 0 {
|
||||
t.Fatalf("HEAD body length = %d, want 0", len(body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_RootKeyPath(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
key := "rootpathkey"
|
||||
putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("root"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create PUT request: %v", err)
|
||||
}
|
||||
putResp, err := http.DefaultClient.Do(putReq)
|
||||
if err != nil {
|
||||
t.Fatalf("PUT request failed: %v", err)
|
||||
}
|
||||
_ = putResp.Body.Close()
|
||||
|
||||
if putResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("PUT status = %d, want %d", putResp.StatusCode, http.StatusCreated)
|
||||
}
|
||||
|
||||
getResp, err := http.Get(srv.URL + "/" + key)
|
||||
if err != nil {
|
||||
t.Fatalf("GET request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = getResp.Body.Close() }()
|
||||
|
||||
if getResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET status = %d, want %d", getResp.StatusCode, http.StatusOK)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_GetMiss(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + "/missing-key")
|
||||
if err != nil {
|
||||
t.Fatalf("GET request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_MethodNotAllowed(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/key", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("status = %d, want %d", w.Code, http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_PathTraversalRejected(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/../secret", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want %d", w.Code, http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_CachePrefixRejected(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/cache/key", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want %d", w.Code, http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_PutOverwriteReturnsCreated(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
key := "overwrite-key"
|
||||
|
||||
for i, payload := range []string{"first", "second"} {
|
||||
req, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create PUT request: %v", err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("PUT request failed: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
want := http.StatusCreated
|
||||
if resp.StatusCode != want {
|
||||
t.Fatalf("PUT #%d status = %d, want %d", i+1, resp.StatusCode, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_PutReadOnly(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.GradleReadOnly = true
|
||||
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, srv.URL+"/readonly-key", strings.NewReader("payload"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create PUT request: %v", err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("PUT request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("PUT status = %d, want %d", resp.StatusCode, http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_PutTooLarge(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.GradleMaxUploadSize = 4
|
||||
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, srv.URL+"/oversized-key", strings.NewReader("12345"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create PUT request: %v", err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("PUT request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("PUT status = %d, want %d", resp.StatusCode, http.StatusRequestEntityTooLarge)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) {
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
h := NewGradleBuildCacheHandler(proxy)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle"))
|
||||
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle"))
|
||||
|
||||
key := "metrics-key"
|
||||
putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("payload"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create PUT request: %v", err)
|
||||
}
|
||||
putResp, err := http.DefaultClient.Do(putReq)
|
||||
if err != nil {
|
||||
t.Fatalf("PUT request failed: %v", err)
|
||||
}
|
||||
_ = putResp.Body.Close()
|
||||
|
||||
getResp, err := http.Get(srv.URL + "/" + key)
|
||||
if err != nil {
|
||||
t.Fatalf("GET request failed: %v", err)
|
||||
}
|
||||
_ = getResp.Body.Close()
|
||||
|
||||
headReq, err := http.NewRequest(http.MethodHead, srv.URL+"/"+key, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create HEAD request: %v", err)
|
||||
}
|
||||
headResp, err := http.DefaultClient.Do(headReq)
|
||||
if err != nil {
|
||||
t.Fatalf("HEAD request failed: %v", err)
|
||||
}
|
||||
_ = headResp.Body.Close()
|
||||
|
||||
missResp, err := http.Get(srv.URL + "/missing-key")
|
||||
if err != nil {
|
||||
t.Fatalf("GET miss request failed: %v", err)
|
||||
}
|
||||
_ = missResp.Body.Close()
|
||||
|
||||
hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle"))
|
||||
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle"))
|
||||
|
||||
if diff := hitsAfter - hitsBefore; diff != 2 {
|
||||
t.Fatalf("cache hits delta = %.0f, want 2", diff)
|
||||
}
|
||||
if diff := missesAfter - missesBefore; diff != 1 {
|
||||
t.Fatalf("cache misses delta = %.0f, want 1", diff)
|
||||
}
|
||||
}
|
||||
|
|
@ -13,10 +13,9 @@ import (
|
|||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/proxy/internal/metrics"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
|
|
@ -25,21 +24,9 @@ import (
|
|||
)
|
||||
|
||||
// containsPathTraversal returns true if the path contains ".." segments
|
||||
// that could be used to escape the intended directory. It checks the path
|
||||
// as given and after URL-decoding, and treats backslashes as separators.
|
||||
// that could be used to escape the intended directory.
|
||||
func containsPathTraversal(path string) bool {
|
||||
if hasDotDotSegment(path) {
|
||||
return true
|
||||
}
|
||||
if decoded, err := url.PathUnescape(path); err == nil && decoded != path {
|
||||
return hasDotDotSegment(decoded)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasDotDotSegment(path string) bool {
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
for segment := range strings.SplitSeq(path, "/") {
|
||||
for _, segment := range strings.Split(path, "/") {
|
||||
if segment == ".." {
|
||||
return true
|
||||
}
|
||||
|
|
@ -49,44 +36,25 @@ func hasDotDotSegment(path string) bool {
|
|||
|
||||
const defaultHTTPTimeout = 30 * time.Second
|
||||
|
||||
const artifactCopyBufferSize = 32 << 10
|
||||
|
||||
var artifactCopyBufferPool = sync.Pool{ //nolint:gochecknoglobals // shared across artifact responses
|
||||
New: func() any {
|
||||
buffer := make([]byte, artifactCopyBufferSize)
|
||||
return &buffer
|
||||
},
|
||||
}
|
||||
|
||||
// canonicalPackagePURL returns a versionless PURL in canonical form so cooldown
|
||||
// lookups match keys produced by config.CooldownConfig.NormalizedPackages.
|
||||
func canonicalPackagePURL(ecosystem, name string) string {
|
||||
return purl.MakePURLString(ecosystem, name, "")
|
||||
}
|
||||
|
||||
const contentTypeJSON = "application/json"
|
||||
|
||||
const headerAcceptEncoding = "Accept-Encoding"
|
||||
// maxMetadataSize is the maximum size of upstream metadata responses (100 MB).
|
||||
// Package metadata (e.g. npm with many versions) can be large, but unbounded
|
||||
// reads risk OOM if an upstream misbehaves.
|
||||
const maxMetadataSize = 100 << 20
|
||||
|
||||
// defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset.
|
||||
const defaultMetadataMaxSize = 100 << 20
|
||||
|
||||
// ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit.
|
||||
// ErrMetadataTooLarge is returned when upstream metadata exceeds maxMetadataSize.
|
||||
var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit")
|
||||
|
||||
// ReadMetadata reads an upstream response body with a size limit to prevent OOM
|
||||
// from unexpectedly large responses. Returns ErrMetadataTooLarge if the response
|
||||
// is truncated by the limit.
|
||||
func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) {
|
||||
limit := p.MetadataMaxSize
|
||||
if limit <= 0 {
|
||||
limit = defaultMetadataMaxSize
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||
func ReadMetadata(r io.Reader) ([]byte, error) {
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxMetadataSize+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
if int64(len(data)) > maxMetadataSize {
|
||||
return nil, ErrMetadataTooLarge
|
||||
}
|
||||
return data, nil
|
||||
|
|
@ -102,9 +70,6 @@ type Proxy struct {
|
|||
Cooldown *cooldown.Config
|
||||
CacheMetadata bool
|
||||
MetadataTTL time.Duration
|
||||
MetadataMaxSize int64
|
||||
GradleReadOnly bool
|
||||
GradleMaxUploadSize int64
|
||||
DirectServe bool
|
||||
DirectServeTTL time.Duration
|
||||
// DirectServeBaseURL, if set, replaces the scheme and host of presigned
|
||||
|
|
@ -112,7 +77,6 @@ type Proxy struct {
|
|||
// storage at an internal one.
|
||||
DirectServeBaseURL string
|
||||
HTTPClient *http.Client
|
||||
AuthForURL func(string) (headerName, headerValue string)
|
||||
}
|
||||
|
||||
// NewProxy creates a new Proxy with the given dependencies.
|
||||
|
|
@ -144,59 +108,41 @@ type CacheResult struct {
|
|||
|
||||
// GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream.
|
||||
func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) {
|
||||
if cached, err := p.GetCachedArtifact(ctx, ecosystem, name, version, filename); err != nil {
|
||||
pkgPURL := purl.MakePURLString(ecosystem, name, "")
|
||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
|
||||
if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil {
|
||||
return nil, err
|
||||
} else if cached != nil {
|
||||
return cached, nil
|
||||
}
|
||||
metrics.RecordCacheMiss(ecosystem)
|
||||
|
||||
pkgPURL := purl.MakePURLString(ecosystem, name, "")
|
||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
return p.fetchAndCache(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL)
|
||||
}
|
||||
|
||||
// GetCachedArtifact retrieves an artifact from cache without contacting an upstream.
|
||||
// It returns nil when no usable cache entry exists.
|
||||
func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) {
|
||||
pkgPURL := purl.MakePURLString(ecosystem, name, "")
|
||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
return p.checkCache(ctx, pkgPURL, versionPURL, filename)
|
||||
}
|
||||
|
||||
// ClearCachedArtifact removes both an artifact cache record and its stored
|
||||
// bytes after an external integrity check fails.
|
||||
func (p *Proxy) ClearCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) error {
|
||||
if p.DB == nil || p.Storage == nil {
|
||||
return nil
|
||||
}
|
||||
pkgPURL := purl.MakePURLString(ecosystem, name, "")
|
||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
cached, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("looking up cached artifact: %w", err)
|
||||
}
|
||||
if cached == nil {
|
||||
return nil
|
||||
}
|
||||
if err := p.Storage.Delete(ctx, cached.StoragePath); err != nil {
|
||||
return fmt.Errorf("deleting cached artifact: %w", err)
|
||||
}
|
||||
return p.DB.ClearArtifactCache(versionPURL, filename)
|
||||
}
|
||||
|
||||
// checkCache looks up an artifact in the cache. Returns nil if not cached.
|
||||
func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) {
|
||||
artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename)
|
||||
pkg, err := p.DB.GetPackageByPURL(pkgPURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("checking package cache: %w", err)
|
||||
}
|
||||
if pkg == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ver, err := p.DB.GetVersionByPURL(versionPURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("checking version cache: %w", err)
|
||||
}
|
||||
if ver == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
artifact, err := p.DB.GetArtifact(versionPURL, filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("checking artifact cache: %w", err)
|
||||
}
|
||||
if artifact == nil {
|
||||
return nil, nil
|
||||
}
|
||||
checks, err := newIntegrityChecks(artifact.ContentHash.String, artifact.Integrity.String)
|
||||
if err != nil {
|
||||
p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err)
|
||||
if artifact == nil || !artifact.IsCached() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
|
|
@ -208,44 +154,30 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
|
|||
}
|
||||
|
||||
if p.DirectServe {
|
||||
signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL)
|
||||
signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL)
|
||||
if err == nil {
|
||||
result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL)
|
||||
p.recordCacheHit(artifact.Ecosystem, versionPURL, filename)
|
||||
p.recordCacheHit(pkgPURL, versionPURL, filename)
|
||||
return result, nil
|
||||
}
|
||||
if !errors.Is(err, storage.ErrSignedURLUnsupported) {
|
||||
p.Logger.Warn("failed to sign storage URL, falling back to streaming",
|
||||
"path", artifact.StoragePath, "error", err)
|
||||
"path", artifact.StoragePath.String, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
reader, err := p.Storage.Open(ctx, artifact.StoragePath)
|
||||
reader, err := p.Storage.Open(ctx, artifact.StoragePath.String)
|
||||
metrics.RecordStorageOperation("read", time.Since(start))
|
||||
if err != nil {
|
||||
metrics.RecordStorageError("read")
|
||||
p.Logger.Warn("cached artifact missing from storage, will refetch",
|
||||
"path", artifact.StoragePath, "error", err)
|
||||
"path", artifact.StoragePath.String, "error", err)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
result.Reader, err = checks.wrap(reader,
|
||||
func(reason string) {
|
||||
p.Logger.Error("cached artifact failed integrity check",
|
||||
"purl", versionPURL, "filename", filename,
|
||||
"path", artifact.StoragePath, "reason", reason)
|
||||
metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem))
|
||||
if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil {
|
||||
p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
_ = reader.Close()
|
||||
p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err)
|
||||
return nil, nil
|
||||
}
|
||||
p.recordCacheHit(artifact.Ecosystem, versionPURL, filename)
|
||||
result.Reader = reader
|
||||
p.recordCacheHit(pkgPURL, versionPURL, filename)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
|
@ -269,28 +201,20 @@ func rewriteSignedURLHost(signed, baseURL string) string {
|
|||
return s.String()
|
||||
}
|
||||
|
||||
func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) {
|
||||
func (p *Proxy) recordCacheHit(pkgPURL, versionPURL, filename string) {
|
||||
_ = p.DB.RecordArtifactHit(versionPURL, filename)
|
||||
metrics.RecordCacheHit(ecosystem)
|
||||
}
|
||||
|
||||
func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, versionPURL, filename string, cause error) {
|
||||
p.Logger.Warn("cached artifact has unusable integrity metadata",
|
||||
"purl", versionPURL, "filename", filename,
|
||||
"path", artifact.StoragePath, "error", cause)
|
||||
metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem))
|
||||
if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil {
|
||||
p.Logger.Warn("failed to clear unusable artifact from cache", "error", err)
|
||||
if parsed, err := purl.Parse(pkgPURL); err == nil {
|
||||
metrics.RecordCacheHit(purl.PURLTypeToEcosystem(parsed.Type))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) {
|
||||
// Record cache miss
|
||||
metrics.RecordCacheMiss(ecosystem)
|
||||
|
||||
// Resolve download URL
|
||||
info, err := p.Resolver.Resolve(ctx, ecosystem, name, version)
|
||||
if err != nil {
|
||||
if errors.Is(err, fetch.ErrNotFound) {
|
||||
return nil, ErrUpstreamNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("resolving download URL: %w", err)
|
||||
}
|
||||
|
||||
|
|
@ -310,9 +234,6 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil
|
|||
if err != nil {
|
||||
metrics.RecordUpstreamFetch(ecosystem, fetchDuration)
|
||||
metrics.RecordUpstreamError(ecosystem, "fetch_failed")
|
||||
if errors.Is(err, fetch.ErrNotFound) {
|
||||
return nil, ErrUpstreamNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("fetching from upstream: %w", err)
|
||||
}
|
||||
metrics.RecordUpstreamFetch(ecosystem, fetchDuration)
|
||||
|
|
@ -399,40 +320,29 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u
|
|||
|
||||
// ServeArtifact writes a CacheResult to an HTTP response.
|
||||
func ServeArtifact(w http.ResponseWriter, result *CacheResult) {
|
||||
serveArtifact(w, http.MethodGet, result)
|
||||
}
|
||||
|
||||
func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) {
|
||||
if result.RedirectURL != "" {
|
||||
if result.Hash != "" {
|
||||
w.Header().Set("ETag", `"`+result.Hash+`"`)
|
||||
w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash))
|
||||
}
|
||||
w.Header().Set("Location", result.RedirectURL)
|
||||
w.WriteHeader(http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
||||
if result.Reader != nil {
|
||||
defer func() { _ = result.Reader.Close() }()
|
||||
}
|
||||
|
||||
if result.ContentType != "" {
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
}
|
||||
if result.Size > 0 || (method == http.MethodHead && result.Size == 0) {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10))
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
if result.Hash != "" {
|
||||
w.Header().Set("ETag", `"`+result.Hash+`"`)
|
||||
w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash))
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if method != http.MethodHead && result.Reader != nil {
|
||||
buffer := artifactCopyBufferPool.Get().(*[]byte)
|
||||
defer artifactCopyBufferPool.Put(buffer)
|
||||
// Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer.
|
||||
_, _ = io.CopyBuffer(struct{ io.Writer }{w}, result.Reader, *buffer)
|
||||
}
|
||||
_, _ = io.Copy(w, result.Reader)
|
||||
}
|
||||
|
||||
// ProxyUpstream forwards a request to an upstream URL without caching.
|
||||
|
|
@ -453,7 +363,6 @@ func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamUR
|
|||
req.Header.Set(header, v)
|
||||
}
|
||||
}
|
||||
p.applyUpstreamAuth(req)
|
||||
|
||||
resp, err := p.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
|
|
@ -480,7 +389,6 @@ func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL st
|
|||
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
p.applyUpstreamAuth(req)
|
||||
|
||||
resp, err := p.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
|
|
@ -507,18 +415,7 @@ func JSONError(w http.ResponseWriter, status int, message string) {
|
|||
}
|
||||
|
||||
// ErrUpstreamNotFound indicates the upstream returned 404.
|
||||
var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound)
|
||||
|
||||
// serveArtifactError writes response for a failed fetch:
|
||||
// 404 when upstream reports artifact missing, 502 otherwise.
|
||||
func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg string) {
|
||||
if errors.Is(err, ErrUpstreamNotFound) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
p.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, clientMsg, http.StatusBadGateway)
|
||||
}
|
||||
var ErrUpstreamNotFound = fmt.Errorf("upstream: not found")
|
||||
|
||||
// errStale304 is returned when upstream sends 304 but the cached file is missing.
|
||||
var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing")
|
||||
|
|
@ -552,20 +449,18 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
|
|||
cached, readErr := p.Storage.Open(ctx, entry.StoragePath)
|
||||
if readErr == nil {
|
||||
defer func() { _ = cached.Close() }()
|
||||
data, readErr := p.ReadMetadata(cached)
|
||||
data, readErr := ReadMetadata(cached)
|
||||
if readErr == nil {
|
||||
ct := contentTypeJSON
|
||||
if entry.ContentType.Valid {
|
||||
ct = entry.ContentType.String
|
||||
}
|
||||
metrics.RecordCacheHit(ecosystem)
|
||||
return data, ct, nil
|
||||
}
|
||||
}
|
||||
// Cache file missing/unreadable, fall through to upstream
|
||||
}
|
||||
}
|
||||
p.recordMetadataCacheMiss(ecosystem)
|
||||
|
||||
accept := contentTypeJSON
|
||||
if len(acceptHeaders) > 0 && acceptHeaders[0] != "" {
|
||||
|
|
@ -599,7 +494,7 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
|
|||
}
|
||||
defer func() { _ = cached.Close() }()
|
||||
|
||||
data, readErr := p.ReadMetadata(cached)
|
||||
data, readErr := ReadMetadata(cached)
|
||||
if readErr != nil {
|
||||
return nil, "", fmt.Errorf("upstream failed and cached read error: %w", err)
|
||||
}
|
||||
|
|
@ -613,12 +508,6 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
|
|||
return data, ct, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) recordMetadataCacheMiss(ecosystem string) {
|
||||
if p.CacheMetadata {
|
||||
metrics.RecordCacheMiss(ecosystem)
|
||||
}
|
||||
}
|
||||
|
||||
// fetchUpstreamMetadata fetches metadata from upstream, using ETag for conditional revalidation.
|
||||
// Returns the body, content type, ETag, upstream Last-Modified time, and any error.
|
||||
func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept string) ([]byte, string, string, time.Time, error) {
|
||||
|
|
@ -629,7 +518,6 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
|
|||
return nil, "", "", zeroTime, fmt.Errorf("creating request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", accept)
|
||||
p.applyUpstreamAuth(req)
|
||||
|
||||
if entry != nil && entry.ETag.Valid {
|
||||
req.Header.Set("If-None-Match", entry.ETag.String)
|
||||
|
|
@ -648,7 +536,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
|
|||
return nil, "", "", zeroTime, errStale304
|
||||
}
|
||||
defer func() { _ = cached.Close() }()
|
||||
data, readErr := p.ReadMetadata(cached)
|
||||
data, readErr := ReadMetadata(cached)
|
||||
if readErr != nil {
|
||||
return nil, "", "", zeroTime, errStale304
|
||||
}
|
||||
|
|
@ -670,7 +558,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
|
|||
return nil, "", "", zeroTime, fmt.Errorf("upstream returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := p.ReadMetadata(resp.Body)
|
||||
body, err := ReadMetadata(resp.Body)
|
||||
if err != nil {
|
||||
return nil, "", "", zeroTime, fmt.Errorf("reading response: %w", err)
|
||||
}
|
||||
|
|
@ -767,14 +655,9 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL,
|
|||
return
|
||||
}
|
||||
|
||||
p.writeMetadataCachedResponse(w, r, ecosystem, cacheKey, body, contentType)
|
||||
}
|
||||
|
||||
// writeMetadataCachedResponse writes a cached metadata response and handles
|
||||
// conditional request headers using metadata cache validators.
|
||||
func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) {
|
||||
cm := p.lookupCachedMeta(ecosystem, cacheKey)
|
||||
|
||||
// Honor client conditional request headers
|
||||
if cm.etag != "" {
|
||||
if match := r.Header.Get("If-None-Match"); match != "" && match == cm.etag {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
|
|
@ -819,9 +702,8 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst
|
|||
accept = acceptHeaders[0]
|
||||
}
|
||||
req.Header.Set("Accept", accept)
|
||||
p.applyUpstreamAuth(req)
|
||||
|
||||
for _, header := range []string{headerAcceptEncoding, "If-Modified-Since", "If-None-Match"} {
|
||||
for _, header := range []string{"Accept-Encoding", "If-Modified-Since", "If-None-Match"} {
|
||||
if v := r.Header.Get(header); v != "" {
|
||||
req.Header.Set(header, v)
|
||||
}
|
||||
|
|
@ -844,17 +726,6 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst
|
|||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
func (p *Proxy) applyUpstreamAuth(req *http.Request) {
|
||||
if p.AuthForURL == nil {
|
||||
return
|
||||
}
|
||||
|
||||
headerName, headerValue := p.AuthForURL(req.URL.String())
|
||||
if headerName != "" && headerValue != "" {
|
||||
req.Header.Set(headerName, headerValue)
|
||||
}
|
||||
}
|
||||
|
||||
// GetOrFetchArtifactFromURL retrieves an artifact from cache or fetches from a specific URL.
|
||||
// This is useful for registries where download URLs are determined from metadata.
|
||||
func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) {
|
||||
|
|
@ -862,17 +733,18 @@ func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name,
|
|||
}
|
||||
|
||||
// GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL
|
||||
// with additional request-specific HTTP headers.
|
||||
// with additional HTTP headers. This is needed for registries that require authentication
|
||||
// (e.g. Docker Hub requires a Bearer token even for public images).
|
||||
func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) {
|
||||
if cached, err := p.GetCachedArtifact(ctx, ecosystem, name, version, filename); err != nil {
|
||||
pkgPURL := purl.MakePURLString(ecosystem, name, "")
|
||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
|
||||
if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil {
|
||||
return nil, err
|
||||
} else if cached != nil {
|
||||
return cached, nil
|
||||
}
|
||||
metrics.RecordCacheMiss(ecosystem)
|
||||
|
||||
pkgPURL := purl.MakePURLString(ecosystem, name, "")
|
||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
return p.fetchAndCacheFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers)
|
||||
}
|
||||
|
||||
|
|
@ -882,9 +754,6 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi
|
|||
|
||||
artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers)
|
||||
if err != nil {
|
||||
if errors.Is(err, fetch.ErrNotFound) {
|
||||
return nil, ErrUpstreamNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("fetching from upstream: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,300 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
"github.com/git-pkgs/purl"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
const benchmarkArtifactSize = 64 << 10
|
||||
|
||||
const benchmarkMetadataSize = 1 << 20
|
||||
|
||||
type benchmarkResponseWriter struct {
|
||||
header http.Header
|
||||
}
|
||||
|
||||
func (w *benchmarkResponseWriter) Header() http.Header {
|
||||
return w.header
|
||||
}
|
||||
|
||||
func (w *benchmarkResponseWriter) Write(p []byte) (int, error) {
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (w *benchmarkResponseWriter) WriteHeader(_ int) {}
|
||||
|
||||
func benchmarkCachedProxy(b *testing.B) (*Proxy, *mockStorage) {
|
||||
b.Helper()
|
||||
|
||||
proxy, db, store, _ := setupTestProxy(b)
|
||||
content := strings.Repeat("x", benchmarkArtifactSize)
|
||||
seedPackage(b, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", content)
|
||||
|
||||
artifact, err := db.GetArtifact("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz")
|
||||
if err != nil {
|
||||
b.Fatalf("get seeded artifact: %v", err)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
artifact.ContentHash.String = hex.EncodeToString(sum[:])
|
||||
if err := db.UpsertArtifact(artifact); err != nil {
|
||||
b.Fatalf("update seeded artifact hash: %v", err)
|
||||
}
|
||||
|
||||
return proxy, store
|
||||
}
|
||||
|
||||
func BenchmarkArtifactCacheHit(b *testing.B) {
|
||||
ctx := context.Background()
|
||||
|
||||
b.Run("stream-64KiB", func(b *testing.B) {
|
||||
proxy, _ := benchmarkCachedProxy(b)
|
||||
w := &benchmarkResponseWriter{header: make(http.Header)}
|
||||
b.SetBytes(benchmarkArtifactSize)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for b.Loop() {
|
||||
result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("direct-serve", func(b *testing.B) {
|
||||
proxy, store := benchmarkCachedProxy(b)
|
||||
proxy.DirectServe = true
|
||||
store.signedURL = "https://storage.example/npm/lodash-4.17.21.tgz?signature=abc"
|
||||
w := &benchmarkResponseWriter{header: make(http.Header)}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for b.Loop() {
|
||||
result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkArtifactCacheHitParallel(b *testing.B) {
|
||||
proxy, _ := benchmarkCachedProxy(b)
|
||||
ctx := context.Background()
|
||||
b.SetBytes(benchmarkArtifactSize)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
w := &benchmarkResponseWriter{header: make(http.Header)}
|
||||
for pb.Next() {
|
||||
result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz")
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
return
|
||||
}
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkReadMetadata(b *testing.B) {
|
||||
payload := bytes.Repeat([]byte("x"), benchmarkMetadataSize)
|
||||
proxy := &Proxy{MetadataMaxSize: benchmarkMetadataSize}
|
||||
b.SetBytes(benchmarkMetadataSize)
|
||||
b.ReportAllocs()
|
||||
|
||||
var data []byte
|
||||
for b.Loop() {
|
||||
var err error
|
||||
data, err = proxy.ReadMetadata(bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(data) != len(payload) {
|
||||
b.Fatalf("metadata size = %d, want %d", len(data), len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkArtifactPURLConstruction(b *testing.B) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ecosystem string
|
||||
packageID string
|
||||
}{
|
||||
{"npm", "npm", "lodash"},
|
||||
{"scoped-npm", "npm", "@scope/package"},
|
||||
{"go", "golang", "github.com/git-pkgs/proxy"},
|
||||
} {
|
||||
b.Run(tc.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
var packagePURL, versionPURL string
|
||||
for b.Loop() {
|
||||
packagePURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "")
|
||||
versionPURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "1.2.3")
|
||||
}
|
||||
if packagePURL == "" || versionPURL == "" {
|
||||
b.Fatal("empty PURL")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type benchmarkNPMServer struct {
|
||||
client *http.Client
|
||||
requestURL string
|
||||
db *database.DB
|
||||
versionPURL string
|
||||
filename string
|
||||
}
|
||||
|
||||
func newBenchmarkNPMServer(b *testing.B) *benchmarkNPMServer {
|
||||
b.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
dir := b.TempDir()
|
||||
db, err := database.Create(filepath.Join(dir, "benchmark.db"))
|
||||
if err != nil {
|
||||
b.Fatalf("create database: %v", err)
|
||||
}
|
||||
b.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
store, err := storage.OpenBucket(ctx, "file://"+filepath.Join(dir, "cache"))
|
||||
if err != nil {
|
||||
b.Fatalf("open storage: %v", err)
|
||||
}
|
||||
b.Cleanup(func() { _ = store.Close() })
|
||||
|
||||
content := bytes.Repeat([]byte("x"), benchmarkArtifactSize)
|
||||
storagePath := storage.ArtifactPath("npm", "", "lodash", "4.17.21", "lodash-4.17.21.tgz")
|
||||
size, hash, err := store.Store(ctx, storagePath, bytes.NewReader(content))
|
||||
if err != nil {
|
||||
b.Fatalf("store artifact: %v", err)
|
||||
}
|
||||
|
||||
pkg := &database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"}
|
||||
if err := db.UpsertPackage(pkg); err != nil {
|
||||
b.Fatalf("seed package: %v", err)
|
||||
}
|
||||
version := &database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: pkg.PURL}
|
||||
if err := db.UpsertVersion(version); err != nil {
|
||||
b.Fatalf("seed version: %v", err)
|
||||
}
|
||||
artifact := &database.Artifact{
|
||||
VersionPURL: version.PURL,
|
||||
Filename: "lodash-4.17.21.tgz",
|
||||
UpstreamURL: "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
|
||||
StoragePath: sql.NullString{String: storagePath, Valid: true},
|
||||
ContentHash: sql.NullString{String: hash, Valid: true},
|
||||
Size: sql.NullInt64{Int64: size, Valid: true},
|
||||
ContentType: sql.NullString{String: "application/gzip", Valid: true},
|
||||
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
}
|
||||
if err := db.UpsertArtifact(artifact); err != nil {
|
||||
b.Fatalf("seed artifact: %v", err)
|
||||
}
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
proxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger)
|
||||
handler := NewNPMHandler(proxy, "http://proxy.example", "https://registry.npmjs.org")
|
||||
server := httptest.NewServer(handler.Routes())
|
||||
b.Cleanup(server.Close)
|
||||
client := server.Client()
|
||||
return &benchmarkNPMServer{
|
||||
client: client,
|
||||
requestURL: server.URL + "/lodash/-/lodash-4.17.21.tgz",
|
||||
db: db,
|
||||
versionPURL: version.PURL,
|
||||
filename: artifact.Filename,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *benchmarkNPMServer) request() error {
|
||||
resp, err := s.client.Get(s.requestURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("GET cached artifact: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("GET cached artifact status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
n, err := io.Copy(io.Discard, resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read cached artifact: %w", err)
|
||||
}
|
||||
if n != benchmarkArtifactSize {
|
||||
return fmt.Errorf("cached artifact size = %d, want %d", n, benchmarkArtifactSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *benchmarkNPMServer) hitCount(b *testing.B) int64 {
|
||||
b.Helper()
|
||||
artifact, err := s.db.GetArtifact(s.versionPURL, s.filename)
|
||||
if err != nil {
|
||||
b.Fatalf("get artifact hit count: %v", err)
|
||||
}
|
||||
return artifact.HitCount
|
||||
}
|
||||
|
||||
func benchmarkNPMArtifactCacheHitHTTP(b *testing.B, parallel bool) {
|
||||
server := newBenchmarkNPMServer(b)
|
||||
if err := server.request(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
startHits := server.hitCount(b)
|
||||
|
||||
b.SetBytes(benchmarkArtifactSize)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
if parallel {
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
if err := server.request(); err != nil {
|
||||
b.Error(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
} else {
|
||||
for b.Loop() {
|
||||
if err := server.request(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
|
||||
if hitCount := server.hitCount(b) - startHits; hitCount != int64(b.N) {
|
||||
b.Fatalf("new artifact hits = %d, want %d", hitCount, b.N)
|
||||
}
|
||||
b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "requests/s")
|
||||
}
|
||||
|
||||
func BenchmarkNPMArtifactCacheHitHTTP(b *testing.B) {
|
||||
benchmarkNPMArtifactCacheHitHTTP(b, false)
|
||||
}
|
||||
|
||||
func BenchmarkNPMArtifactCacheHitHTTPParallel(b *testing.B) {
|
||||
benchmarkNPMArtifactCacheHitHTTP(b, true)
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
|
@ -13,13 +14,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/config"
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/proxy/internal/metrics"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
"github.com/git-pkgs/purl"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
|
||||
// mockStorage implements storage.Storage for testing.
|
||||
|
|
@ -44,7 +41,7 @@ func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64,
|
|||
return 0, "", err
|
||||
}
|
||||
s.files[path] = data
|
||||
return int64(len(data)), sha256Hex(string(data)), nil
|
||||
return int64(len(data)), "fakehash123", nil
|
||||
}
|
||||
|
||||
func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error) {
|
||||
|
|
@ -102,7 +99,6 @@ func (s *mockStorage) Close() error { return nil }
|
|||
type mockFetcher struct {
|
||||
artifact *fetch.Artifact
|
||||
fetchErr error
|
||||
fetchErrByURL map[string]error
|
||||
fetchCalled bool
|
||||
fetchedURL string
|
||||
}
|
||||
|
|
@ -114,11 +110,6 @@ func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, e
|
|||
func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) {
|
||||
f.fetchCalled = true
|
||||
f.fetchedURL = url
|
||||
if f.fetchErrByURL != nil {
|
||||
if err, ok := f.fetchErrByURL[url]; ok {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if f.fetchErr != nil {
|
||||
return nil, f.fetchErr
|
||||
}
|
||||
|
|
@ -130,7 +121,7 @@ func (f *mockFetcher) Head(_ context.Context, _ string) (int64, string, error) {
|
|||
}
|
||||
|
||||
// setupTestProxy creates a Proxy with a real DB (SQLite in temp dir) and mock storage/fetcher.
|
||||
func setupTestProxy(t testing.TB) (*Proxy, *database.DB, *mockStorage, *mockFetcher) {
|
||||
func setupTestProxy(t *testing.T) (*Proxy, *database.DB, *mockStorage, *mockFetcher) {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
|
|
@ -150,11 +141,11 @@ func setupTestProxy(t testing.TB) (*Proxy, *database.DB, *mockStorage, *mockFetc
|
|||
}
|
||||
|
||||
// seedPackage creates a package, version, and cached artifact in the test DB and storage.
|
||||
func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) {
|
||||
func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) {
|
||||
t.Helper()
|
||||
|
||||
pkg := &database.Package{
|
||||
PURL: purl.MakePURLString(ecosystem, name, ""),
|
||||
PURL: fmt.Sprintf("pkg:%s/%s", ecosystem, name),
|
||||
Ecosystem: ecosystem,
|
||||
Name: name,
|
||||
}
|
||||
|
|
@ -162,7 +153,7 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n
|
|||
t.Fatalf("failed to upsert package: %v", err)
|
||||
}
|
||||
|
||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
versionPURL := fmt.Sprintf("pkg:%s/%s@%s", ecosystem, name, version)
|
||||
ver := &database.Version{
|
||||
PURL: versionPURL,
|
||||
PackagePURL: pkg.PURL,
|
||||
|
|
@ -179,7 +170,7 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n
|
|||
Filename: filename,
|
||||
UpstreamURL: "https://example.com/" + filename,
|
||||
StoragePath: sql.NullString{String: storagePath, Valid: true},
|
||||
ContentHash: sql.NullString{String: sha256Hex(content), Valid: true},
|
||||
ContentHash: sql.NullString{String: "abc123", Valid: true},
|
||||
Size: sql.NullInt64{Int64: int64(len(content)), Valid: true},
|
||||
ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
|
||||
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
|
|
@ -268,80 +259,13 @@ func TestGetOrFetchArtifact_CacheHit(t *testing.T) {
|
|||
if result.ContentType != "application/octet-stream" {
|
||||
t.Errorf("got content type %q, want %q", result.ContentType, "application/octet-stream")
|
||||
}
|
||||
if result.Hash != sha256Hex("cached content") {
|
||||
t.Errorf("got hash %q, want %q", result.Hash, sha256Hex("cached content"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetCachedArtifactRejectsMalformedIntegrityMetadata(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
malformedHash string
|
||||
malformedIntegrity string
|
||||
}{
|
||||
{name: "content hash", malformedHash: "abc123"},
|
||||
{name: "native integrity", malformedIntegrity: "sha512-abc123"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assertMalformedCacheRejected(t, test.malformedHash, test.malformedIntegrity)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertMalformedCacheRejected(t *testing.T, malformedHash, malformedIntegrity string) {
|
||||
t.Helper()
|
||||
proxy, db, store, _ := setupTestProxy(t)
|
||||
const (
|
||||
packageName = "broken"
|
||||
version = "1.0.0"
|
||||
filename = "broken-1.0.0.tgz"
|
||||
)
|
||||
seedPackage(t, db, store, "npm", packageName, version, filename, "cached content")
|
||||
versionPURL := purl.MakePURLString("npm", packageName, version)
|
||||
|
||||
if malformedHash != "" {
|
||||
artifact, err := db.GetArtifact(versionPURL, filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
artifact.ContentHash = sql.NullString{String: malformedHash, Valid: true}
|
||||
if err := db.UpsertArtifact(artifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if malformedIntegrity != "" {
|
||||
versionRecord := &database.Version{
|
||||
PURL: versionPURL,
|
||||
PackagePURL: purl.MakePURLString("npm", packageName, ""),
|
||||
Integrity: sql.NullString{String: malformedIntegrity, Valid: true},
|
||||
}
|
||||
if err := db.UpsertVersion(versionRecord); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
proxy.DirectServe = true
|
||||
store.signedURL = "https://cache.example/broken"
|
||||
result, err := proxy.GetCachedArtifact(context.Background(), "npm", packageName, version, filename)
|
||||
if err != nil {
|
||||
t.Fatalf("GetCachedArtifact: %v", err)
|
||||
}
|
||||
if result != nil {
|
||||
t.Errorf("GetCachedArtifact = %+v, want nil", result)
|
||||
}
|
||||
artifact, err := db.GetArtifact(versionPURL, filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if artifact.StoragePath.Valid {
|
||||
t.Error("unusable cache record retained its storage path")
|
||||
if result.Hash != "abc123" {
|
||||
t.Errorf("got hash %q, want %q", result.Hash, "abc123")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm"))
|
||||
|
||||
// The resolver will fail because "nonexistent" isn't a real package,
|
||||
// but we're testing that it tries to fetch (doesn't return from cache).
|
||||
|
|
@ -351,10 +275,6 @@ func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) {
|
|||
if err == nil {
|
||||
t.Fatal("expected error for uncached package")
|
||||
}
|
||||
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm"))
|
||||
if diff := missesAfter - missesBefore; diff != 1 {
|
||||
t.Errorf("cache misses delta = %.0f, want 1", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) {
|
||||
|
|
@ -370,7 +290,7 @@ func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) {
|
|||
Filename: "missing-1.0.0.tgz",
|
||||
UpstreamURL: "https://example.com/missing.tgz",
|
||||
StoragePath: sql.NullString{String: "nonexistent/path.tgz", Valid: true},
|
||||
ContentHash: sql.NullString{String: sha256Hex("missing content"), Valid: true},
|
||||
ContentHash: sql.NullString{String: "hash", Valid: true},
|
||||
Size: sql.NullInt64{Int64: 100, Valid: true},
|
||||
ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
|
||||
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
|
|
@ -613,7 +533,6 @@ func TestServeArtifact_Stream(t *testing.T) {
|
|||
func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) {
|
||||
proxy, db, store, fetcher := setupTestProxy(t)
|
||||
seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content")
|
||||
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi"))
|
||||
|
||||
result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "https://pypi.org/files/requests-2.28.0.tar.gz")
|
||||
if err != nil {
|
||||
|
|
@ -627,15 +546,10 @@ func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) {
|
|||
if fetcher.fetchCalled {
|
||||
t.Error("fetcher should not be called on cache hit")
|
||||
}
|
||||
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi"))
|
||||
if diff := missesAfter - missesBefore; diff != 0 {
|
||||
t.Errorf("cache misses delta = %.0f, want 0", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) {
|
||||
proxy, _, store, fetcher := setupTestProxy(t)
|
||||
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi"))
|
||||
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("fetched content")),
|
||||
|
|
@ -668,10 +582,6 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) {
|
|||
if _, ok := store.files[storagePath]; !ok {
|
||||
t.Error("artifact was not stored in storage")
|
||||
}
|
||||
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi"))
|
||||
if diff := missesAfter - missesBefore; diff != 1 {
|
||||
t.Errorf("cache misses delta = %.0f, want 1", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) {
|
||||
|
|
@ -961,8 +871,6 @@ func TestProxyCached_NoValidators_OmitsHeaders(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) {
|
||||
hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test"))
|
||||
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test"))
|
||||
upstreamHits := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
upstreamHits++
|
||||
|
|
@ -989,12 +897,6 @@ func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) {
|
|||
if upstreamHits != 1 {
|
||||
t.Fatalf("expected 1 upstream hit, got %d", upstreamHits)
|
||||
}
|
||||
if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) - missesBefore; diff != 1 {
|
||||
t.Errorf("cache misses delta after first request = %.0f, want 1", diff)
|
||||
}
|
||||
if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test")) - hitsBefore; diff != 0 {
|
||||
t.Errorf("cache hits delta after first request = %.0f, want 0", diff)
|
||||
}
|
||||
|
||||
// Second request within TTL should serve from cache without hitting upstream
|
||||
body, _, err = proxy.FetchOrCacheMetadata(ctx, "test", "ttl-pkg", upstream.URL+"/pkg")
|
||||
|
|
@ -1007,16 +909,9 @@ func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) {
|
|||
if upstreamHits != 1 {
|
||||
t.Errorf("expected upstream to still be hit only once, got %d", upstreamHits)
|
||||
}
|
||||
if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test")) - hitsBefore; diff != 1 {
|
||||
t.Errorf("cache hits delta after second request = %.0f, want 1", diff)
|
||||
}
|
||||
if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) - missesBefore; diff != 1 {
|
||||
t.Errorf("cache misses delta after second request = %.0f, want 1", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) {
|
||||
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test"))
|
||||
upstreamHits := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
upstreamHits++
|
||||
|
|
@ -1045,40 +940,6 @@ func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) {
|
|||
if upstreamHits != 2 {
|
||||
t.Errorf("expected 2 upstream hits with TTL=0, got %d", upstreamHits)
|
||||
}
|
||||
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test"))
|
||||
if diff := missesAfter - missesBefore; diff != 2 {
|
||||
t.Errorf("cache misses delta = %.0f, want 2", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchOrCacheMetadata_CacheDisabledDoesNotRecordMetrics(t *testing.T) {
|
||||
const ecosystem = "metadata-disabled"
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"v":1}`))
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
|
||||
hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem))
|
||||
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem))
|
||||
|
||||
_, _, err := proxy.FetchOrCacheMetadata(context.Background(), ecosystem, "pkg", upstream.URL+"/pkg")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch metadata: %v", err)
|
||||
}
|
||||
|
||||
hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem))
|
||||
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem))
|
||||
if diff := hitsAfter - hitsBefore; diff != 0 {
|
||||
t.Errorf("cache hits delta = %.0f, want 0", diff)
|
||||
}
|
||||
if diff := missesAfter - missesBefore; diff != 0 {
|
||||
t.Errorf("cache misses delta = %.0f, want 0", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyCached_StaleWarningHeader(t *testing.T) {
|
||||
|
|
@ -1143,33 +1004,3 @@ func TestProxyCached_FreshResponse_NoWarningHeader(t *testing.T) {
|
|||
t.Errorf("Warning should be empty for fresh response, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCanonicalPackagePURLMatchesConfig ensures the runtime cooldown lookup key
|
||||
// agrees with config.CooldownConfig.NormalizedPackages for the same package,
|
||||
// so a configured override is actually found regardless of how the user wrote it.
|
||||
func TestCanonicalPackagePURLMatchesConfig(t *testing.T) {
|
||||
tests := []struct {
|
||||
ecosystem string
|
||||
requestName string
|
||||
configKey string
|
||||
}{
|
||||
{"npm", "@babel/core", "pkg:npm/@babel/core"},
|
||||
{"npm", "@babel/core", "pkg:npm/%40babel/core"},
|
||||
{"npm", "@typescript/typescript-darwin-arm64", "pkg:npm/@typescript/typescript-darwin-arm64"},
|
||||
{"pypi", "Django", "pkg:pypi/Django"},
|
||||
{"pypi", "django", "pkg:pypi/Django"},
|
||||
{"composer", "symfony/console", "pkg:composer/Symfony/Console"},
|
||||
{"cargo", "serde", "pkg:cargo/serde"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.ecosystem+"/"+tt.requestName+"<="+tt.configKey, func(t *testing.T) {
|
||||
cfg := config.CooldownConfig{Packages: map[string]string{tt.configKey: "1d"}}
|
||||
normalized := cfg.NormalizedPackages()
|
||||
|
||||
lookup := canonicalPackagePURL(tt.ecosystem, tt.requestName)
|
||||
if _, ok := normalized[lookup]; !ok {
|
||||
t.Errorf("lookup key %q not found in normalized config %v", lookup, normalized)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,364 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
helmMetadataEcosystem = "helm"
|
||||
helmIndexFilename = "index.yaml"
|
||||
sha256HexLength = 64
|
||||
)
|
||||
|
||||
// HelmHandler serves read-only HTTP Helm chart repositories. Each configured
|
||||
// repository is mounted at /helm/{repository}/.
|
||||
type HelmHandler struct {
|
||||
proxy *Proxy
|
||||
proxyURL string
|
||||
repositories map[string]string
|
||||
}
|
||||
|
||||
// NewHelmHandler creates a Helm chart repository protocol handler.
|
||||
func NewHelmHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *HelmHandler {
|
||||
h := &HelmHandler{
|
||||
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
||||
repositories: make(map[string]string, len(repositories)),
|
||||
proxy: proxy,
|
||||
}
|
||||
for name, repositoryURL := range repositories {
|
||||
h.repositories[name] = strings.TrimSuffix(repositoryURL, "/")
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Routes returns the HTTP handler for Helm chart repository requests.
|
||||
func (h *HelmHandler) Routes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /{repository}/index.yaml", h.handleIndex)
|
||||
mux.HandleFunc("GET /{repository}/charts/{digest}/{filename}", h.handleChart)
|
||||
return mux
|
||||
}
|
||||
|
||||
func (h *HelmHandler) handleIndex(w http.ResponseWriter, r *http.Request) {
|
||||
repository, upstreamURL, ok := h.repositoryForRequest(r)
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
body, contentType, err := h.fetchIndex(r, repository, upstreamURL)
|
||||
if err != nil {
|
||||
h.serveIndexError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
rewritten, err := h.rewriteIndex(repository, upstreamURL, body)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Warn("failed to rewrite Helm index", "repository", repository, "error", err)
|
||||
http.Error(w, "invalid Helm repository index", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.writeMetadataCachedResponse(w, r, helmMetadataEcosystem, h.indexCacheKey(repository, upstreamURL), rewritten, contentType)
|
||||
}
|
||||
|
||||
func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) {
|
||||
repository, upstreamURL, ok := h.repositoryForRequest(r)
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
digest, ok := normalizeHelmDigest(r.PathValue("digest"))
|
||||
filename := r.PathValue("filename")
|
||||
if !ok || filename == "" || strings.Contains(filename, "/") || containsPathTraversal(filename) {
|
||||
http.Error(w, "invalid chart request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("failed to check Helm chart cache", "error", err)
|
||||
http.Error(w, "failed to check chart cache", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if cached != nil {
|
||||
h.serveChart(w, r, repository, digest, filename, cached)
|
||||
return
|
||||
}
|
||||
|
||||
body, _, err := h.fetchIndex(r, repository, upstreamURL)
|
||||
if err != nil {
|
||||
h.serveIndexError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
downloadURL, err := h.findChartDownload(upstreamURL, body, digest, filename)
|
||||
if err != nil {
|
||||
if errors.Is(err, errHelmChartNotFound) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
h.proxy.Logger.Warn("failed to read Helm index", "repository", repository, "error", err)
|
||||
http.Error(w, "invalid Helm repository index", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(
|
||||
r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch chart")
|
||||
return
|
||||
}
|
||||
h.serveChart(w, r, repository, digest, filename, result)
|
||||
}
|
||||
|
||||
func (h *HelmHandler) serveChart(w http.ResponseWriter, r *http.Request, repository, digest, filename string, result *CacheResult) {
|
||||
if !strings.EqualFold(result.Hash, digest) {
|
||||
if result.Reader != nil {
|
||||
_ = result.Reader.Close()
|
||||
}
|
||||
if clearErr := h.proxy.ClearCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename); clearErr != nil {
|
||||
h.proxy.Logger.Warn("failed to clear Helm chart with invalid digest", "error", clearErr)
|
||||
}
|
||||
http.Error(w, "chart digest verification failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
if result.ContentType == "" {
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
}
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
func (h *HelmHandler) repositoryForRequest(r *http.Request) (name, upstreamURL string, ok bool) {
|
||||
name = r.PathValue("repository")
|
||||
upstreamURL, ok = h.repositories[name]
|
||||
return name, upstreamURL, ok
|
||||
}
|
||||
|
||||
func (h *HelmHandler) fetchIndex(r *http.Request, repository, upstreamURL string) ([]byte, string, error) {
|
||||
return h.proxy.FetchOrCacheMetadata(
|
||||
r.Context(),
|
||||
helmMetadataEcosystem,
|
||||
h.indexCacheKey(repository, upstreamURL),
|
||||
upstreamURL+"/"+helmIndexFilename,
|
||||
"application/x-yaml, text/yaml;q=0.9, */*;q=0.1",
|
||||
)
|
||||
}
|
||||
|
||||
func (h *HelmHandler) indexCacheKey(repository, upstreamURL string) string {
|
||||
identity := repository + "\x00" + upstreamURL
|
||||
digest := sha256.Sum256([]byte(identity))
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func (h *HelmHandler) serveIndexError(w http.ResponseWriter, err error) {
|
||||
if errors.Is(err, ErrUpstreamNotFound) {
|
||||
http.Error(w, "Helm repository not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
h.proxy.Logger.Error("failed to fetch Helm index", "error", err)
|
||||
http.Error(w, "failed to fetch Helm repository index", http.StatusBadGateway)
|
||||
}
|
||||
|
||||
func (h *HelmHandler) rewriteIndex(repository, upstreamURL string, body []byte) ([]byte, error) {
|
||||
document, entries, err := parseHelmIndex(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := 0; i < len(entries.Content); i += 2 {
|
||||
chartName := entries.Content[i].Value
|
||||
releases := entries.Content[i+1]
|
||||
if releases.Kind != yaml.SequenceNode {
|
||||
return nil, fmt.Errorf("chart %q releases must be a sequence", chartName)
|
||||
}
|
||||
|
||||
filtered := make([]*yaml.Node, 0, len(releases.Content))
|
||||
for _, release := range releases.Content {
|
||||
chart, err := h.parseChartRelease(chartName, upstreamURL, release)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if h.chartOnCooldown(chartName, chart.created) {
|
||||
continue
|
||||
}
|
||||
for _, download := range chart.downloads {
|
||||
download.node.Value = h.chartProxyURL(repository, chart.digest, download.filename)
|
||||
}
|
||||
filtered = append(filtered, release)
|
||||
}
|
||||
releases.Content = filtered
|
||||
}
|
||||
|
||||
return yaml.Marshal(document)
|
||||
}
|
||||
|
||||
func (h *HelmHandler) findChartDownload(upstreamURL string, body []byte, digest, filename string) (string, error) {
|
||||
_, entries, err := parseHelmIndex(body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for i := 0; i < len(entries.Content); i += 2 {
|
||||
chartName := entries.Content[i].Value
|
||||
releases := entries.Content[i+1]
|
||||
if releases.Kind != yaml.SequenceNode {
|
||||
return "", fmt.Errorf("chart %q releases must be a sequence", chartName)
|
||||
}
|
||||
for _, release := range releases.Content {
|
||||
chart, err := h.parseChartRelease(chartName, upstreamURL, release)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if chart.digest != digest || h.chartOnCooldown(chartName, chart.created) {
|
||||
continue
|
||||
}
|
||||
for _, download := range chart.downloads {
|
||||
if download.filename == filename {
|
||||
return download.url, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", errHelmChartNotFound
|
||||
}
|
||||
|
||||
func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool {
|
||||
return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() &&
|
||||
!h.proxy.Cooldown.IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created)
|
||||
}
|
||||
|
||||
type helmChartDownload struct {
|
||||
node *yaml.Node
|
||||
url string
|
||||
filename string
|
||||
}
|
||||
|
||||
type helmChartRelease struct {
|
||||
created time.Time
|
||||
digest string
|
||||
downloads []helmChartDownload
|
||||
}
|
||||
|
||||
var errHelmChartNotFound = errors.New("chart not found in Helm index")
|
||||
|
||||
func (h *HelmHandler) parseChartRelease(chartName, upstreamURL string, release *yaml.Node) (helmChartRelease, error) {
|
||||
digestNode := helmMappingValue(release, "digest")
|
||||
urlsNode := helmMappingValue(release, "urls")
|
||||
if digestNode == nil || urlsNode == nil || urlsNode.Kind != yaml.SequenceNode || len(urlsNode.Content) == 0 {
|
||||
return helmChartRelease{}, fmt.Errorf("chart %q has no digest or URLs", chartName)
|
||||
}
|
||||
digest, ok := normalizeHelmDigest(digestNode.Value)
|
||||
if !ok {
|
||||
return helmChartRelease{}, fmt.Errorf("chart %q has invalid digest", chartName)
|
||||
}
|
||||
|
||||
baseURL, err := url.Parse(upstreamURL + "/" + helmIndexFilename)
|
||||
if err != nil {
|
||||
return helmChartRelease{}, fmt.Errorf("parsing Helm repository URL: %w", err)
|
||||
}
|
||||
|
||||
chart := helmChartRelease{digest: digest}
|
||||
if createdNode := helmMappingValue(release, "created"); createdNode != nil && createdNode.Value != "" {
|
||||
chart.created, err = time.Parse(time.RFC3339Nano, createdNode.Value)
|
||||
if err != nil {
|
||||
return helmChartRelease{}, fmt.Errorf("chart %q has invalid creation time: %w", chartName, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, urlNode := range urlsNode.Content {
|
||||
if urlNode.Kind != yaml.ScalarNode {
|
||||
return helmChartRelease{}, fmt.Errorf("chart %q has invalid URL", chartName)
|
||||
}
|
||||
reference, err := url.Parse(urlNode.Value)
|
||||
if err != nil {
|
||||
return helmChartRelease{}, fmt.Errorf("parsing chart %q URL: %w", chartName, err)
|
||||
}
|
||||
downloadURL := baseURL.ResolveReference(reference)
|
||||
if (downloadURL.Scheme != "http" && downloadURL.Scheme != "https") || downloadURL.Host == "" {
|
||||
return helmChartRelease{}, fmt.Errorf("chart %q URL must be HTTP(S)", chartName)
|
||||
}
|
||||
filename := path.Base(downloadURL.Path)
|
||||
if filename == "." || filename == "/" || filename == "" || !strings.HasSuffix(filename, ".tgz") {
|
||||
return helmChartRelease{}, fmt.Errorf("chart %q URL must point to a .tgz file", chartName)
|
||||
}
|
||||
chart.downloads = append(chart.downloads, helmChartDownload{
|
||||
node: urlNode,
|
||||
url: downloadURL.String(),
|
||||
filename: filename,
|
||||
})
|
||||
}
|
||||
return chart, nil
|
||||
}
|
||||
|
||||
func (h *HelmHandler) chartProxyURL(repository, digest, filename string) string {
|
||||
return fmt.Sprintf("%s/helm/%s/charts/%s/%s", h.proxyURL,
|
||||
url.PathEscape(repository), digest, url.PathEscape(filename))
|
||||
}
|
||||
|
||||
func parseHelmIndex(body []byte) (*yaml.Node, *yaml.Node, error) {
|
||||
var document yaml.Node
|
||||
if err := yaml.Unmarshal(body, &document); err != nil {
|
||||
return nil, nil, fmt.Errorf("parsing Helm index: %w", err)
|
||||
}
|
||||
entries, err := helmIndexEntries(&document)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return &document, entries, nil
|
||||
}
|
||||
|
||||
func helmIndexEntries(document *yaml.Node) (*yaml.Node, error) {
|
||||
if document == nil {
|
||||
return nil, errors.New("helm index is empty")
|
||||
}
|
||||
if len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode {
|
||||
return nil, errors.New("helm index must be a mapping")
|
||||
}
|
||||
entries := helmMappingValue(document.Content[0], "entries")
|
||||
if entries == nil || entries.Kind != yaml.MappingNode {
|
||||
return nil, errors.New("helm index has no entries mapping")
|
||||
}
|
||||
if len(entries.Content)%2 != 0 {
|
||||
return nil, errors.New("helm index entries mapping has an incomplete key-value pair")
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func helmMappingValue(mapping *yaml.Node, key string) *yaml.Node {
|
||||
if mapping == nil || mapping.Kind != yaml.MappingNode {
|
||||
return nil
|
||||
}
|
||||
for i := 0; i+1 < len(mapping.Content); i += 2 {
|
||||
if mapping.Content[i].Value == key {
|
||||
return mapping.Content[i+1]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeHelmDigest(value string) (string, bool) {
|
||||
digest := strings.TrimPrefix(strings.ToLower(value), "sha256:")
|
||||
if len(digest) != sha256HexLength {
|
||||
return "", false
|
||||
}
|
||||
for _, char := range digest {
|
||||
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return digest, true
|
||||
}
|
||||
|
|
@ -1,337 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestHelmHandler_RewritesIndexAndCachesChart(t *testing.T) {
|
||||
chart := []byte("a Helm chart")
|
||||
digest := helmSHA256Hex(chart)
|
||||
var available atomic.Bool
|
||||
available.Store(true)
|
||||
var indexRequests atomic.Int32
|
||||
var chartRequests atomic.Int32
|
||||
|
||||
var upstream *httptest.Server
|
||||
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !available.Load() {
|
||||
http.Error(w, "unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/charts/index.yaml":
|
||||
indexRequests.Add(1)
|
||||
w.Header().Set("Content-Type", "application/x-yaml")
|
||||
_, _ = fmt.Fprintf(w, `apiVersion: v1
|
||||
entries:
|
||||
demo:
|
||||
- annotations:
|
||||
example.com/retained: "true"
|
||||
created: 2020-01-02T03:04:05Z
|
||||
digest: %s
|
||||
name: demo
|
||||
urls:
|
||||
- demo-1.0.0.tgz
|
||||
- %s/charts/mirror/demo-1.0.0.tgz
|
||||
version: 1.0.0
|
||||
generated: 2020-01-02T03:04:05Z
|
||||
`, digest, upstream.URL)
|
||||
case "/charts/demo-1.0.0.tgz", "/charts/mirror/demo-1.0.0.tgz":
|
||||
chartRequests.Add(1)
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
_, _ = w.Write(chart)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.CacheMetadata = true
|
||||
proxy.MetadataTTL = time.Hour
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
|
||||
proxy.Fetcher = fetcher
|
||||
t.Cleanup(func() { _ = fetcher.Close() })
|
||||
|
||||
h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL + "/charts"})
|
||||
|
||||
indexResponse := serveHelmRequest(h, "/stable/index.yaml")
|
||||
if indexResponse.Code != http.StatusOK {
|
||||
t.Fatalf("index status = %d, want 200: %s", indexResponse.Code, indexResponse.Body.String())
|
||||
}
|
||||
if got := indexResponse.Header().Get("Content-Type"); got != "application/x-yaml" {
|
||||
t.Errorf("index Content-Type = %q, want application/x-yaml", got)
|
||||
}
|
||||
if strings.Contains(indexResponse.Body.String(), upstream.URL) {
|
||||
t.Errorf("rewritten index contains upstream URL: %s", indexResponse.Body.String())
|
||||
}
|
||||
if !strings.Contains(indexResponse.Body.String(), "example.com/retained") {
|
||||
t.Errorf("rewritten index lost an unrelated field: %s", indexResponse.Body.String())
|
||||
}
|
||||
|
||||
var index map[string]any
|
||||
if err := yaml.Unmarshal(indexResponse.Body.Bytes(), &index); err != nil {
|
||||
t.Fatalf("parse rewritten index: %v", err)
|
||||
}
|
||||
entries := index["entries"].(map[string]any)
|
||||
release := entries["demo"].([]any)[0].(map[string]any)
|
||||
urls := release["urls"].([]any)
|
||||
wantURL := "http://proxy.example/helm/stable/charts/" + digest + "/demo-1.0.0.tgz"
|
||||
for _, rawURL := range urls {
|
||||
if rawURL != wantURL {
|
||||
t.Errorf("rewritten URL = %q, want %q", rawURL, wantURL)
|
||||
}
|
||||
}
|
||||
|
||||
firstChart := serveHelmRequest(h, "/stable/charts/"+digest+"/demo-1.0.0.tgz")
|
||||
if firstChart.Code != http.StatusOK {
|
||||
t.Fatalf("chart status = %d, want 200: %s", firstChart.Code, firstChart.Body.String())
|
||||
}
|
||||
if got := firstChart.Body.String(); got != string(chart) {
|
||||
t.Errorf("chart body = %q, want %q", got, chart)
|
||||
}
|
||||
if got := firstChart.Header().Get("Content-Type"); got != "application/gzip" {
|
||||
t.Errorf("chart Content-Type = %q, want application/gzip", got)
|
||||
}
|
||||
|
||||
// Artifact cache availability must not depend on metadata caching or a
|
||||
// reachable index upstream.
|
||||
proxy.CacheMetadata = false
|
||||
available.Store(false)
|
||||
cachedChart := serveHelmRequest(h, "/stable/charts/"+digest+"/demo-1.0.0.tgz")
|
||||
if cachedChart.Code != http.StatusOK {
|
||||
t.Fatalf("cached chart status = %d, want 200: %s", cachedChart.Code, cachedChart.Body.String())
|
||||
}
|
||||
if got := cachedChart.Body.String(); got != string(chart) {
|
||||
t.Errorf("cached chart body = %q, want %q", got, chart)
|
||||
}
|
||||
if got := indexRequests.Load(); got != 1 {
|
||||
t.Errorf("index requests = %d, want 1", got)
|
||||
}
|
||||
if got := chartRequests.Load(); got != 1 {
|
||||
t.Errorf("chart requests = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmHandler_RejectsChartDigestMismatch(t *testing.T) {
|
||||
chart := []byte("tampered chart")
|
||||
digest := helmSHA256Hex([]byte("expected chart"))
|
||||
requests := 0
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/index.yaml":
|
||||
_, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", digest)
|
||||
case "/demo.tgz":
|
||||
requests++
|
||||
_, _ = w.Write(chart)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, store, _ := setupTestProxy(t)
|
||||
proxy.CacheMetadata = true
|
||||
proxy.MetadataTTL = time.Hour
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
|
||||
proxy.Fetcher = fetcher
|
||||
t.Cleanup(func() { _ = fetcher.Close() })
|
||||
h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"test": upstream.URL})
|
||||
|
||||
for range 2 {
|
||||
response := serveHelmRequest(h, "/test/charts/"+digest+"/demo.tgz")
|
||||
if response.Code != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want 502: %s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
if requests != 2 {
|
||||
t.Errorf("chart requests = %d, want 2 after invalid cache entry is cleared", requests)
|
||||
}
|
||||
storagePath := storage.ArtifactPath(helmMetadataEcosystem, "", "test", digest, "demo.tgz")
|
||||
if exists, err := store.Exists(t.Context(), storagePath); err != nil {
|
||||
t.Fatalf("checking rejected chart storage: %v", err)
|
||||
} else if exists {
|
||||
t.Errorf("rejected chart remains in storage at %q", storagePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmHandler_IndexCacheChangesWithUpstreamURL(t *testing.T) {
|
||||
firstDigest := strings.Repeat("a", sha256HexLength)
|
||||
secondDigest := strings.Repeat("b", sha256HexLength)
|
||||
firstRequests := 0
|
||||
secondRequests := 0
|
||||
first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
firstRequests++
|
||||
_, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", firstDigest)
|
||||
}))
|
||||
defer first.Close()
|
||||
second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
secondRequests++
|
||||
_, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", secondDigest)
|
||||
}))
|
||||
defer second.Close()
|
||||
|
||||
proxy, db, store, _ := setupTestProxy(t)
|
||||
proxy.CacheMetadata = true
|
||||
proxy.MetadataTTL = time.Hour
|
||||
proxy.HTTPClient = first.Client()
|
||||
firstHandler := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": first.URL})
|
||||
if response := serveHelmRequest(firstHandler, "/stable/index.yaml"); response.Code != http.StatusOK {
|
||||
t.Fatalf("first index status = %d, want 200: %s", response.Code, response.Body.String())
|
||||
}
|
||||
|
||||
// Model a restarted server with the same database and storage but a changed
|
||||
// repository URL. Its cache key must not reuse the previous index or ETag.
|
||||
restartedProxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), nil)
|
||||
restartedProxy.CacheMetadata = true
|
||||
restartedProxy.MetadataTTL = time.Hour
|
||||
restartedProxy.HTTPClient = second.Client()
|
||||
secondHandler := NewHelmHandler(restartedProxy, "http://proxy.example", map[string]string{"stable": second.URL})
|
||||
response := serveHelmRequest(secondHandler, "/stable/index.yaml")
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("second index status = %d, want 200: %s", response.Code, response.Body.String())
|
||||
}
|
||||
if !strings.Contains(response.Body.String(), secondDigest) {
|
||||
t.Errorf("second index did not use the new upstream: %s", response.Body.String())
|
||||
}
|
||||
if firstRequests != 1 {
|
||||
t.Errorf("first upstream requests = %d, want 1", firstRequests)
|
||||
}
|
||||
if secondRequests != 1 {
|
||||
t.Errorf("second upstream requests = %d, want 1", secondRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmHandler_UsesConfiguredUpstreamAuthentication(t *testing.T) {
|
||||
chart := []byte("private Helm chart")
|
||||
digest := helmSHA256Hex(chart)
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer private-token" {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/index.yaml":
|
||||
_, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", digest)
|
||||
case "/demo.tgz":
|
||||
_, _ = w.Write(chart)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.CacheMetadata = true
|
||||
proxy.MetadataTTL = time.Hour
|
||||
authClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport,
|
||||
upstreamhttp.AuthFunc(func(string) (string, string) {
|
||||
return "Authorization", "Bearer private-token"
|
||||
}))}
|
||||
proxy.HTTPClient = authClient
|
||||
fetcher := fetch.NewFetcher(fetch.WithHTTPClient(authClient), fetch.WithMaxRetries(0))
|
||||
proxy.Fetcher = fetcher
|
||||
t.Cleanup(func() { _ = fetcher.Close() })
|
||||
h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"private": upstream.URL})
|
||||
|
||||
response := serveHelmRequest(h, "/private/charts/"+digest+"/demo.tgz")
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200: %s", response.Code, response.Body.String())
|
||||
}
|
||||
if got := response.Body.String(); got != string(chart) {
|
||||
t.Errorf("body = %q, want %q", got, chart)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmHandler_FiltersNewChartsFromIndex(t *testing.T) {
|
||||
oldDigest := strings.Repeat("a", 64)
|
||||
newDigest := strings.Repeat("b", 64)
|
||||
proxy := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}}
|
||||
h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"test": "https://charts.example"})
|
||||
|
||||
body := fmt.Sprintf(`apiVersion: v1
|
||||
entries:
|
||||
demo:
|
||||
- created: %s
|
||||
digest: %s
|
||||
urls: [demo-old.tgz]
|
||||
- created: %s
|
||||
digest: %s
|
||||
urls: [demo-new.tgz]
|
||||
`, time.Now().Add(-10*24*time.Hour).Format(time.RFC3339), oldDigest,
|
||||
time.Now().Add(-time.Hour).Format(time.RFC3339), newDigest)
|
||||
|
||||
rewritten, err := h.rewriteIndex("test", "https://charts.example", []byte(body))
|
||||
if err != nil {
|
||||
t.Fatalf("rewriteIndex() error = %v", err)
|
||||
}
|
||||
if strings.Contains(string(rewritten), newDigest) {
|
||||
t.Errorf("rewritten index includes a chart still in cooldown: %s", rewritten)
|
||||
}
|
||||
if !strings.Contains(string(rewritten), oldDigest) {
|
||||
t.Errorf("rewritten index omitted an old chart: %s", rewritten)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeHelmDigest(t *testing.T) {
|
||||
digest := strings.Repeat("a", 64)
|
||||
for _, input := range []string{digest, "sha256:" + digest, "SHA256:" + strings.ToUpper(digest)} {
|
||||
if got, ok := normalizeHelmDigest(input); !ok || got != digest {
|
||||
t.Errorf("normalizeHelmDigest(%q) = %q, %t; want %q, true", input, got, ok, digest)
|
||||
}
|
||||
}
|
||||
if _, ok := normalizeHelmDigest("bad"); ok {
|
||||
t.Error("normalizeHelmDigest accepted an invalid digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmIndexEntriesRejectsIncompleteMapping(t *testing.T) {
|
||||
entries := &yaml.Node{
|
||||
Kind: yaml.MappingNode,
|
||||
Content: []*yaml.Node{
|
||||
{Kind: yaml.ScalarNode, Value: "demo"},
|
||||
},
|
||||
}
|
||||
document := &yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{{
|
||||
Kind: yaml.MappingNode,
|
||||
Content: []*yaml.Node{
|
||||
{Kind: yaml.ScalarNode, Value: "entries"},
|
||||
entries,
|
||||
},
|
||||
}},
|
||||
}
|
||||
|
||||
if _, err := helmIndexEntries(document); err == nil {
|
||||
t.Fatal("helmIndexEntries() error = nil, want incomplete mapping error")
|
||||
}
|
||||
}
|
||||
|
||||
func serveHelmRequest(h *HelmHandler, target string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil))
|
||||
return w
|
||||
}
|
||||
|
||||
func helmSHA256Hex(data []byte) string {
|
||||
digest := sha256.Sum256(data)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
"google.golang.org/protobuf/encoding/protowire"
|
||||
)
|
||||
|
||||
|
|
@ -53,13 +54,30 @@ func (h *HexHandler) Routes() http.Handler {
|
|||
|
||||
// handleDownload serves a package tarball, fetching and caching from upstream if needed.
|
||||
func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
||||
h.proxy.handleFilenameDownload(w, r, filenameDownload{
|
||||
ecosystem: "hex",
|
||||
suffix: ".tar",
|
||||
parseErr: "could not parse tarball filename",
|
||||
fetchErr: "failed to fetch package",
|
||||
parse: h.parseTarballFilename,
|
||||
})
|
||||
filename := r.PathValue("filename")
|
||||
if filename == "" || !strings.HasSuffix(filename, ".tar") {
|
||||
http.Error(w, "invalid filename", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract name and version from filename (e.g., "phoenix-1.7.10.tar")
|
||||
name, version := h.parseTarballFilename(filename)
|
||||
if name == "" || version == "" {
|
||||
http.Error(w, "could not parse tarball filename", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.Logger.Info("hex download request",
|
||||
"name", name, "version", version, "filename", filename)
|
||||
|
||||
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "hex", name, version, filename)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// parseTarballFilename extracts name and version from a hex tarball filename.
|
||||
|
|
@ -219,7 +237,7 @@ func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[st
|
|||
return nil, err
|
||||
}
|
||||
|
||||
packagePURL := canonicalPackagePURL("hex", name)
|
||||
packagePURL := purl.MakePURLString("hex", name, "")
|
||||
filtered := make(map[string]bool)
|
||||
|
||||
for _, release := range pkg.Releases {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
"google.golang.org/protobuf/encoding/protowire"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/git-pkgs/integrity"
|
||||
)
|
||||
|
||||
type integrityChecks struct {
|
||||
contentHash integrity.SRI
|
||||
native integrity.SRI
|
||||
algorithms []integrity.Algorithm
|
||||
}
|
||||
|
||||
func newIntegrityChecks(contentHash, native string) (integrityChecks, error) {
|
||||
checks := integrityChecks{}
|
||||
|
||||
if contentHash != "" {
|
||||
digest, err := integrity.ParseHex(integrity.SHA256, contentHash)
|
||||
if err != nil {
|
||||
return integrityChecks{}, fmt.Errorf("parse content_hash: %w", err)
|
||||
}
|
||||
checks.contentHash = integrity.SRI{digest}
|
||||
checks.algorithms = append(checks.algorithms, integrity.SHA256)
|
||||
}
|
||||
|
||||
if native != "" {
|
||||
digests, err := integrity.ParseSRI(native)
|
||||
if err != nil {
|
||||
return integrityChecks{}, fmt.Errorf("parse integrity: %w", err)
|
||||
}
|
||||
checks.native = digests
|
||||
for _, digest := range digests {
|
||||
checks.algorithms = append(checks.algorithms, digest.Algorithm())
|
||||
}
|
||||
}
|
||||
|
||||
return checks, nil
|
||||
}
|
||||
|
||||
func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) {
|
||||
if len(c.algorithms) == 0 {
|
||||
return source, nil
|
||||
}
|
||||
reader, err := integrity.NewReader(source, c.algorithms...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create integrity reader: %w", err)
|
||||
}
|
||||
return &verifyingReader{
|
||||
source: source,
|
||||
reader: reader,
|
||||
checks: c,
|
||||
onMismatch: onMismatch,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// verifyingReader forwards Close to its source and reports completed digest
|
||||
// mismatches after its shared integrity reader observes EOF.
|
||||
type verifyingReader struct {
|
||||
source io.ReadCloser
|
||||
reader *integrity.Reader
|
||||
checks integrityChecks
|
||||
onMismatch func(reason string)
|
||||
verified bool
|
||||
}
|
||||
|
||||
func (r *verifyingReader) Read(p []byte) (int, error) {
|
||||
n, err := r.reader.Read(p)
|
||||
if err == io.EOF {
|
||||
r.verify()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *verifyingReader) Close() error {
|
||||
return r.source.Close()
|
||||
}
|
||||
|
||||
func (r *verifyingReader) verify() {
|
||||
if r.verified {
|
||||
return
|
||||
}
|
||||
r.verified = true
|
||||
result := r.reader.Result()
|
||||
if !result.Complete {
|
||||
return
|
||||
}
|
||||
|
||||
if len(r.checks.contentHash) > 0 {
|
||||
if err := result.Verify(r.checks.contentHash); err != nil {
|
||||
r.onMismatch("content_hash: " + err.Error())
|
||||
}
|
||||
}
|
||||
if len(r.checks.native) > 0 {
|
||||
if err := result.Verify(r.checks.native); err != nil {
|
||||
r.onMismatch("integrity: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,250 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func sha256Hex(data string) string {
|
||||
sum := sha256.Sum256([]byte(data))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func sha256SRI(data string) string {
|
||||
sum := sha256.Sum256([]byte(data))
|
||||
return "sha256-" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func sha384SRI(data string) string {
|
||||
sum := sha512.Sum384([]byte(data))
|
||||
return "sha384-" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func sha512SRI(data string) string {
|
||||
sum := sha512.Sum512([]byte(data))
|
||||
return "sha512-" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func wrapIntegrityReader(t *testing.T, source io.ReadCloser, contentHash, native string, onMismatch func(string)) io.ReadCloser {
|
||||
t.Helper()
|
||||
checks, err := newIntegrityChecks(contentHash, native)
|
||||
if err != nil {
|
||||
t.Fatalf("newIntegrityChecks: %v", err)
|
||||
}
|
||||
reader, err := checks.wrap(source, onMismatch)
|
||||
if err != nil {
|
||||
t.Fatalf("wrap: %v", err)
|
||||
}
|
||||
return reader
|
||||
}
|
||||
|
||||
func TestNewIntegrityChecksCollectsAlgorithms(t *testing.T) {
|
||||
checks, err := newIntegrityChecks(
|
||||
sha256Hex("hello"),
|
||||
strings.Join([]string{sha256SRI("first"), sha512SRI("second"), sha384SRI("third"), sha512SRI("alternative")}, " "),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(checks.algorithms) != 5 {
|
||||
t.Fatalf("algorithms = %v, want 5 entries", checks.algorithms)
|
||||
}
|
||||
if len(checks.native) != 4 {
|
||||
t.Errorf("native digests = %d, want 4", len(checks.native))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewIntegrityChecksRejectsMalformedMetadata(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
contentHash string
|
||||
native string
|
||||
}{
|
||||
{name: "short content hash", contentHash: "abc123"},
|
||||
{name: "non-hex content hash", contentHash: strings.Repeat("z", sha256.Size*2)},
|
||||
{name: "missing SRI separator", native: "sha512"},
|
||||
{name: "malformed SRI base64", native: "sha512-not!base64"},
|
||||
{name: "wrong SRI length", native: "sha512-" + base64.StdEncoding.EncodeToString([]byte("short"))},
|
||||
{name: "unsupported SRI algorithm", native: "md5-1B2M2Y8AsgTpgAmY7PhCfg=="},
|
||||
{name: "invalid SRI alternative", native: sha512SRI("valid") + " sha384-nope"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if _, err := newIntegrityChecks(test.contentHash, test.native); err == nil {
|
||||
t.Fatal("newIntegrityChecks returned nil error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyingReader(t *testing.T) {
|
||||
const data = "hello world"
|
||||
goodSHA := sha256Hex(data)
|
||||
goodSRI := sha512SRI(data)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
hash string
|
||||
sri string
|
||||
wantCalls int
|
||||
}{
|
||||
{name: "both match", hash: goodSHA, sri: goodSRI},
|
||||
{name: "SHA-256 only match", hash: goodSHA},
|
||||
{name: "SRI only match", sri: goodSRI},
|
||||
{name: "SHA-256 mismatch", hash: sha256Hex("other"), wantCalls: 1},
|
||||
{name: "SRI mismatch", sri: sha512SRI("other"), wantCalls: 1},
|
||||
{name: "both mismatch", hash: sha256Hex("other"), sri: sha512SRI("other"), wantCalls: 2},
|
||||
{name: "no checks"},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var calls []string
|
||||
reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), test.hash, test.sri,
|
||||
func(reason string) { calls = append(calls, reason) })
|
||||
|
||||
got, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll: %v", err)
|
||||
}
|
||||
if string(got) != data {
|
||||
t.Errorf("data corrupted: got %q", got)
|
||||
}
|
||||
if err := reader.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
if len(calls) != test.wantCalls {
|
||||
t.Errorf("onMismatch called %d times, want %d: %v", len(calls), test.wantCalls, calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyingReaderUsesStrongestNativeAlgorithm(t *testing.T) {
|
||||
const data = "artifact"
|
||||
tests := []struct {
|
||||
name string
|
||||
native string
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
name: "weaker match does not override stronger mismatch",
|
||||
native: sha256SRI(data) + " " + sha512SRI("other"),
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "stronger match ignores weaker mismatch",
|
||||
native: sha256SRI("other") + " " + sha512SRI(data),
|
||||
},
|
||||
{
|
||||
name: "same algorithm alternative matches",
|
||||
native: sha512SRI("other") + " " + sha512SRI(data),
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var calls int
|
||||
reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), "", test.native, func(string) { calls++ })
|
||||
if _, err := io.Copy(io.Discard, reader); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != test.wantCalls {
|
||||
t.Errorf("onMismatch called %d times, want %d", calls, test.wantCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyingReaderMismatchMessages(t *testing.T) {
|
||||
const data = "actual"
|
||||
wantHash := sha256Hex("expected")
|
||||
wantSRI := sha512SRI("expected")
|
||||
var reasons []string
|
||||
reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), wantHash, wantSRI,
|
||||
func(reason string) { reasons = append(reasons, reason) })
|
||||
if _, err := io.Copy(io.Discard, reader); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(reasons) != 2 {
|
||||
t.Fatalf("reasons = %v, want two", reasons)
|
||||
}
|
||||
wantContentReason := "content_hash: integrity mismatch: expected " + sha256SRI("expected") + ", calculated " + sha256SRI(data)
|
||||
if reasons[0] != wantContentReason {
|
||||
t.Errorf("content reason = %q, want %q", reasons[0], wantContentReason)
|
||||
}
|
||||
wantNativeReason := "integrity: integrity mismatch: expected " + wantSRI + ", calculated " + sha512SRI(data)
|
||||
if reasons[1] != wantNativeReason {
|
||||
t.Errorf("native reason = %q, want %q", reasons[1], wantNativeReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyingReaderPassthrough(t *testing.T) {
|
||||
source := io.NopCloser(strings.NewReader("x"))
|
||||
reader := wrapIntegrityReader(t, source, "", "", func(string) { t.Fatal("should not be called") })
|
||||
if reader != source {
|
||||
t.Error("expected passthrough when no hashes were provided")
|
||||
}
|
||||
}
|
||||
|
||||
type closeTrackingReader struct {
|
||||
io.Reader
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (r *closeTrackingReader) Close() error {
|
||||
r.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestVerifyingReaderPartialRead(t *testing.T) {
|
||||
source := &closeTrackingReader{Reader: strings.NewReader("hello world")}
|
||||
var calls int
|
||||
reader := wrapIntegrityReader(t, source, sha256Hex("other"), "", func(string) { calls++ })
|
||||
|
||||
buffer := make([]byte, 5)
|
||||
_, _ = reader.Read(buffer)
|
||||
_ = reader.Close()
|
||||
|
||||
if calls != 0 {
|
||||
t.Errorf("onMismatch called %d times for partial read, want 0", calls)
|
||||
}
|
||||
if !source.closed {
|
||||
t.Error("Close was not forwarded to the source")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyingReaderNonEOFError(t *testing.T) {
|
||||
var calls int
|
||||
reader := wrapIntegrityReader(t, io.NopCloser(errorFixtureReader{}), sha256Hex("data"), "", func(string) { calls++ })
|
||||
if _, err := io.ReadAll(reader); !errors.Is(err, errIntegrityReadFixture) {
|
||||
t.Fatalf("ReadAll error = %v", err)
|
||||
}
|
||||
if calls != 0 {
|
||||
t.Errorf("onMismatch called %d times after non-EOF error", calls)
|
||||
}
|
||||
}
|
||||
|
||||
var errIntegrityReadFixture = errors.New("integrity read fixture")
|
||||
|
||||
type errorFixtureReader struct{}
|
||||
|
||||
func (errorFixtureReader) Read(p []byte) (int, error) {
|
||||
return copy(p, "data"), errIntegrityReadFixture
|
||||
}
|
||||
|
||||
func TestVerifyingReaderVerifyOnce(t *testing.T) {
|
||||
var calls int
|
||||
reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader("x")), sha256Hex("y"), "", func(string) { calls++ })
|
||||
_, _ = io.ReadAll(reader)
|
||||
_ = reader.Close()
|
||||
_ = reader.Close()
|
||||
if calls != 1 {
|
||||
t.Errorf("onMismatch called %d times, want 1", calls)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,344 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
)
|
||||
|
||||
const (
|
||||
juliaUpstream = "https://pkg.julialang.org"
|
||||
juliaGeneralRegistryUUID = "23338594-aafe-5451-b93e-139f81909106"
|
||||
juliaArtifactName = "_artifact"
|
||||
juliaRegistryName = "_registry"
|
||||
)
|
||||
|
||||
var (
|
||||
juliaHexPattern = regexp.MustCompile(`^[0-9a-f]{40,64}$`)
|
||||
juliaUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
|
||||
)
|
||||
|
||||
// JuliaHandler handles Julia Pkg server protocol requests.
|
||||
//
|
||||
// See https://pkgdocs.julialang.org/v1/registries/ and the PkgServer.jl
|
||||
// reference implementation. The protocol is content-addressed: registry,
|
||||
// package and artifact resources are all identified by git tree hashes
|
||||
// and are immutable once published.
|
||||
type JuliaHandler struct {
|
||||
proxy *Proxy
|
||||
upstreamURL string
|
||||
|
||||
mu sync.RWMutex
|
||||
names map[string]string
|
||||
namesHash string
|
||||
loadMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewJuliaHandler creates a new Julia Pkg server handler.
|
||||
func NewJuliaHandler(proxy *Proxy, _ string) *JuliaHandler {
|
||||
return &JuliaHandler{
|
||||
proxy: proxy,
|
||||
upstreamURL: juliaUpstream,
|
||||
names: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
// Routes returns the HTTP handler for Julia requests.
|
||||
func (h *JuliaHandler) Routes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("GET /registries", h.handleRegistries)
|
||||
mux.HandleFunc("GET /registries.eager", h.handleRegistries)
|
||||
mux.HandleFunc("GET /registries.conservative", h.handleRegistries)
|
||||
mux.HandleFunc("GET /registry/{uuid}/{hash}", h.handleRegistry)
|
||||
mux.HandleFunc("GET /package/{uuid}/{hash}", h.handlePackage)
|
||||
mux.HandleFunc("GET /artifact/{hash}", h.handleArtifact)
|
||||
mux.HandleFunc("GET /meta", h.proxyUpstream)
|
||||
|
||||
return mux
|
||||
}
|
||||
|
||||
// handleRegistries serves the list of available registries. This is the only
|
||||
// mutable endpoint in the protocol so it goes through the metadata cache.
|
||||
func (h *JuliaHandler) handleRegistries(w http.ResponseWriter, r *http.Request) {
|
||||
cacheKey := strings.TrimPrefix(r.URL.Path, "/")
|
||||
h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "julia", cacheKey, "*/*")
|
||||
}
|
||||
|
||||
// handleRegistry serves an immutable registry tarball and refreshes the
|
||||
// UUID→name map from its Registry.toml.
|
||||
func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) {
|
||||
uuid := r.PathValue("uuid")
|
||||
hash := r.PathValue("hash")
|
||||
if !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
|
||||
http.Error(w, "invalid registry reference", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.Logger.Info("julia registry request", "uuid", uuid, "hash", hash)
|
||||
|
||||
upstreamURL := h.upstreamURL + r.URL.Path
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch registry")
|
||||
return
|
||||
}
|
||||
|
||||
go h.refreshNamesFromRegistry(uuid, hash)
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// handlePackage serves an immutable package source tarball.
|
||||
func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) {
|
||||
uuid := r.PathValue("uuid")
|
||||
hash := r.PathValue("hash")
|
||||
if !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
|
||||
http.Error(w, "invalid package reference", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.ensureNames(r.Context()); err != nil {
|
||||
h.proxy.Logger.Warn("julia name map unavailable, using uuid", "error", err)
|
||||
}
|
||||
name := h.resolveName(uuid)
|
||||
|
||||
h.proxy.Logger.Info("julia package request", "name", name, "uuid", uuid, "hash", hash)
|
||||
|
||||
upstreamURL := h.upstreamURL + r.URL.Path
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
return
|
||||
}
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// handleArtifact serves an immutable binary artifact tarball. Artifacts are
|
||||
// anonymous content-addressed blobs with no associated package name.
|
||||
func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) {
|
||||
hash := r.PathValue("hash")
|
||||
if !juliaHexPattern.MatchString(hash) {
|
||||
http.Error(w, "invalid artifact hash", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.Logger.Info("julia artifact request", "hash", hash)
|
||||
|
||||
upstreamURL := h.upstreamURL + r.URL.Path
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch artifact")
|
||||
return
|
||||
}
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// proxyUpstream forwards a request to the upstream Pkg server without caching.
|
||||
func (h *JuliaHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil)
|
||||
}
|
||||
|
||||
// resolveName returns the human-readable package name for a UUID, falling
|
||||
// back to the UUID itself if it is not present in the loaded registry.
|
||||
func (h *JuliaHandler) resolveName(uuid string) string {
|
||||
h.mu.RLock()
|
||||
defer h.mu.RUnlock()
|
||||
if name, ok := h.names[uuid]; ok {
|
||||
return name
|
||||
}
|
||||
return uuid
|
||||
}
|
||||
|
||||
// ensureNames lazily populates the UUID→name map from the General registry.
|
||||
// Returns immediately if the map is already populated; otherwise blocks until
|
||||
// a single in-flight load completes. Failed loads are retried on the next call.
|
||||
func (h *JuliaHandler) ensureNames(ctx context.Context) error {
|
||||
if h.namesLoaded() {
|
||||
return nil
|
||||
}
|
||||
|
||||
h.loadMu.Lock()
|
||||
defer h.loadMu.Unlock()
|
||||
|
||||
if h.namesLoaded() {
|
||||
return nil
|
||||
}
|
||||
return h.loadNamesFromUpstream(ctx)
|
||||
}
|
||||
|
||||
func (h *JuliaHandler) namesLoaded() bool {
|
||||
h.mu.RLock()
|
||||
defer h.mu.RUnlock()
|
||||
return len(h.names) > 0
|
||||
}
|
||||
|
||||
// loadNamesFromUpstream fetches the current /registries listing, downloads the
|
||||
// General registry tarball at its current hash, and parses Registry.toml.
|
||||
func (h *JuliaHandler) loadNamesFromUpstream(ctx context.Context) error {
|
||||
hash, err := h.fetchGeneralRegistryHash(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return h.loadRegistryTarball(ctx, juliaGeneralRegistryUUID, hash)
|
||||
}
|
||||
|
||||
// fetchGeneralRegistryHash reads /registries and returns the current tree hash
|
||||
// for the General registry.
|
||||
func (h *JuliaHandler) fetchGeneralRegistryHash(ctx context.Context) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.upstreamURL+"/registries", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("upstream /registries returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
for scanner.Scan() {
|
||||
uuid, hash, ok := parseRegistryLine(scanner.Text())
|
||||
if ok && uuid == juliaGeneralRegistryUUID {
|
||||
return hash, nil
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "", fmt.Errorf("general registry not listed in /registries")
|
||||
}
|
||||
|
||||
// refreshNamesFromRegistry reloads the UUID→name map from a registry tarball
|
||||
// that has just been cached. Errors are logged but do not affect the response.
|
||||
func (h *JuliaHandler) refreshNamesFromRegistry(uuid, hash string) {
|
||||
if uuid != juliaGeneralRegistryUUID {
|
||||
return
|
||||
}
|
||||
h.mu.RLock()
|
||||
current := h.namesHash
|
||||
h.mu.RUnlock()
|
||||
if current == hash {
|
||||
return
|
||||
}
|
||||
if err := h.loadRegistryTarball(context.Background(), uuid, hash); err != nil {
|
||||
h.proxy.Logger.Warn("failed to refresh julia name map", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// loadRegistryTarball downloads a registry tarball and replaces the name map
|
||||
// with the contents of its Registry.toml.
|
||||
func (h *JuliaHandler) loadRegistryTarball(ctx context.Context, uuid, hash string) error {
|
||||
url := fmt.Sprintf("%s/registry/%s/%s", h.upstreamURL, uuid, hash)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("upstream registry returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
names, err := extractRegistryNames(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
h.mu.Lock()
|
||||
h.names = names
|
||||
h.namesHash = hash
|
||||
h.mu.Unlock()
|
||||
|
||||
h.proxy.Logger.Info("loaded julia registry name map", "packages", len(names), "hash", hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractRegistryNames reads a gzipped registry tarball, finds Registry.toml
|
||||
// at the root, and returns its [packages] table as a UUID→name map.
|
||||
func extractRegistryNames(r io.Reader) (map[string]string, error) {
|
||||
gz, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening gzip stream: %w", err)
|
||||
}
|
||||
defer func() { _ = gz.Close() }()
|
||||
|
||||
tr := tar.NewReader(gz)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil, fmt.Errorf("no Registry.toml in tarball")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimPrefix(hdr.Name, "./") != "Registry.toml" {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseRegistryToml(data)
|
||||
}
|
||||
}
|
||||
|
||||
type juliaRegistryFile struct {
|
||||
Packages map[string]struct {
|
||||
Name string `toml:"name"`
|
||||
} `toml:"packages"`
|
||||
}
|
||||
|
||||
// parseRegistryToml decodes the [packages] table of a Registry.toml file.
|
||||
func parseRegistryToml(data []byte) (map[string]string, error) {
|
||||
var reg juliaRegistryFile
|
||||
if _, err := toml.NewDecoder(bytes.NewReader(data)).Decode(®); err != nil {
|
||||
return nil, fmt.Errorf("parsing Registry.toml: %w", err)
|
||||
}
|
||||
|
||||
names := make(map[string]string, len(reg.Packages))
|
||||
for uuid, pkg := range reg.Packages {
|
||||
if pkg.Name != "" {
|
||||
names[uuid] = pkg.Name
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// parseRegistryLine parses a single line from /registries of the form
|
||||
// "/registry/{uuid}/{hash}" and returns the uuid and hash.
|
||||
func parseRegistryLine(line string) (uuid, hash string, ok bool) {
|
||||
line = strings.TrimSpace(line)
|
||||
line = strings.TrimPrefix(line, "/registry/")
|
||||
uuid, hash, found := strings.Cut(line, "/")
|
||||
if !found || !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
|
||||
return "", "", false
|
||||
}
|
||||
return uuid, hash, true
|
||||
}
|
||||
|
||||
// validJuliaUUID reports whether s looks like a lowercase RFC 4122 UUID.
|
||||
func validJuliaUUID(s string) bool {
|
||||
return juliaUUIDPattern.MatchString(s)
|
||||
}
|
||||
|
|
@ -1,167 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestJuliaParseRegistryLine(t *testing.T) {
|
||||
tests := []struct {
|
||||
line string
|
||||
wantUUID string
|
||||
wantHash string
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
"/registry/23338594-aafe-5451-b93e-139f81909106/342327538ed6c1ec54c69fa145e7b6bf5934201e",
|
||||
"23338594-aafe-5451-b93e-139f81909106",
|
||||
"342327538ed6c1ec54c69fa145e7b6bf5934201e",
|
||||
true,
|
||||
},
|
||||
{
|
||||
" /registry/23338594-aafe-5451-b93e-139f81909106/342327538ed6c1ec54c69fa145e7b6bf5934201e\n",
|
||||
"23338594-aafe-5451-b93e-139f81909106",
|
||||
"342327538ed6c1ec54c69fa145e7b6bf5934201e",
|
||||
true,
|
||||
},
|
||||
{"/registry/not-a-uuid/0000", "", "", false},
|
||||
{"junk", "", "", false},
|
||||
{"", "", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
uuid, hash, ok := parseRegistryLine(tt.line)
|
||||
if uuid != tt.wantUUID || hash != tt.wantHash || ok != tt.wantOK {
|
||||
t.Errorf("parseRegistryLine(%q) = (%q, %q, %v), want (%q, %q, %v)",
|
||||
tt.line, uuid, hash, ok, tt.wantUUID, tt.wantHash, tt.wantOK)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJuliaValidUUID(t *testing.T) {
|
||||
tests := []struct {
|
||||
s string
|
||||
want bool
|
||||
}{
|
||||
{"23338594-aafe-5451-b93e-139f81909106", true},
|
||||
{"295af30f-e4ad-537b-8983-00126c2a3abe", true},
|
||||
{"23338594-AAFE-5451-b93e-139f81909106", false},
|
||||
{"23338594aafe5451b93e139f81909106", false},
|
||||
{"23338594-aafe-5451-b93e-139f8190910", false},
|
||||
{"23338594-aafe-5451-b93e-139f81909106-", false},
|
||||
{"23338594-gafe-5451-b93e-139f81909106", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := validJuliaUUID(tt.s); got != tt.want {
|
||||
t.Errorf("validJuliaUUID(%q) = %v, want %v", tt.s, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJuliaParseRegistryToml(t *testing.T) {
|
||||
data := []byte(`name = "General"
|
||||
uuid = "23338594-aafe-5451-b93e-139f81909106"
|
||||
|
||||
[packages]
|
||||
295af30f-e4ad-537b-8983-00126c2a3abe = { name = "Revise", path = "R/Revise" }
|
||||
91a5bcdd-55d7-5caf-9e0b-520d859cae80 = { name = "Plots", path = "P/Plots" }
|
||||
`)
|
||||
|
||||
names, err := parseRegistryToml(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parseRegistryToml: %v", err)
|
||||
}
|
||||
if got := names["295af30f-e4ad-537b-8983-00126c2a3abe"]; got != "Revise" {
|
||||
t.Errorf("names[Revise uuid] = %q, want Revise", got)
|
||||
}
|
||||
if got := names["91a5bcdd-55d7-5caf-9e0b-520d859cae80"]; got != "Plots" {
|
||||
t.Errorf("names[Plots uuid] = %q, want Plots", got)
|
||||
}
|
||||
if len(names) != 2 {
|
||||
t.Errorf("len(names) = %d, want 2", len(names))
|
||||
}
|
||||
}
|
||||
|
||||
func TestJuliaExtractRegistryNames(t *testing.T) {
|
||||
registryToml := `name = "General"
|
||||
[packages]
|
||||
295af30f-e4ad-537b-8983-00126c2a3abe = { name = "Revise", path = "R/Revise" }
|
||||
`
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
for _, f := range []struct{ name, body string }{
|
||||
{"R/Revise/Package.toml", "name = \"Revise\"\n"},
|
||||
{"Registry.toml", registryToml},
|
||||
} {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o644, Size: int64(len(f.body))}); err != nil {
|
||||
t.Fatalf("WriteHeader: %v", err)
|
||||
}
|
||||
if _, err := tw.Write([]byte(f.body)); err != nil {
|
||||
t.Fatalf("Write: %v", err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("tar Close: %v", err)
|
||||
}
|
||||
if err := gw.Close(); err != nil {
|
||||
t.Fatalf("gzip Close: %v", err)
|
||||
}
|
||||
|
||||
names, err := extractRegistryNames(bytes.NewReader(buf.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("extractRegistryNames: %v", err)
|
||||
}
|
||||
if got := names["295af30f-e4ad-537b-8983-00126c2a3abe"]; got != "Revise" {
|
||||
t.Errorf("names[Revise uuid] = %q, want Revise", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJuliaResolveName(t *testing.T) {
|
||||
h := &JuliaHandler{
|
||||
proxy: &Proxy{Logger: slog.Default()},
|
||||
names: map[string]string{
|
||||
"295af30f-e4ad-537b-8983-00126c2a3abe": "Revise",
|
||||
},
|
||||
}
|
||||
|
||||
if got := h.resolveName("295af30f-e4ad-537b-8983-00126c2a3abe"); got != "Revise" {
|
||||
t.Errorf("resolveName(known) = %q, want Revise", got)
|
||||
}
|
||||
if got := h.resolveName("00000000-0000-0000-0000-000000000000"); got != "00000000-0000-0000-0000-000000000000" {
|
||||
t.Errorf("resolveName(unknown) = %q, want uuid fallback", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJuliaRoutesValidation(t *testing.T) {
|
||||
h := NewJuliaHandler(&Proxy{Logger: slog.Default()}, "")
|
||||
routes := h.Routes()
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
want int
|
||||
}{
|
||||
{"/package/not-a-uuid/342327538ed6c1ec54c69fa145e7b6bf5934201e", http.StatusBadRequest},
|
||||
{"/package/295af30f-e4ad-537b-8983-00126c2a3abe/short", http.StatusBadRequest},
|
||||
{"/registry/295af30f-e4ad-537b-8983-00126c2a3abe/zzzz", http.StatusBadRequest},
|
||||
{"/artifact/nothex", http.StatusBadRequest},
|
||||
{"/nope", http.StatusNotFound},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
req := httptest.NewRequest(http.MethodGet, tt.path, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
routes.ServeHTTP(rr, req)
|
||||
if rr.Code != tt.want {
|
||||
t.Errorf("GET %s = %d, want %d", tt.path, rr.Code, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
|
|
@ -9,8 +8,7 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
mavenCentralUpstream = "https://repo1.maven.org/maven2"
|
||||
gradlePluginPortalUpstream = "https://plugins.gradle.org/m2"
|
||||
mavenUpstream = "https://repo1.maven.org/maven2"
|
||||
minMavenParts = 4 // group path segments + artifact + version + filename
|
||||
)
|
||||
|
||||
|
|
@ -18,23 +16,14 @@ const (
|
|||
type MavenHandler struct {
|
||||
proxy *Proxy
|
||||
upstreamURL string
|
||||
pluginPortalUpstreamURL string
|
||||
proxyURL string
|
||||
}
|
||||
|
||||
// NewMavenHandler creates a new Maven repository handler.
|
||||
func NewMavenHandler(proxy *Proxy, proxyURL, upstreamURL, pluginPortalUpstreamURL string) *MavenHandler {
|
||||
if strings.TrimSpace(upstreamURL) == "" {
|
||||
upstreamURL = mavenCentralUpstream
|
||||
}
|
||||
if strings.TrimSpace(pluginPortalUpstreamURL) == "" {
|
||||
pluginPortalUpstreamURL = gradlePluginPortalUpstream
|
||||
}
|
||||
|
||||
func NewMavenHandler(proxy *Proxy, proxyURL string) *MavenHandler {
|
||||
return &MavenHandler{
|
||||
proxy: proxy,
|
||||
upstreamURL: strings.TrimSuffix(upstreamURL, "/"),
|
||||
pluginPortalUpstreamURL: strings.TrimSuffix(pluginPortalUpstreamURL, "/"),
|
||||
upstreamURL: mavenUpstream,
|
||||
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
||||
}
|
||||
}
|
||||
|
|
@ -62,7 +51,8 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) {
|
|||
filename := path.Base(urlPath)
|
||||
|
||||
if h.isMetadataFile(filename) {
|
||||
h.handleMetadata(w, r, urlPath)
|
||||
cacheKey := strings.ReplaceAll(urlPath, "/", "_")
|
||||
h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "maven", cacheKey, "*/*")
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -76,32 +66,6 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) {
|
|||
h.proxyUpstream(w, r)
|
||||
}
|
||||
|
||||
func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, urlPath string) {
|
||||
cacheKey := strings.ReplaceAll(urlPath, "/", "_")
|
||||
upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, urlPath)
|
||||
|
||||
body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*")
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUpstreamNotFound) {
|
||||
pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath)
|
||||
h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal",
|
||||
"path", urlPath)
|
||||
body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*")
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUpstreamNotFound) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
h.proxy.Logger.Error("metadata fetch failed", "error", err)
|
||||
http.Error(w, "failed to fetch from upstream", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType)
|
||||
}
|
||||
|
||||
// handleDownload serves an artifact file, fetching and caching from upstream if needed.
|
||||
func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, urlPath string) {
|
||||
// Parse Maven path: group/artifact/version/filename
|
||||
|
|
@ -122,15 +86,8 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, ur
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, upstreamURL)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUpstreamNotFound) {
|
||||
pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath)
|
||||
h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal",
|
||||
"group", group, "artifact", artifact, "version", version, "filename", filename)
|
||||
result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, pluginPortalURL)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch artifact")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch artifact", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -158,7 +115,7 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file
|
|||
// isArtifactFile returns true if the filename looks like a Maven artifact.
|
||||
func (h *MavenHandler) isArtifactFile(filename string) bool {
|
||||
// Common artifact extensions
|
||||
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"}
|
||||
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib"}
|
||||
for _, ext := range extensions {
|
||||
if strings.HasSuffix(filename, ext) {
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -52,7 +52,6 @@ func TestMavenIsArtifactFile(t *testing.T) {
|
|||
}{
|
||||
{"guava-32.1.3-jre.jar", true},
|
||||
{"guava-32.1.3-jre.pom", true},
|
||||
{"guava-32.1.3-jre.module", true},
|
||||
{"app-1.0.war", true},
|
||||
{"lib-1.0.aar", true},
|
||||
{"maven-metadata.xml", false},
|
||||
|
|
@ -66,63 +65,3 @@ func TestMavenIsArtifactFile(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMavenIsMetadataFile(t *testing.T) {
|
||||
h := &MavenHandler{}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "pom is artifact, not metadata",
|
||||
filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "pom checksum is metadata",
|
||||
filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "metadata file",
|
||||
filename: "maven-metadata.xml",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "metadata checksum",
|
||||
filename: "maven-metadata.xml.sha256",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "jar checksum is metadata",
|
||||
filename: "guava-32.1.3-jre.jar.sha1",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "asc signature is metadata",
|
||||
filename: "guava-32.1.3-jre.jar.asc",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "regular jar is not metadata",
|
||||
filename: "guava-32.1.3-jre.jar",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "pom checksum is metadata",
|
||||
filename: "guava-32.1.3-jre.pom.sha1",
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := h.isMetadataFile(tt.filename)
|
||||
if got != tt.want {
|
||||
t.Errorf("isMetadataFile(%q) = %v, want %v", tt.filename, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,139 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
handler func(p *Proxy) http.Handler
|
||||
}{
|
||||
{"debian", "/pool/main/n/nginx/nginx_1.18.0-6_amd64.deb",
|
||||
func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost", "").Routes() }},
|
||||
{"rpm", "/releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm",
|
||||
func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }},
|
||||
{"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg",
|
||||
func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }},
|
||||
{"pypi", "/packages/packages/ab/cd/ef0123456789/requests-2.31.0-py3-none-any.whl",
|
||||
func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://localhost").Routes() }},
|
||||
{"cran", "/src/contrib/ggplot2_3.4.4.tar.gz",
|
||||
func(p *Proxy) http.Handler { return NewCRANHandler(p, "http://localhost").Routes() }},
|
||||
{"conda", "/conda-forge/linux-64/numpy-1.26.0-py311_0.tar.bz2",
|
||||
func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }},
|
||||
{"conan", "/v1/files/zlib/1.3.1/_/_/0/recipe/conan_sources.tgz",
|
||||
func(p *Proxy) http.Handler { return NewConanHandler(p, "http://localhost").Routes() }},
|
||||
{"gem", "/gems/rails-7.1.0.gem",
|
||||
func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }},
|
||||
{"hex", "/tarballs/phoenix-1.7.10.tar",
|
||||
func(p *Proxy) http.Handler { return NewHexHandler(p, "http://localhost").Routes() }},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.fetchErr = fetch.ErrNotFound
|
||||
|
||||
srv := httptest.NewServer(tt.handler(proxy))
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + tt.path)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJuliaPackageUpstreamNotFoundReturns404(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.fetchErr = fetch.ErrNotFound
|
||||
|
||||
dead := httptest.NewServer(http.NotFoundHandler())
|
||||
defer dead.Close()
|
||||
|
||||
h := NewJuliaHandler(proxy, "http://localhost")
|
||||
h.upstreamURL = dead.URL
|
||||
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL +
|
||||
"/package/7876af07-990d-54b4-ab0e-23690620f79a/0123456789abcdef0123456789abcdef01234567")
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Errorf("want 404 for missing upstream package, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposerDownloadUpstreamNotFoundReturns404(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.fetchErr = fetch.ErrNotFound
|
||||
|
||||
meta := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/p2/monolog/monolog.json" {
|
||||
_, _ = w.Write([]byte(`{
|
||||
"packages": {
|
||||
"monolog/monolog": [
|
||||
{"version": "2.9.1", "dist": {"url": "https://example.com/monolog-2.9.1.zip", "type": "zip"}}
|
||||
]
|
||||
}
|
||||
}`))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer meta.Close()
|
||||
|
||||
h := &ComposerHandler{proxy: proxy, repoURL: meta.URL, proxyURL: "http://localhost"}
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + "/files/monolog/monolog/2.9.1/monolog-2.9.1.zip")
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Errorf("want 404 for missing upstream dist, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerBlobUpstreamNotFoundReturns404(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.fetchErr = fetch.ErrNotFound
|
||||
|
||||
h := &ContainerHandler{
|
||||
proxy: proxy,
|
||||
registryURL: "https://registry-1.docker.io",
|
||||
proxyURL: "http://localhost:8080",
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet,
|
||||
"/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("want 404 for missing upstream blob, got %d; body: %s", w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "BLOB_UNKNOWN") {
|
||||
t.Errorf("want BLOB_UNKNOWN error code in body, got: %s", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
func TestErrUpstreamNotFoundWrapsFetchErrNotFound(t *testing.T) {
|
||||
if !errors.Is(ErrUpstreamNotFound, fetch.ErrNotFound) {
|
||||
t.Fatal("ErrUpstreamNotFound does not wrap fetch.ErrNotFound")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOrFetchArtifactFromURL_NotFound(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.fetchErr = fetch.ErrNotFound
|
||||
|
||||
_, err := proxy.GetOrFetchArtifactFromURL(context.Background(),
|
||||
"maven", "org.example:missing", "1.0", "missing-1.0.jar",
|
||||
"http://upstream.test/org/example/missing/1.0/missing-1.0.jar")
|
||||
|
||||
if !errors.Is(err, ErrUpstreamNotFound) {
|
||||
t.Fatalf("want ErrUpstreamNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMavenHandler_UpstreamNotFoundReturns404(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.fetchErr = fetch.ErrNotFound
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test")
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + "/org/example/missing/1.0/missing-1.0.jar")
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMavenHandler_PluginPortalFallback(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
fetcher.fetchErrByURL = map[string]error{
|
||||
"http://upstream.test/org/example/plugin/1.0/plugin-1.0.jar": fetch.ErrNotFound,
|
||||
}
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("portal artifact")),
|
||||
ContentType: "application/java-archive",
|
||||
}
|
||||
|
||||
h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test")
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + "/org/example/plugin/1.0/plugin-1.0.jar")
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("want 200 via plugin portal fallback, got %d", resp.StatusCode)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if string(body) != "portal artifact" {
|
||||
t.Errorf("want portal artifact body, got %q", body)
|
||||
}
|
||||
if fetcher.fetchedURL != "http://portal.test/org/example/plugin/1.0/plugin-1.0.jar" {
|
||||
t.Errorf("fallback did not hit plugin portal, last URL: %s", fetcher.fetchedURL)
|
||||
}
|
||||
}
|
||||
|
|
@ -9,11 +9,13 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
)
|
||||
|
||||
const (
|
||||
npmUpstream = "https://registry.npmjs.org"
|
||||
npmAcceptDefault = "application/vnd.npm.install-v1+json;q=1.0, application/json;q=0.8"
|
||||
npmAbbreviatedCT = "application/vnd.npm.install-v1+json"
|
||||
scopedParts = 2 // scope + name in scoped packages
|
||||
)
|
||||
|
||||
|
|
@ -25,14 +27,10 @@ type NPMHandler struct {
|
|||
}
|
||||
|
||||
// NewNPMHandler creates a new npm protocol handler.
|
||||
func NewNPMHandler(proxy *Proxy, proxyURL, upstreamURL string) *NPMHandler {
|
||||
if strings.TrimSpace(upstreamURL) == "" {
|
||||
upstreamURL = npmUpstream
|
||||
}
|
||||
|
||||
func NewNPMHandler(proxy *Proxy, proxyURL string) *NPMHandler {
|
||||
return &NPMHandler{
|
||||
proxy: proxy,
|
||||
upstreamURL: strings.TrimSuffix(upstreamURL, "/"),
|
||||
upstreamURL: npmUpstream,
|
||||
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
||||
}
|
||||
}
|
||||
|
|
@ -71,12 +69,9 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques
|
|||
|
||||
upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName))
|
||||
|
||||
// Prefer the smaller abbreviated packument format but include application/json
|
||||
// as a fallback so upstreams that reject the abbreviated type (e.g. JFrog
|
||||
// Artifactory, which returns 406) can still respond with full metadata.
|
||||
// When cooldown is enabled we must use full metadata exclusively because the
|
||||
// abbreviated format omits the "time" map required for version age filtering.
|
||||
accept := npmAcceptDefault
|
||||
// Use abbreviated metadata when cooldown is disabled — it's much smaller
|
||||
// (e.g. drizzle-orm: 4MB vs 92MB) but lacks the time map needed for cooldown.
|
||||
accept := npmAbbreviatedCT
|
||||
if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() {
|
||||
accept = contentTypeJSON
|
||||
}
|
||||
|
|
@ -139,7 +134,7 @@ func (h *NPMHandler) applyCooldownFiltering(metadata map[string]any, versions ma
|
|||
return
|
||||
}
|
||||
|
||||
packagePURL := canonicalPackagePURL("npm", packageName)
|
||||
packagePURL := purl.MakePURLString("npm", packageName, "")
|
||||
|
||||
for version := range versions {
|
||||
publishedStr, ok := timeMap[version].(string)
|
||||
|
|
@ -268,27 +263,8 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|||
h.proxy.Logger.Info("npm download request",
|
||||
"package", packageName, "version", version, "filename", filename)
|
||||
|
||||
if h.versionInCooldown(r, packageName, version) {
|
||||
h.proxy.Logger.Info("cooldown: withholding npm tarball",
|
||||
"package", packageName, "version", version)
|
||||
JSONError(w, http.StatusNotFound, "version not found")
|
||||
return
|
||||
}
|
||||
|
||||
downloadURL := fmt.Sprintf(
|
||||
"%s/%s/-/%s",
|
||||
h.upstreamURL,
|
||||
escapeNPMDownloadPackage(packageName),
|
||||
url.PathEscape(filename),
|
||||
)
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(
|
||||
r.Context(), "npm", packageName, version, filename, downloadURL,
|
||||
)
|
||||
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "npm", packageName, version, filename)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrUpstreamNotFound) {
|
||||
JSONError(w, http.StatusNotFound, "package not found")
|
||||
return
|
||||
}
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
JSONError(w, http.StatusBadGateway, "failed to fetch package")
|
||||
return
|
||||
|
|
@ -297,58 +273,6 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// versionInCooldown reports whether a version is still inside the cooldown
|
||||
// window. Filtering the packument is not enough on its own: tarball URLs are
|
||||
// predictable and lockfiles record them directly, so `npm ci` reaches the
|
||||
// download path without ever requesting metadata.
|
||||
//
|
||||
// The packument is served from the metadata cache, so this normally costs no
|
||||
// extra upstream request. A version with no usable publish time is allowed
|
||||
// through, matching how applyCooldownFiltering treats it.
|
||||
func (h *NPMHandler) versionInCooldown(r *http.Request, packageName, version string) bool {
|
||||
if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() {
|
||||
return false
|
||||
}
|
||||
|
||||
upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName))
|
||||
|
||||
body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Warn("cooldown: could not fetch npm metadata for download check",
|
||||
"package", packageName, "version", version, "error", err)
|
||||
return false
|
||||
}
|
||||
|
||||
var metadata struct {
|
||||
Time map[string]string `json:"time"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &metadata); err != nil {
|
||||
h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check",
|
||||
"package", packageName, "version", version, "error", err)
|
||||
return false
|
||||
}
|
||||
|
||||
published, ok := metadata.Time[version]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
publishedAt, err := time.Parse(time.RFC3339, published)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt)
|
||||
}
|
||||
|
||||
func escapeNPMDownloadPackage(packageName string) string {
|
||||
scope, name, scoped := strings.Cut(packageName, "/")
|
||||
if scoped && strings.HasPrefix(scope, "@") && len(scope) > 1 && name != "" && !strings.Contains(name, "/") {
|
||||
return url.PathEscape(scope) + "/" + url.PathEscape(name)
|
||||
}
|
||||
return url.PathEscape(packageName)
|
||||
}
|
||||
|
||||
// extractPackageName extracts the package name from the request path.
|
||||
// Handles both scoped (@scope/name) and unscoped (name) packages.
|
||||
func (h *NPMHandler) extractPackageName(r *http.Request) string {
|
||||
|
|
|
|||
|
|
@ -2,16 +2,13 @@ package handler
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
)
|
||||
|
||||
const testVersion100 = "1.0.0"
|
||||
|
|
@ -49,86 +46,6 @@ func TestNPMExtractVersionFromFilename(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestNPMHandlerUsesConfiguredUpstream(t *testing.T) {
|
||||
t.Run("metadata", func(t *testing.T) {
|
||||
var requestPath, authHeader string
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestPath = r.URL.Path
|
||||
authHeader = r.Header.Get("Authorization")
|
||||
if authHeader != "Bearer npm-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"versions":{}}`)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, _ := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
proxy.AuthForURL = func(string) (string, string) {
|
||||
return "Authorization", "Bearer npm-token"
|
||||
}
|
||||
h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/root/")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/testpkg", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
}
|
||||
if requestPath != "/root/testpkg" {
|
||||
t.Errorf("upstream path = %q, want %q", requestPath, "/root/testpkg")
|
||||
}
|
||||
if authHeader != "Bearer npm-token" {
|
||||
t.Errorf("Authorization = %q, want %q", authHeader, "Bearer npm-token")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("download", func(t *testing.T) {
|
||||
proxy, _, _, artifactFetcher := setupTestProxy(t)
|
||||
artifactFetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("package")),
|
||||
ContentType: "application/gzip",
|
||||
}
|
||||
h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/testpkg/-/testpkg-1.0.0.tgz", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
}
|
||||
want := "https://npm.example.test/root/testpkg/-/testpkg-1.0.0.tgz"
|
||||
if artifactFetcher.fetchedURL != want {
|
||||
t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("scoped download", func(t *testing.T) {
|
||||
proxy, _, _, artifactFetcher := setupTestProxy(t)
|
||||
artifactFetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("package")),
|
||||
ContentType: "application/gzip",
|
||||
}
|
||||
h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/@scope/name/-/name-1.0.0.tgz", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.Routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
|
||||
}
|
||||
want := "https://npm.example.test/root/@scope/name/-/name-1.0.0.tgz"
|
||||
if artifactFetcher.fetchedURL != want {
|
||||
t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNPMRewriteMetadata(t *testing.T) {
|
||||
h := &NPMHandler{
|
||||
proxy: testProxy(),
|
||||
|
|
@ -396,7 +313,7 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) {
|
|||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
t.Run("no cooldown uses combined accept header", func(t *testing.T) {
|
||||
t.Run("no cooldown uses abbreviated metadata", func(t *testing.T) {
|
||||
h := &NPMHandler{
|
||||
proxy: testProxy(),
|
||||
upstreamURL: upstream.URL,
|
||||
|
|
@ -407,12 +324,12 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) {
|
|||
w := httptest.NewRecorder()
|
||||
h.handlePackageMetadata(w, req)
|
||||
|
||||
if gotAccept != npmAcceptDefault {
|
||||
t.Errorf("Accept = %q, want %q", gotAccept, npmAcceptDefault)
|
||||
if gotAccept != npmAbbreviatedCT {
|
||||
t.Errorf("Accept = %q, want abbreviated metadata header", gotAccept)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cooldown enabled uses full metadata only", func(t *testing.T) {
|
||||
t.Run("cooldown enabled uses full metadata", func(t *testing.T) {
|
||||
proxy := testProxy()
|
||||
proxy.Cooldown = &cooldown.Config{Default: "3d"}
|
||||
|
||||
|
|
@ -426,8 +343,8 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) {
|
|||
w := httptest.NewRecorder()
|
||||
h.handlePackageMetadata(w, req)
|
||||
|
||||
if gotAccept != contentTypeJSON {
|
||||
t.Errorf("Accept = %q, want %q (cooldown requires full metadata)", gotAccept, contentTypeJSON)
|
||||
if gotAccept == npmAbbreviatedCT {
|
||||
t.Error("cooldown enabled should use full metadata, not abbreviated")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -454,81 +371,3 @@ func TestNPMHandlerMetadataNotFound(t *testing.T) {
|
|||
t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNPMDownloadCooldown(t *testing.T) {
|
||||
now := time.Now()
|
||||
packument := `{
|
||||
"name": "leftpad",
|
||||
"dist-tags": {"latest": "2.0.0"},
|
||||
"time": {
|
||||
"1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `",
|
||||
"2.0.0": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"
|
||||
},
|
||||
"versions": {"1.0.0": {}, "2.0.0": {}}
|
||||
}`
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", contentTypeJSON)
|
||||
_, _ = io.WriteString(w, packument)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
version string
|
||||
wantStatus int
|
||||
}{
|
||||
{"published before the window serves the tarball", testVersion100, http.StatusOK},
|
||||
{"published inside the window is withheld", "2.0.0", http.StatusNotFound},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
proxy.Cooldown = &cooldown.Config{Default: "7d"}
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("tarball data")),
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL)
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz")
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != tt.wantStatus {
|
||||
t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus)
|
||||
}
|
||||
if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled {
|
||||
t.Error("fetched a version that is still inside the cooldown window")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNPMDownloadCooldownDisabled(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
t.Error("metadata must not be fetched when cooldown is disabled")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("tarball data")),
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL)
|
||||
|
||||
if h.versionInCooldown(httptest.NewRequest(http.MethodGet, "/", nil), "leftpad", testVersion100) {
|
||||
t.Error("versionInCooldown = true, want false when cooldown is not configured")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import (
|
|||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -170,7 +172,7 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request
|
|||
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
req.Header.Set(headerAcceptEncoding, "gzip")
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
|
|
@ -191,7 +193,7 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
body, err := h.proxy.ReadMetadata(resp.Body)
|
||||
body, err := ReadMetadata(resp.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to read response", http.StatusInternalServerError)
|
||||
return
|
||||
|
|
@ -269,7 +271,7 @@ func (h *NuGetHandler) applyCooldownFiltering(body []byte) ([]byte, error) {
|
|||
}
|
||||
}
|
||||
|
||||
packagePURL := canonicalPackagePURL("nuget", strings.ToLower(id))
|
||||
packagePURL := purl.MakePURLString("nuget", strings.ToLower(id), "")
|
||||
|
||||
if !h.proxy.Cooldown.IsAllowed("nuget", packagePURL, publishedAt) {
|
||||
h.proxy.Logger.Info("cooldown: filtering nuget version",
|
||||
|
|
@ -314,7 +316,8 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "nuget", name, version, filename, upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -335,8 +338,8 @@ func (h *NuGetHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
// Copy accept-encoding for compression
|
||||
if ae := r.Header.Get(headerAcceptEncoding); ae != "" {
|
||||
req.Header.Set(headerAcceptEncoding, ae)
|
||||
if ae := r.Header.Get("Accept-Encoding"); ae != "" {
|
||||
req.Header.Set("Accept-Encoding", ae)
|
||||
}
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
)
|
||||
|
||||
func nugetTestProxy() *Proxy {
|
||||
|
|
|
|||
|
|
@ -14,15 +14,6 @@ func TestContainsPathTraversal(t *testing.T) {
|
|||
{"pool/main/../../../etc/shadow", true},
|
||||
{"pool/..hidden/file", false}, // ".." as a segment, not "..hidden"
|
||||
{"", false},
|
||||
{"%2e%2e/etc/passwd", true},
|
||||
{"%2e%2e%2fetc%2fpasswd", true},
|
||||
{"pool/%2e%2e/%2e%2e/etc/shadow", true},
|
||||
{"%2E%2E%2Fetc", true},
|
||||
{`..\\etc\\passwd`, true},
|
||||
{`pool\\..\\..\\etc`, true},
|
||||
{"%2e%2e%5cetc%5cpasswd", true},
|
||||
{"pool/%2e%2ehidden/file", false},
|
||||
{"pool/%zz/bad-encoding", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import (
|
|||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -67,7 +69,8 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "pub", name, version, filename)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -124,7 +127,7 @@ func (h *PubHandler) rewriteMetadata(name string, body []byte) ([]byte, error) {
|
|||
return body, nil
|
||||
}
|
||||
|
||||
packagePURL := canonicalPackagePURL("pub", name)
|
||||
packagePURL := purl.MakePURLString("pub", name, "")
|
||||
filtered := h.filterAndRewriteVersions(name, packagePURL, versions)
|
||||
metadata["versions"] = filtered
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
)
|
||||
|
||||
func TestPubRewriteMetadata(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ import (
|
|||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -19,14 +21,7 @@ const (
|
|||
minWheelParts = 5 // name + version + python + abi + platform
|
||||
minSubmatchParts = 2 // full match + first capture group
|
||||
minPyPIPathParts = 3 // hash_prefix + hash + filename
|
||||
minEggParts = 3 // name + version + python tag
|
||||
|
||||
// PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip
|
||||
// appends to a distribution URL when the index advertises core metadata.
|
||||
// A sidecar resolves to the same name and version as the distribution it
|
||||
// describes, so it is cached alongside it; consumers that expect an openable
|
||||
// archive must skip these.
|
||||
PyPIMetadataSuffix = ".metadata"
|
||||
minPythonTagLen = 2 // minimum length for a python tag (e.g., "py")
|
||||
)
|
||||
|
||||
// PyPIHandler handles PyPI registry protocol requests.
|
||||
|
|
@ -110,14 +105,24 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request
|
|||
// that should be filtered out due to cooldown.
|
||||
func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[string]bool {
|
||||
jsonURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name)
|
||||
|
||||
body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", name+"/json", jsonURL)
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, jsonURL, nil)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
|
||||
var metadata map[string]any
|
||||
if err := json.Unmarshal(body, &metadata); err != nil {
|
||||
if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -126,7 +131,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[st
|
|||
return nil
|
||||
}
|
||||
|
||||
packagePURL := canonicalPackagePURL("pypi", name)
|
||||
packagePURL := purl.MakePURLString("pypi", name, "")
|
||||
filtered := make(map[string]bool)
|
||||
|
||||
for version, files := range releases {
|
||||
|
|
@ -257,7 +262,7 @@ func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, error) {
|
|||
packageName, _ := extractPyPIName(metadata)
|
||||
packagePURL := ""
|
||||
if packageName != "" {
|
||||
packagePURL = canonicalPackagePURL("pypi", packageName)
|
||||
packagePURL = purl.MakePURLString("pypi", packageName, "")
|
||||
}
|
||||
|
||||
h.filterAndRewriteReleases(metadata, packageName, packagePURL)
|
||||
|
|
@ -301,21 +306,6 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool {
|
|||
return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt)
|
||||
}
|
||||
|
||||
// versionInCooldown reports whether a version is still inside the cooldown
|
||||
// window. Filtering the simple index is not enough on its own: file URLs are
|
||||
// recorded in lockfiles and requirements pins, so pip can reach the download
|
||||
// path without ever reading the index.
|
||||
//
|
||||
// A release whose upload time cannot be determined is allowed through, matching
|
||||
// how fetchFilteredVersions treats it.
|
||||
func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) bool {
|
||||
if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() {
|
||||
return false
|
||||
}
|
||||
|
||||
return h.fetchFilteredVersions(r, name)[version]
|
||||
}
|
||||
|
||||
// rewriteFileEntries rewrites URLs in a list of file entries.
|
||||
func (h *PyPIHandler) rewriteFileEntries(files any) {
|
||||
filesArr, ok := files.([]any)
|
||||
|
|
@ -422,13 +412,6 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|||
filename := parts[len(parts)-1]
|
||||
name, version := h.parseFilename(filename)
|
||||
|
||||
if name != "" && h.versionInCooldown(r, name, version) {
|
||||
h.proxy.Logger.Info("cooldown: withholding pypi file",
|
||||
"name", name, "version", version, "filename", filename)
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
if name == "" {
|
||||
// Can't determine name/version, use hash as identifier
|
||||
name = fmt.Sprintf("_hash_%s", hashPath(path))
|
||||
|
|
@ -445,171 +428,65 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "pypi", name, version, filename, upstreamURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
ServeArtifact(w, result)
|
||||
}
|
||||
|
||||
// archiveExtensions are sdist formats of the form {name}-{version}{ext}. They
|
||||
// carry no trailing tags, but legacy sdist names may contain hyphens.
|
||||
var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".tgz", ".tar", ".zip"}
|
||||
|
||||
// windowsInstallerExtensions are the legacy distutils bdist_wininst and
|
||||
// bdist_msi formats, which share a filename layout.
|
||||
var windowsInstallerExtensions = []string{".exe", ".msi"}
|
||||
|
||||
// parseFilename extracts package name and version from a PyPI filename.
|
||||
// Handles wheels, sdists and legacy bdist formats:
|
||||
// Handles both wheels and sdists:
|
||||
// - requests-2.31.0-py3-none-any.whl
|
||||
// - requests-2.31.0.tar.gz
|
||||
// - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg
|
||||
// - numpy-1.8.0.win32-py2.7.exe
|
||||
func (h *PyPIHandler) parseFilename(filename string) (name, version string) {
|
||||
// PEP 658/714 core-metadata sidecars are the distribution filename plus
|
||||
// ".metadata"; they describe the same name and version. Without this, pip's
|
||||
// metadata-only fetches fall back to a hash-derived package identifier.
|
||||
filename = strings.TrimSuffix(filename, PyPIMetadataSuffix)
|
||||
|
||||
switch {
|
||||
case strings.HasSuffix(filename, ".whl"):
|
||||
return parseWheelFilename(strings.TrimSuffix(filename, ".whl"))
|
||||
case strings.HasSuffix(filename, ".egg"):
|
||||
return parseEggFilename(strings.TrimSuffix(filename, ".egg"))
|
||||
}
|
||||
|
||||
for _, ext := range windowsInstallerExtensions {
|
||||
if strings.HasSuffix(filename, ext) {
|
||||
return parseWindowsInstallerFilename(strings.TrimSuffix(filename, ext))
|
||||
}
|
||||
}
|
||||
|
||||
for _, ext := range archiveExtensions {
|
||||
if strings.HasSuffix(filename, ext) {
|
||||
return splitNameVersion(strings.TrimSuffix(filename, ext))
|
||||
}
|
||||
}
|
||||
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// parseWheelFilename parses the PEP 427 layout
|
||||
// {name}-{version}(-{build})?-{python}-{abi}-{platform}, base being the
|
||||
// filename without its ".whl" suffix. The spec escapes every hyphen in the name
|
||||
// and version to '_', so the first two fields are authoritative even when the
|
||||
// optional build tag is present.
|
||||
func parseWheelFilename(base string) (name, version string) {
|
||||
// Try wheel format first: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl
|
||||
if strings.HasSuffix(filename, ".whl") {
|
||||
base := strings.TrimSuffix(filename, ".whl")
|
||||
parts := strings.Split(base, "-")
|
||||
if len(parts) < minWheelParts {
|
||||
return "", ""
|
||||
if len(parts) >= minWheelParts {
|
||||
// Find where version ends (version followed by python tag)
|
||||
for i := 1; i < len(parts)-2; i++ {
|
||||
// Check if this looks like a python tag (py2, py3, cp39, etc)
|
||||
if isPythonTag(parts[i]) {
|
||||
name = strings.Join(parts[:i-1], "-")
|
||||
version = parts[i-1]
|
||||
return
|
||||
}
|
||||
|
||||
return parts[0], parts[1]
|
||||
}
|
||||
|
||||
// parseEggFilename parses the setuptools bdist_egg layout
|
||||
// {name}-{version}-py{X.Y}(-{platform})?, base being the filename without its
|
||||
// ".egg" suffix. setuptools escapes hyphens in the name and version to '_', but
|
||||
// eggs built by other tooling do not always, so the version is located relative
|
||||
// to the interpreter field rather than assumed to be the second field.
|
||||
func parseEggFilename(base string) (name, version string) {
|
||||
parts := strings.Split(base, "-")
|
||||
// Scan from the end: the trailing platform fields never look like an
|
||||
// interpreter tag, so the last match is the real one even when the package
|
||||
// name itself carries a "py{N}" component. Stop before index 1, since a tag
|
||||
// any earlier would leave no room for both a name and a version.
|
||||
for i := len(parts) - 1; i >= minEggParts-1; i-- {
|
||||
if !isEggPythonTag(parts[i]) || !isVersionField(parts[i-1]) {
|
||||
continue
|
||||
}
|
||||
|
||||
return strings.Join(parts[:i-1], "-"), parts[i-1]
|
||||
}
|
||||
|
||||
// No interpreter field: {name}-{version}.
|
||||
return splitNameVersion(base)
|
||||
}
|
||||
|
||||
// parseWindowsInstallerFilename parses the distutils bdist_wininst and
|
||||
// bdist_msi layout {name}-{version}.{platform}(-py{X.Y})?, base being the
|
||||
// filename without its ".exe" or ".msi" suffix. The platform is joined to the
|
||||
// version with a '.' rather than a '-' and may itself contain a hyphen
|
||||
// ("win-amd64"), so both trailing fields are stripped before the name and
|
||||
// version are split apart.
|
||||
func parseWindowsInstallerFilename(base string) (name, version string) {
|
||||
if i := strings.LastIndex(base, "-py"); i >= 0 && isDottedNumber(base[i+len("-py"):]) {
|
||||
base = base[:i]
|
||||
}
|
||||
|
||||
// The platform is the final '.'-separated field. Requiring it to start with
|
||||
// a non-digit keeps a dotted version from being truncated when a filename
|
||||
// carries no platform tag.
|
||||
i := strings.LastIndex(base, ".")
|
||||
if i < 0 || i+1 >= len(base) || isVersionStart(base[i+1]) {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
return splitFullname(base[:i])
|
||||
}
|
||||
|
||||
// splitFullname splits the distutils fullname {name}-{version} that precedes a
|
||||
// Windows installer's platform field. Unlike an sdist, a wininst fullname may
|
||||
// carry a trailing build variant ("cx_Oracle-5.1.2-11g"), which belongs to
|
||||
// neither the name nor the version, so the first purely numeric field wins and
|
||||
// anything after it is discarded.
|
||||
func splitFullname(fullname string) (name, version string) {
|
||||
parts := strings.Split(fullname, "-")
|
||||
for i := 1; i < len(parts); i++ {
|
||||
if isDottedNumber(parts[i]) {
|
||||
return strings.Join(parts[:i], "-"), parts[i]
|
||||
}
|
||||
}
|
||||
|
||||
// No purely numeric field, e.g. a prerelease version like "1.0b1".
|
||||
return splitNameVersion(fullname)
|
||||
}
|
||||
|
||||
// splitNameVersion splits a {name}-{version} pair at the last hyphen that
|
||||
// starts a version, leaving hyphens inside the name intact.
|
||||
func splitNameVersion(base string) (name, version string) {
|
||||
// Try sdist formats: {name}-{version}.tar.gz, {name}-{version}.zip
|
||||
for _, ext := range []string{".tar.gz", ".tar.bz2", ".zip", ".tar"} {
|
||||
if strings.HasSuffix(filename, ext) {
|
||||
base := strings.TrimSuffix(filename, ext)
|
||||
// Find last hyphen followed by version
|
||||
for i := len(base) - 1; i >= 0; i-- {
|
||||
if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) {
|
||||
return base[:i], base[i+1:]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// isEggPythonTag reports whether field is the py{X.Y} interpreter field that
|
||||
// setuptools places directly after the version in an egg filename.
|
||||
func isEggPythonTag(field string) bool {
|
||||
const prefix = "py"
|
||||
|
||||
return len(field) > len(prefix) && strings.HasPrefix(field, prefix) && isVersionStart(field[len(prefix)])
|
||||
}
|
||||
|
||||
// isVersionField reports whether field can be a version, i.e. it is non-empty
|
||||
// and starts with a digit as every PEP 440 release segment does.
|
||||
func isVersionField(field string) bool {
|
||||
return field != "" && isVersionStart(field[0])
|
||||
}
|
||||
|
||||
// isDottedNumber reports whether s is a dotted numeric version such as "2.7".
|
||||
func isDottedNumber(s string) bool {
|
||||
if s == "" || !isVersionStart(s[0]) {
|
||||
func isPythonTag(s string) bool {
|
||||
if len(s) < minPythonTagLen {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range len(s) {
|
||||
if !isVersionStart(s[i]) && s[i] != '.' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Python tags start with py, cp, pp, ip, jy
|
||||
prefixes := []string{"py", "cp", "pp", "ip", "jy"}
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(s, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isVersionStart(c byte) bool {
|
||||
return c >= '0' && c <= '9'
|
||||
|
|
|
|||
|
|
@ -7,11 +7,10 @@ import (
|
|||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
"github.com/git-pkgs/registries/fetch"
|
||||
)
|
||||
|
||||
|
|
@ -29,58 +28,13 @@ func TestPyPIParseFilename(t *testing.T) {
|
|||
{"aws-sdk-1.0.0.tar.gz", "aws-sdk", "1.0.0"},
|
||||
{"zipp-3.17.0.zip", "zipp", "3.17.0"},
|
||||
|
||||
// Additional sdist archive formats
|
||||
{"lxml-4.9.3.tar.xz", "lxml", "4.9.3"},
|
||||
{"docutils-0.20.1.tgz", "docutils", "0.20.1"},
|
||||
{"psycopg2-2.9.9.tar.bz2", "psycopg2", "2.9.9"},
|
||||
|
||||
// Wheel formats
|
||||
{"requests-2.31.0-py3-none-any.whl", "requests", "2.31.0"},
|
||||
{"numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.whl", "numpy", "1.26.2"},
|
||||
{"cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", "cryptography", "41.0.5"},
|
||||
|
||||
// Wheels with a build tag must not fold the tag into the version
|
||||
{"foo-1.0-1-py3-none-any.whl", "foo", "1.0"},
|
||||
{"tensorflow-2.15.0-2-cp311-cp311-manylinux_2_17_x86_64.whl", "tensorflow", "2.15.0"},
|
||||
|
||||
// PEP 658 core-metadata sidecars resolve to the distribution they describe
|
||||
{"backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", "backports_asyncio_runner", "1.2.0"},
|
||||
{"requests-2.31.0-py3-none-any.whl.metadata", "requests", "2.31.0"},
|
||||
{"requests-2.31.0.tar.gz.metadata", "requests", "2.31.0"},
|
||||
|
||||
// Eggs: {name}-{version}-py{X.Y}(-{platform})?.egg. Unescaped hyphens in
|
||||
// the name must not be mistaken for the field separator before the version.
|
||||
{"numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg", "numpy", "1.8.0"},
|
||||
{"aws-sdk-1.0.0-py3.11.egg", "aws-sdk", "1.0.0"},
|
||||
{"aws-sdk-1.0.0-py2.7-macosx-10.9-x86_64.egg", "aws-sdk", "1.0.0"},
|
||||
{"aws-sdk-1.0.0.egg", "aws-sdk", "1.0.0"},
|
||||
// A "py{N}" component inside the name is not the interpreter field, so
|
||||
// the interpreter must be located from the end of the filename.
|
||||
{"django-rest-py3-1.0-py3.6.egg", "django-rest-py3", "1.0"},
|
||||
|
||||
// Windows installers: {name}-{version}.{platform}(-py{X.Y})?.{exe,msi}.
|
||||
// The platform is not part of the version, and may contain a hyphen.
|
||||
{"foo-1.0.win32-py2.0.exe", "foo", "1.0"},
|
||||
{"pywin32-223.win32-py2.7.exe", "pywin32", "223"},
|
||||
{"numpy-1.8.0.win-amd64-py2.7.exe", "numpy", "1.8.0"},
|
||||
{"aws-sdk-1.0.0.win32-py2.7.exe", "aws-sdk", "1.0.0"},
|
||||
{"pywin32-223.win32.exe", "pywin32", "223"},
|
||||
{"cx_Oracle-5.1.2.win32-py2.7.msi", "cx_Oracle", "5.1.2"},
|
||||
{"numpy-1.8.0.win-amd64.msi", "numpy", "1.8.0"},
|
||||
// A trailing build variant belongs to neither the name nor the version.
|
||||
{"cx_Oracle-5.1.2-11g.win32-py2.7.exe", "cx_Oracle", "5.1.2"},
|
||||
// A prerelease version has no purely numeric field to anchor on.
|
||||
{"foo-1.0b1.win32-py2.7.exe", "foo", "1.0b1"},
|
||||
|
||||
// Invalid
|
||||
{"invalid", "", ""},
|
||||
{"invalid.metadata", "", ""},
|
||||
{"backports.ssl_match_hostname-3.4.0.2-py2.7.whl", "", ""},
|
||||
{"invalid.exe", "", ""},
|
||||
{"foo-1.0.exe", "", ""},
|
||||
// An egg with an interpreter field but no version must not promote the
|
||||
// trailing component of a hyphenated name to the version.
|
||||
{"aws-sdk-py2.7.egg", "", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
@ -140,24 +94,25 @@ func TestPyPIRewriteJSONMetadataCooldown(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestPyPIParseFilenameNoHashFallback guards the identifier used for caching:
|
||||
// a filename that parses to an empty name makes handleDownload fall back to a
|
||||
// "_hash_<digest>" package name, which surfaces as a bogus PURL in the package
|
||||
// overview.
|
||||
func TestPyPIParseFilenameNoHashFallback(t *testing.T) {
|
||||
h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}}
|
||||
|
||||
filenames := []string{
|
||||
"backports_asyncio_runner-1.2.0-py3-none-any.whl",
|
||||
"backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata",
|
||||
"backports_asyncio_runner-1.2.0.tar.gz",
|
||||
func TestIsPythonTag(t *testing.T) {
|
||||
tests := []struct {
|
||||
tag string
|
||||
want bool
|
||||
}{
|
||||
{"py3", true},
|
||||
{"py2", true},
|
||||
{"cp311", true},
|
||||
{"cp37", true},
|
||||
{"pp39", true},
|
||||
{"none", false},
|
||||
{"any", false},
|
||||
{"manylinux", false},
|
||||
}
|
||||
|
||||
for _, filename := range filenames {
|
||||
name, version := h.parseFilename(filename)
|
||||
if name != "backports_asyncio_runner" || version != "1.2.0" {
|
||||
t.Errorf("parseFilename(%q) = (%q, %q), want (%q, %q)",
|
||||
filename, name, version, "backports_asyncio_runner", "1.2.0")
|
||||
for _, tt := range tests {
|
||||
got := isPythonTag(tt.tag)
|
||||
if got != tt.want {
|
||||
t.Errorf("isPythonTag(%q) = %v, want %v", tt.tag, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -237,116 +192,3 @@ func TestPyPIHandler_DownloadCacheMiss(t *testing.T) {
|
|||
t.Error("expected fetcher to be called on cache miss")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPIDownloadCooldown(t *testing.T) {
|
||||
now := time.Now()
|
||||
releases := `{"releases": {
|
||||
"1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}],
|
||||
"2.0.0": [{"upload_time_iso_8601": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"}]
|
||||
}}`
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", contentTypeJSON)
|
||||
_, _ = io.WriteString(w, releases)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
wantStatus int
|
||||
}{
|
||||
{"published before the window serves the file", "newpkg-1.0.0.tar.gz", http.StatusOK},
|
||||
{"published inside the window is withheld", "newpkg-2.0.0.tar.gz", http.StatusNotFound},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
proxy.Cooldown = &cooldown.Config{Default: "7d"}
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("sdist data")),
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
h := &PyPIHandler{
|
||||
proxy: proxy,
|
||||
upstreamURL: upstream.URL,
|
||||
proxyURL: "http://localhost",
|
||||
}
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
resp, err := http.Get(srv.URL + "/packages/packages/ab/cd/ef0123456789/" + tt.filename)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != tt.wantStatus {
|
||||
t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus)
|
||||
}
|
||||
if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled {
|
||||
t.Error("fetched a version that is still inside the cooldown window")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyPIDownloadCooldownMetadataCache ensures that repeated downloads that
|
||||
// trigger cooldown filtering reuse the cached PyPI JSON metadata instead of
|
||||
// fetching it from upstream once per download.
|
||||
func TestPyPIDownloadCooldownMetadataCache(t *testing.T) {
|
||||
now := time.Now()
|
||||
releases := `{"releases": {
|
||||
"1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}],
|
||||
"2.0.0": [{"upload_time_iso_8601": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"}]
|
||||
}}`
|
||||
|
||||
var metadataRequests atomic.Int64
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/pypi/newpkg/json" {
|
||||
metadataRequests.Add(1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, releases)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = io.WriteString(w, "package data")
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy, _, _, fetcher := setupTestProxy(t)
|
||||
proxy.HTTPClient = upstream.Client()
|
||||
proxy.CacheMetadata = true
|
||||
proxy.MetadataTTL = time.Hour
|
||||
proxy.Cooldown = &cooldown.Config{Default: "7d"}
|
||||
fetcher.artifact = &fetch.Artifact{
|
||||
Body: io.NopCloser(strings.NewReader("package data")),
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
h := &PyPIHandler{
|
||||
proxy: proxy,
|
||||
upstreamURL: upstream.URL,
|
||||
proxyURL: "http://localhost",
|
||||
}
|
||||
srv := httptest.NewServer(h.Routes())
|
||||
defer srv.Close()
|
||||
|
||||
// Two downloads of the same package: one outside the cooldown window
|
||||
// (served) and one inside (withheld). Both go through the download path
|
||||
// that resolves filtered versions.
|
||||
for _, filename := range []string{"newpkg-1.0.0.tar.gz", "newpkg-2.0.0.tar.gz"} {
|
||||
resp, err := http.Get(srv.URL + "/packages/packages/ab/cd/ef0123456789/" + filename)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
|
||||
if got := metadataRequests.Load(); got != 1 {
|
||||
t.Errorf("upstream metadata JSON requests = %d, want 1 (repeated downloads should reuse the cached metadata)", got)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,12 +7,9 @@ import (
|
|||
)
|
||||
|
||||
func TestReadMetadata(t *testing.T) {
|
||||
const limit = 1024
|
||||
p := &Proxy{MetadataMaxSize: limit}
|
||||
|
||||
t.Run("small body", func(t *testing.T) {
|
||||
data := []byte("hello world")
|
||||
got, err := p.ReadMetadata(bytes.NewReader(data))
|
||||
got, err := ReadMetadata(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
|
@ -22,39 +19,27 @@ func TestReadMetadata(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("exactly at limit", func(t *testing.T) {
|
||||
data := make([]byte, limit)
|
||||
data := make([]byte, maxMetadataSize)
|
||||
for i := range data {
|
||||
data[i] = 'x'
|
||||
}
|
||||
got, err := p.ReadMetadata(bytes.NewReader(data))
|
||||
got, err := ReadMetadata(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(got) != limit {
|
||||
t.Errorf("got length %d, want %d", len(got), limit)
|
||||
if len(got) != int(maxMetadataSize) {
|
||||
t.Errorf("got length %d, want %d", len(got), maxMetadataSize)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("over limit returns error", func(t *testing.T) {
|
||||
data := make([]byte, limit+100)
|
||||
data := make([]byte, maxMetadataSize+100)
|
||||
for i := range data {
|
||||
data[i] = 'x'
|
||||
}
|
||||
_, err := p.ReadMetadata(bytes.NewReader(data))
|
||||
_, err := ReadMetadata(bytes.NewReader(data))
|
||||
if !errors.Is(err, ErrMetadataTooLarge) {
|
||||
t.Errorf("got error %v, want ErrMetadataTooLarge", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("zero limit uses default", func(t *testing.T) {
|
||||
p := &Proxy{}
|
||||
data := make([]byte, 1<<20)
|
||||
got, err := p.ReadMetadata(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(got) != len(data) {
|
||||
t.Errorf("got length %d, want %d", len(got), len(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,7 +83,8 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques
|
|||
result, err := h.proxy.GetOrFetchArtifactFromURL(
|
||||
r.Context(), "rpm", name, version, filename, downloadURL)
|
||||
if err != nil {
|
||||
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
||||
h.proxy.Logger.Error("failed to get rpm package", "error", err)
|
||||
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
package httpclient
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/accesslog"
|
||||
)
|
||||
|
||||
type accessLogTransport struct {
|
||||
base http.RoundTripper
|
||||
accessLog *accesslog.Logger
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewAccessLogTransport records each upstream HTTP exchange around base.
|
||||
func NewAccessLogTransport(base http.RoundTripper, log *accesslog.Logger, logger *slog.Logger) http.RoundTripper {
|
||||
if base == nil {
|
||||
base = http.DefaultTransport
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
if log == nil {
|
||||
return base
|
||||
}
|
||||
return &accessLogTransport{
|
||||
base: base,
|
||||
accessLog: log,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *accessLogTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
start := time.Now()
|
||||
resp, err := t.base.RoundTrip(req)
|
||||
|
||||
entry := accesslog.Entry{
|
||||
Event: accesslog.EventUpstream,
|
||||
RequestID: accesslog.RequestID(req.Context()),
|
||||
Method: req.Method,
|
||||
URL: accesslog.URLWithoutSecrets(req.URL),
|
||||
DurationMS: time.Since(start).Milliseconds(),
|
||||
}
|
||||
if resp != nil {
|
||||
entry.StatusCode = resp.StatusCode
|
||||
}
|
||||
if err != nil {
|
||||
entry.Error = errorWithoutSecrets(err, req.URL)
|
||||
}
|
||||
if writeErr := t.accessLog.Write(entry); writeErr != nil {
|
||||
t.logger.Error("failed to write access log", "error", writeErr)
|
||||
}
|
||||
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func errorWithoutSecrets(err error, requestURL *url.URL) string {
|
||||
message := err.Error()
|
||||
if requestURL == nil {
|
||||
return message
|
||||
}
|
||||
|
||||
cleanURL := accesslog.URLWithoutSecrets(requestURL)
|
||||
for _, value := range []string{requestURL.String(), requestURL.Redacted()} {
|
||||
if value != "" {
|
||||
message = strings.ReplaceAll(message, value, cleanURL)
|
||||
}
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
|
@ -1,121 +0,0 @@
|
|||
package httpclient
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/accesslog"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return f(req)
|
||||
}
|
||||
|
||||
func TestAccessLogTransportRecordsUpstreamStatus(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "access.jsonl")
|
||||
accessLogger, err := accesslog.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
base := roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Body: io.NopCloser(strings.NewReader("rate limited")),
|
||||
Request: req,
|
||||
}, nil
|
||||
})
|
||||
client := &http.Client{Transport: NewAccessLogTransport(base, accessLogger, slog.Default())}
|
||||
req, err := http.NewRequest(http.MethodGet, "https://user:password@registry.example/package.tgz?token=secret", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req = req.WithContext(accesslog.WithRequestID(req.Context(), "request-123"))
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if err := accessLogger.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
entry := readAccessLogEntry(t, path)
|
||||
if entry.Event != accesslog.EventUpstream {
|
||||
t.Errorf("event = %q, want %q", entry.Event, accesslog.EventUpstream)
|
||||
}
|
||||
if entry.RequestID != "request-123" {
|
||||
t.Errorf("request_id = %q, want %q", entry.RequestID, "request-123")
|
||||
}
|
||||
if entry.StatusCode != http.StatusTooManyRequests {
|
||||
t.Errorf("status_code = %d, want %d", entry.StatusCode, http.StatusTooManyRequests)
|
||||
}
|
||||
if entry.URL != "https://registry.example/package.tgz" {
|
||||
t.Errorf("url = %q, want URL without credentials or query", entry.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccessLogTransportRecordsUpstreamError(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "access.jsonl")
|
||||
accessLogger, err := accesslog.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wantErr := errors.New("GET https://user:password@registry.example/package.tgz?token=secret: connection refused")
|
||||
base := roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, wantErr
|
||||
})
|
||||
client := &http.Client{Transport: NewAccessLogTransport(base, accessLogger, slog.Default())}
|
||||
|
||||
_, err = client.Get("https://user:password@registry.example/package.tgz?token=secret")
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("GET error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if err := accessLogger.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
entry := readAccessLogEntry(t, path)
|
||||
if entry.StatusCode != 0 {
|
||||
t.Errorf("status_code = %d, want 0", entry.StatusCode)
|
||||
}
|
||||
if strings.Contains(entry.Error, "password") || strings.Contains(entry.Error, "secret") {
|
||||
t.Errorf("error contains URL credentials or query: %q", entry.Error)
|
||||
}
|
||||
if !strings.Contains(entry.Error, "connection refused") {
|
||||
t.Errorf("error = %q, want connection failure", entry.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func readAccessLogEntry(t *testing.T, path string) accesslog.Entry {
|
||||
t.Helper()
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
if !scanner.Scan() {
|
||||
t.Fatalf("access log is empty: %v", scanner.Err())
|
||||
}
|
||||
|
||||
var entry accesslog.Entry
|
||||
if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
|
||||
t.Fatalf("decoding access log: %v", err)
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
|
@ -1,433 +0,0 @@
|
|||
// Package httpclient provides authentication-aware HTTP transports for upstream requests.
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTokenLifetime = 60 * time.Second
|
||||
tokenExpirySkew = 5 * time.Second
|
||||
maxTokenResponseSize = 1 << 20
|
||||
shortTokenSkewDivisor = 10
|
||||
)
|
||||
|
||||
// AuthFunc returns a configured authentication header for a URL.
|
||||
type AuthFunc func(url string) (headerName, headerValue string)
|
||||
|
||||
// Transport adds configured authentication and follows OCI Bearer challenges.
|
||||
type Transport struct {
|
||||
base http.RoundTripper
|
||||
authForURL AuthFunc
|
||||
|
||||
mu sync.Mutex
|
||||
tokens map[string]cachedToken
|
||||
challenges map[string]bearerChallenge
|
||||
}
|
||||
|
||||
type cachedToken struct {
|
||||
value string
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type bearerChallenge struct {
|
||||
realm string
|
||||
service string
|
||||
scopes []string
|
||||
}
|
||||
|
||||
type tokenResponse struct {
|
||||
Token string `json:"token"`
|
||||
AccessToken string `json:"access_token"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
IssuedAt string `json:"issued_at"`
|
||||
}
|
||||
|
||||
// NewTransport creates an authentication-aware transport around base.
|
||||
func NewTransport(base http.RoundTripper, authForURL AuthFunc) *Transport {
|
||||
if base == nil {
|
||||
base = http.DefaultTransport
|
||||
}
|
||||
return &Transport{
|
||||
base: base,
|
||||
authForURL: authForURL,
|
||||
tokens: make(map[string]cachedToken),
|
||||
challenges: make(map[string]bearerChallenge),
|
||||
}
|
||||
}
|
||||
|
||||
// RoundTrip implements http.RoundTripper.
|
||||
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
hasExplicitAuthorization := req.Header.Get("Authorization") != ""
|
||||
outbound := cloneRequest(req)
|
||||
t.applyAuthentication(outbound, hasExplicitAuthorization)
|
||||
|
||||
resp, err := t.base.RoundTrip(outbound)
|
||||
if err != nil || resp.StatusCode != http.StatusUnauthorized {
|
||||
return resp, err
|
||||
}
|
||||
if hasExplicitAuthorization {
|
||||
return resp, nil
|
||||
}
|
||||
if registryProtectionSpace(req.URL) == "" {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate"))
|
||||
if !ok || !canReplay(req) {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
drainAndClose(resp.Body)
|
||||
token, err := t.token(req.Context(), challenge)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("registry authentication: %w", err)
|
||||
}
|
||||
t.rememberChallenge(req.URL, challenge)
|
||||
|
||||
retry, err := cloneRequestForRetry(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.applyConfiguredAuthentication(retry)
|
||||
retry.Header.Set("Authorization", "Bearer "+token)
|
||||
return t.base.RoundTrip(retry)
|
||||
}
|
||||
|
||||
func (t *Transport) applyAuthentication(req *http.Request, hasExplicitAuthorization bool) {
|
||||
t.applyConfiguredAuthentication(req)
|
||||
if hasExplicitAuthorization {
|
||||
return
|
||||
}
|
||||
if token := t.cachedTokenForRequest(req.URL); token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Transport) applyConfiguredAuthentication(req *http.Request) {
|
||||
if t.authForURL == nil {
|
||||
return
|
||||
}
|
||||
name, value := t.authForURL(req.URL.String())
|
||||
if name != "" && value != "" && req.Header.Get(name) == "" {
|
||||
req.Header.Set(name, value)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Transport) token(ctx context.Context, challenge bearerChallenge) (string, error) {
|
||||
key := challenge.key()
|
||||
if token := t.cachedToken(key); token != "" {
|
||||
return token, nil
|
||||
}
|
||||
|
||||
token, expiresAt, err := t.fetchToken(ctx, challenge)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
t.cacheToken(key, cachedToken{value: token, expiresAt: expiresAt})
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (t *Transport) cacheToken(key string, token cachedToken) {
|
||||
now := time.Now()
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
for cachedKey, cached := range t.tokens {
|
||||
if !now.Before(cached.expiresAt) {
|
||||
delete(t.tokens, cachedKey)
|
||||
}
|
||||
}
|
||||
t.tokens[key] = token
|
||||
}
|
||||
|
||||
func (t *Transport) fetchToken(ctx context.Context, challenge bearerChallenge) (string, time.Time, error) {
|
||||
tokenURL, err := url.Parse(challenge.realm)
|
||||
if err != nil || !tokenURL.IsAbs() || (tokenURL.Scheme != "https" && tokenURL.Scheme != "http") {
|
||||
return "", time.Time{}, fmt.Errorf("invalid token realm %q", challenge.realm)
|
||||
}
|
||||
|
||||
query := tokenURL.Query()
|
||||
if challenge.service != "" {
|
||||
query.Set("service", challenge.service)
|
||||
}
|
||||
for _, scope := range challenge.scopes {
|
||||
query.Add("scope", scope)
|
||||
}
|
||||
query.Set("client_id", "git-pkgs-proxy")
|
||||
tokenURL.RawQuery = query.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL.String(), nil)
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
|
||||
client := &http.Client{Transport: configuredTransport{parent: t}}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", time.Time{}, fmt.Errorf("requesting token: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTokenResponseSize))
|
||||
return "", time.Time{}, fmt.Errorf("token service returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
var payload tokenResponse
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxTokenResponseSize)).Decode(&payload); err != nil {
|
||||
return "", time.Time{}, fmt.Errorf("decoding token response: %w", err)
|
||||
}
|
||||
token := payload.Token
|
||||
if token == "" {
|
||||
token = payload.AccessToken
|
||||
}
|
||||
if token == "" {
|
||||
return "", time.Time{}, fmt.Errorf("token response did not contain a token")
|
||||
}
|
||||
|
||||
issuedAt := time.Now()
|
||||
if payload.IssuedAt != "" {
|
||||
if parsed, parseErr := time.Parse(time.RFC3339, payload.IssuedAt); parseErr == nil {
|
||||
issuedAt = parsed
|
||||
}
|
||||
}
|
||||
lifetime := time.Duration(payload.ExpiresIn) * time.Second
|
||||
if lifetime <= 0 {
|
||||
lifetime = defaultTokenLifetime
|
||||
}
|
||||
expiresAt := issuedAt.Add(lifetime).Add(-expirySkew(lifetime))
|
||||
return token, expiresAt, nil
|
||||
}
|
||||
|
||||
type configuredTransport struct {
|
||||
parent *Transport
|
||||
}
|
||||
|
||||
func (t configuredTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
outbound := cloneRequest(req)
|
||||
t.parent.applyConfiguredAuthentication(outbound)
|
||||
return t.parent.base.RoundTrip(outbound)
|
||||
}
|
||||
|
||||
func (t *Transport) cachedTokenForRequest(requestURL *url.URL) string {
|
||||
space := registryProtectionSpace(requestURL)
|
||||
if space == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
t.mu.Lock()
|
||||
challenge, ok := t.challenges[space]
|
||||
t.mu.Unlock()
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return t.cachedToken(challenge.key())
|
||||
}
|
||||
|
||||
func (t *Transport) cachedToken(key string) string {
|
||||
now := time.Now()
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
token, ok := t.tokens[key]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if !now.Before(token.expiresAt) {
|
||||
delete(t.tokens, key)
|
||||
return ""
|
||||
}
|
||||
return token.value
|
||||
}
|
||||
|
||||
func (t *Transport) rememberChallenge(requestURL *url.URL, challenge bearerChallenge) {
|
||||
space := registryProtectionSpace(requestURL)
|
||||
if space == "" {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.challenges[space] = challenge
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c bearerChallenge) key() string {
|
||||
return c.realm + "\x00" + c.service + "\x00" + strings.Join(c.scopes, "\x00")
|
||||
}
|
||||
|
||||
func registryProtectionSpace(u *url.URL) string {
|
||||
const registryPrefix = "/v2/"
|
||||
if u == nil || !strings.HasPrefix(u.Path, registryPrefix) {
|
||||
return ""
|
||||
}
|
||||
rest := strings.TrimPrefix(u.Path, registryPrefix)
|
||||
end := len(rest)
|
||||
for _, marker := range []string{"/blobs/", "/manifests/", "/tags/", "/referrers/"} {
|
||||
if index := strings.Index(rest, marker); index >= 0 && index < end {
|
||||
end = index
|
||||
}
|
||||
}
|
||||
if end == len(rest) || end == 0 {
|
||||
return ""
|
||||
}
|
||||
return u.Scheme + "://" + u.Host + registryPrefix + rest[:end]
|
||||
}
|
||||
|
||||
func parseBearerChallenge(values []string) (bearerChallenge, bool) {
|
||||
for _, value := range values {
|
||||
params, ok := bearerParameters(value)
|
||||
if !ok || params["realm"] == "" {
|
||||
continue
|
||||
}
|
||||
challenge := bearerChallenge{
|
||||
realm: params["realm"],
|
||||
service: params["service"],
|
||||
}
|
||||
if scope := params["scope"]; scope != "" {
|
||||
challenge.scopes = append(challenge.scopes, scope)
|
||||
}
|
||||
return challenge, true
|
||||
}
|
||||
return bearerChallenge{}, false
|
||||
}
|
||||
|
||||
func bearerParameters(value string) (map[string]string, bool) {
|
||||
start := findAuthScheme(value, "Bearer")
|
||||
if start < 0 {
|
||||
return nil, false
|
||||
}
|
||||
rest := value[start+len("Bearer"):]
|
||||
params := make(map[string]string)
|
||||
for {
|
||||
rest = strings.TrimLeft(rest, " \t,")
|
||||
if rest == "" {
|
||||
break
|
||||
}
|
||||
|
||||
keyEnd := strings.IndexAny(rest, "= \t,")
|
||||
if keyEnd <= 0 {
|
||||
break
|
||||
}
|
||||
key := strings.ToLower(rest[:keyEnd])
|
||||
rest = strings.TrimLeft(rest[keyEnd:], " \t")
|
||||
if rest == "" || rest[0] != '=' {
|
||||
break
|
||||
}
|
||||
rest = strings.TrimLeft(rest[1:], " \t")
|
||||
|
||||
parsed, remaining, ok := parseAuthValue(rest)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
params[key] = parsed
|
||||
rest = remaining
|
||||
}
|
||||
return params, true
|
||||
}
|
||||
|
||||
func findAuthScheme(value, scheme string) int {
|
||||
inQuote := false
|
||||
escaped := false
|
||||
for index := 0; index+len(scheme) <= len(value); index++ {
|
||||
char := value[index]
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if char == '\\' && inQuote {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if char == '"' {
|
||||
inQuote = !inQuote
|
||||
continue
|
||||
}
|
||||
if inQuote || !strings.EqualFold(value[index:index+len(scheme)], scheme) {
|
||||
continue
|
||||
}
|
||||
beforeOK := index == 0 || value[index-1] == ',' || value[index-1] == ' ' || value[index-1] == '\t'
|
||||
after := index + len(scheme)
|
||||
afterOK := after < len(value) && (value[after] == ' ' || value[after] == '\t')
|
||||
if beforeOK && afterOK {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func parseAuthValue(value string) (parsed, remaining string, ok bool) {
|
||||
if value == "" {
|
||||
return "", "", false
|
||||
}
|
||||
if value[0] != '"' {
|
||||
end := strings.IndexAny(value, " \t,")
|
||||
if end < 0 {
|
||||
return value, "", true
|
||||
}
|
||||
return value[:end], value[end:], end > 0
|
||||
}
|
||||
|
||||
var builder strings.Builder
|
||||
escaped := false
|
||||
for index := 1; index < len(value); index++ {
|
||||
char := value[index]
|
||||
if escaped {
|
||||
builder.WriteByte(char)
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if char == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if char == '"' {
|
||||
return builder.String(), value[index+1:], true
|
||||
}
|
||||
builder.WriteByte(char)
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
func cloneRequest(req *http.Request) *http.Request {
|
||||
clone := req.Clone(req.Context())
|
||||
clone.Header = req.Header.Clone()
|
||||
return clone
|
||||
}
|
||||
|
||||
func canReplay(req *http.Request) bool {
|
||||
return req.Body == nil || req.GetBody != nil
|
||||
}
|
||||
|
||||
func cloneRequestForRetry(req *http.Request) (*http.Request, error) {
|
||||
clone := cloneRequest(req)
|
||||
if req.Body == nil {
|
||||
return clone, nil
|
||||
}
|
||||
body, err := req.GetBody()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("replaying authenticated request: %w", err)
|
||||
}
|
||||
clone.Body = body
|
||||
return clone, nil
|
||||
}
|
||||
|
||||
func expirySkew(lifetime time.Duration) time.Duration {
|
||||
if lifetime < tokenExpirySkew*2 {
|
||||
return lifetime / shortTokenSkewDivisor
|
||||
}
|
||||
return tokenExpirySkew
|
||||
}
|
||||
|
||||
func drainAndClose(body io.ReadCloser) {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(body, maxTokenResponseSize))
|
||||
_ = body.Close()
|
||||
}
|
||||
|
|
@ -1,251 +0,0 @@
|
|||
package httpclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTransportFollowsBearerChallengeAndCachesToken(t *testing.T) {
|
||||
var registryRequests int
|
||||
var tokenRequests int
|
||||
var server *httptest.Server
|
||||
|
||||
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/token":
|
||||
tokenRequests++
|
||||
if got := r.URL.Query().Get("service"); got != "registry.test" {
|
||||
t.Errorf("service = %q, want %q", got, "registry.test")
|
||||
}
|
||||
if got := r.URL.Query().Get("scope"); got != "repository:library/test:pull" {
|
||||
t.Errorf("scope = %q, want %q", got, "repository:library/test:pull")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"token":"registry-token","expires_in":3600}`)
|
||||
case "/v2/library/test/blobs/sha256:first", "/v2/library/test/blobs/sha256:second":
|
||||
registryRequests++
|
||||
if r.Header.Get("Authorization") != "Bearer registry-token" {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token",service="registry.test",scope="repository:library/test:pull"`)
|
||||
http.Error(w, "authentication required", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, "blob")
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
|
||||
for _, digest := range []string{"sha256:first", "sha256:second"} {
|
||||
resp, err := client.Get(server.URL + "/v2/library/test/blobs/" + digest)
|
||||
if err != nil {
|
||||
t.Fatalf("GET %s: %v", digest, err)
|
||||
}
|
||||
body, readErr := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if readErr != nil {
|
||||
t.Fatalf("read %s response: %v", digest, readErr)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("GET %s status = %d, want %d", digest, resp.StatusCode, http.StatusOK)
|
||||
}
|
||||
if string(body) != "blob" {
|
||||
t.Errorf("GET %s body = %q, want %q", digest, body, "blob")
|
||||
}
|
||||
}
|
||||
|
||||
if tokenRequests != 1 {
|
||||
t.Errorf("token requests = %d, want 1", tokenRequests)
|
||||
}
|
||||
if registryRequests != 3 {
|
||||
t.Errorf("registry requests = %d, want 3", registryRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportAddsConfiguredAuthentication(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("X-Registry-Token"); got != "configured-token" {
|
||||
t.Errorf("X-Registry-Token = %q, want %q", got, "configured-token")
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
authForURL := func(url string) (string, string) {
|
||||
if strings.HasPrefix(url, server.URL) {
|
||||
return "X-Registry-Token", "configured-token"
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)}
|
||||
|
||||
resp, err := client.Get(server.URL + "/metadata")
|
||||
if err != nil {
|
||||
t.Fatalf("GET metadata: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportPreservesExplicitAuthentication(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" {
|
||||
t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token")
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
authForURL := func(string) (string, string) {
|
||||
return "Authorization", "Bearer configured-token"
|
||||
}
|
||||
client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)}
|
||||
req, err := http.NewRequest(http.MethodGet, server.URL+"/artifact", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer explicit-token")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("GET artifact: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportDoesNotReplaceExplicitAuthenticationAfterBearerChallenge(t *testing.T) {
|
||||
var registryRequests int
|
||||
var tokenRequests int
|
||||
var server *httptest.Server
|
||||
|
||||
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/token":
|
||||
tokenRequests++
|
||||
_, _ = io.WriteString(w, `{"token":"registry-token"}`)
|
||||
case "/v2/library/test/blobs/sha256:test":
|
||||
registryRequests++
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" {
|
||||
t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token")
|
||||
}
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`)
|
||||
http.Error(w, "authentication required", http.StatusUnauthorized)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
|
||||
req, err := http.NewRequest(http.MethodGet, server.URL+"/v2/library/test/blobs/sha256:test", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer explicit-token")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("GET blob: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized)
|
||||
}
|
||||
if registryRequests != 1 {
|
||||
t.Errorf("registry requests = %d, want 1", registryRequests)
|
||||
}
|
||||
if tokenRequests != 0 {
|
||||
t.Errorf("token requests = %d, want 0", tokenRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportDoesNotForwardConfiguredAuthenticationOnTokenRedirect(t *testing.T) {
|
||||
destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("X-Registry-Token"); got != "" {
|
||||
t.Errorf("redirected X-Registry-Token = %q, want empty", got)
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"token":"registry-token"}`)
|
||||
}))
|
||||
defer destination.Close()
|
||||
|
||||
source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("X-Registry-Token"); got != "configured-token" {
|
||||
t.Errorf("source X-Registry-Token = %q, want %q", got, "configured-token")
|
||||
}
|
||||
http.Redirect(w, r, destination.URL+"/token", http.StatusFound)
|
||||
}))
|
||||
defer source.Close()
|
||||
|
||||
authForURL := func(rawURL string) (string, string) {
|
||||
if strings.HasPrefix(rawURL, source.URL) {
|
||||
return "X-Registry-Token", "configured-token"
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
transport := NewTransport(http.DefaultTransport, authForURL)
|
||||
token, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: source.URL + "/token"})
|
||||
if err != nil {
|
||||
t.Fatalf("fetchToken: %v", err)
|
||||
}
|
||||
if token != "registry-token" {
|
||||
t.Errorf("token = %q, want %q", token, "registry-token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportPrunesExpiredTokens(t *testing.T) {
|
||||
transport := NewTransport(http.DefaultTransport, nil)
|
||||
transport.tokens["expired-unused"] = cachedToken{
|
||||
value: "expired-token",
|
||||
expiresAt: time.Now().Add(-time.Minute),
|
||||
}
|
||||
transport.cacheToken("current", cachedToken{
|
||||
value: "current-token",
|
||||
expiresAt: time.Now().Add(time.Minute),
|
||||
})
|
||||
|
||||
if got := transport.cachedToken("current"); got != "current-token" {
|
||||
t.Errorf("cachedToken(current) = %q, want %q", got, "current-token")
|
||||
}
|
||||
if _, ok := transport.tokens["expired-unused"]; ok {
|
||||
t.Error("expired unused token was not pruned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportDoesNotFollowBearerChallengeOutsideOCIRegistry(t *testing.T) {
|
||||
tokenRequests := 0
|
||||
var server *httptest.Server
|
||||
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/token" {
|
||||
tokenRequests++
|
||||
_, _ = io.WriteString(w, `{"token":"unexpected"}`)
|
||||
return
|
||||
}
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`)
|
||||
http.Error(w, "authentication required", http.StatusUnauthorized)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
|
||||
resp, err := client.Get(server.URL + "/api/packages")
|
||||
if err != nil {
|
||||
t.Fatalf("GET API: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized)
|
||||
}
|
||||
if tokenRequests != 0 {
|
||||
t.Errorf("token requests = %d, want 0", tokenRequests)
|
||||
}
|
||||
}
|
||||
|
|
@ -6,7 +6,6 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
|
@ -121,22 +120,6 @@ var (
|
|||
Help: "Number of currently active requests",
|
||||
},
|
||||
)
|
||||
|
||||
IntegrityFailures = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "proxy_integrity_failures_total",
|
||||
Help: "Cached artifacts that failed hash verification on read",
|
||||
},
|
||||
[]string{"ecosystem"},
|
||||
)
|
||||
|
||||
HealthProbeFailures = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "proxy_health_probe_failures_total",
|
||||
Help: "Total number of storage health probe failures, by step (write|size|read|verify|delete).",
|
||||
},
|
||||
[]string{"step"},
|
||||
)
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
@ -155,8 +138,6 @@ func init() {
|
|||
StorageOperationDuration,
|
||||
StorageErrors,
|
||||
ActiveRequests,
|
||||
IntegrityFailures,
|
||||
HealthProbeFailures,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -174,12 +155,12 @@ func RecordRequest(ecosystem string, status int, duration time.Duration) {
|
|||
|
||||
// RecordCacheHit increments cache hit counter.
|
||||
func RecordCacheHit(ecosystem string) {
|
||||
CacheHits.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc()
|
||||
CacheHits.WithLabelValues(ecosystem).Inc()
|
||||
}
|
||||
|
||||
// RecordCacheMiss increments cache miss counter.
|
||||
func RecordCacheMiss(ecosystem string) {
|
||||
CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc()
|
||||
CacheMisses.WithLabelValues(ecosystem).Inc()
|
||||
}
|
||||
|
||||
// RecordUpstreamFetch tracks upstream fetch duration.
|
||||
|
|
@ -197,17 +178,6 @@ func RecordStorageOperation(operation string, duration time.Duration) {
|
|||
StorageOperationDuration.WithLabelValues(operation).Observe(duration.Seconds())
|
||||
}
|
||||
|
||||
// RecordIntegrityFailure increments the integrity failure counter.
|
||||
func RecordIntegrityFailure(ecosystem string) {
|
||||
IntegrityFailures.WithLabelValues(ecosystem).Inc()
|
||||
}
|
||||
|
||||
// RecordHealthProbeFailure increments the health probe failure counter.
|
||||
// step is one of: "write", "size", "read", "verify", "delete".
|
||||
func RecordHealthProbeFailure(step string) {
|
||||
HealthProbeFailures.WithLabelValues(step).Inc()
|
||||
}
|
||||
|
||||
// RecordStorageError increments storage error counter.
|
||||
func RecordStorageError(operation string) {
|
||||
StorageErrors.WithLabelValues(operation).Inc()
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
)
|
||||
|
||||
|
|
@ -192,45 +191,22 @@ func TestMetricsEndpointOutput(t *testing.T) {
|
|||
|
||||
func TestMetricsLabeling(t *testing.T) {
|
||||
// Test that different ecosystems are properly labeled
|
||||
ecosystems := []struct {
|
||||
input string
|
||||
label string
|
||||
}{
|
||||
{input: "npm", label: "npm"},
|
||||
{input: "pypi", label: "pypi"},
|
||||
{input: "cargo", label: "cargo"},
|
||||
{input: "gem", label: "rubygems"},
|
||||
}
|
||||
ecosystems := []string{"npm", "pypi", "cargo", "gem"}
|
||||
|
||||
for _, eco := range ecosystems {
|
||||
RecordRequest(eco.input, 200, 10*time.Millisecond)
|
||||
RecordCacheHit(eco.input)
|
||||
RecordRequest(eco, 200, 10*time.Millisecond)
|
||||
RecordCacheHit(eco)
|
||||
}
|
||||
|
||||
// Verify each ecosystem has metrics
|
||||
for _, eco := range ecosystems {
|
||||
val := getMetricValue(t, CacheHits, eco.label)
|
||||
val := getMetricValue(t, CacheHits, eco)
|
||||
if val == 0 {
|
||||
t.Errorf("no cache hits recorded for %s", eco.label)
|
||||
t.Errorf("no cache hits recorded for %s", eco)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheMetricLabelsAreNormalized(t *testing.T) {
|
||||
rubyHitsBefore := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems"))
|
||||
composerMissesBefore := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist"))
|
||||
|
||||
RecordCacheHit("gem")
|
||||
RecordCacheMiss("composer")
|
||||
|
||||
if diff := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems")) - rubyHitsBefore; diff != 1 {
|
||||
t.Errorf("rubygems cache hits delta = %.0f, want 1", diff)
|
||||
}
|
||||
if diff := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist")) - composerMissesBefore; diff != 1 {
|
||||
t.Errorf("packagist cache misses delta = %.0f, want 1", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricNames(t *testing.T) {
|
||||
// Verify metric names follow Prometheus naming conventions
|
||||
expectedMetrics := []string{
|
||||
|
|
|
|||
16
internal/mirror/registry.go
Normal file
16
internal/mirror/registry.go
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
package mirror
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// RegistrySource enumerates all packages in a registry for full mirroring.
|
||||
// Registry enumeration is not yet implemented for any ecosystem.
|
||||
type RegistrySource struct {
|
||||
Ecosystem string
|
||||
}
|
||||
|
||||
func (s *RegistrySource) Enumerate(_ context.Context, _ func(PackageVersion) error) error {
|
||||
return fmt.Errorf("registry enumeration is not yet implemented for ecosystem %q", s.Ecosystem)
|
||||
}
|
||||
46
internal/mirror/registry_test.go
Normal file
46
internal/mirror/registry_test.go
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
package mirror
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRegistrySourceUnsupported(t *testing.T) {
|
||||
source := &RegistrySource{Ecosystem: "golang"}
|
||||
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for unsupported ecosystem")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrySourceNPMNotImplemented(t *testing.T) {
|
||||
source := &RegistrySource{Ecosystem: "npm"}
|
||||
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected not-implemented error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrySourcePyPINotImplemented(t *testing.T) {
|
||||
source := &RegistrySource{Ecosystem: "pypi"}
|
||||
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected not-implemented error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrySourceCargoNotImplemented(t *testing.T) {
|
||||
source := &RegistrySource{Ecosystem: "cargo"}
|
||||
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected not-implemented error")
|
||||
}
|
||||
}
|
||||
|
|
@ -139,14 +139,15 @@ type BulkResponse struct {
|
|||
// Resolves namespaced package names (Composer vendor/name, npm @scope/name) from the path.
|
||||
func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
|
||||
ecosystem := chi.URLParam(r, "ecosystem")
|
||||
segments, err := packagePathSegments(r)
|
||||
if err != nil {
|
||||
badRequest(w, err.Error())
|
||||
wildcard := chi.URLParam(r, "*")
|
||||
if err := validatePackagePath(wildcard); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
segments := splitWildcardPath(wildcard)
|
||||
|
||||
if ecosystem == "" || len(segments) == 0 {
|
||||
badRequest(w, "ecosystem and name are required")
|
||||
http.Error(w, "ecosystem and name are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -193,12 +194,12 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
|
|||
func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) {
|
||||
info, err := h.enrichment.EnrichPackage(r.Context(), ecosystem, name)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich package")
|
||||
http.Error(w, "failed to enrich package", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if info == nil {
|
||||
notFound(w, "package not found")
|
||||
http.Error(w, "package not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -220,7 +221,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste
|
|||
func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) {
|
||||
result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich version")
|
||||
http.Error(w, "failed to enrich version", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -276,14 +277,15 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste
|
|||
// Supports both {name} and {name}/{version} paths with namespaced package names.
|
||||
func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
|
||||
ecosystem := chi.URLParam(r, "ecosystem")
|
||||
segments, err := packagePathSegments(r)
|
||||
if err != nil {
|
||||
badRequest(w, err.Error())
|
||||
wildcard := chi.URLParam(r, "*")
|
||||
if err := validatePackagePath(wildcard); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
segments := splitWildcardPath(wildcard)
|
||||
|
||||
if ecosystem == "" || len(segments) == 0 {
|
||||
badRequest(w, "ecosystem and name are required")
|
||||
http.Error(w, "ecosystem and name are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -304,7 +306,7 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to check vulnerabilities")
|
||||
http.Error(w, "failed to check vulnerabilities", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -336,19 +338,19 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
|
|||
// @Produce json
|
||||
// @Param request body OutdatedRequest true "Packages to check"
|
||||
// @Success 200 {object} OutdatedResponse
|
||||
// @Failure 400 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Failure 400 {string} string
|
||||
// @Failure 500 {string} string
|
||||
// @Router /api/outdated [post]
|
||||
func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
|
||||
var req OutdatedRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
badRequest(w, "invalid request body")
|
||||
http.Error(w, "invalid request body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if len(req.Packages) == 0 {
|
||||
badRequest(w, "packages list is required")
|
||||
http.Error(w, "packages list is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -382,19 +384,19 @@ func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) {
|
|||
// @Produce json
|
||||
// @Param request body BulkRequest true "PURLs"
|
||||
// @Success 200 {object} BulkResponse
|
||||
// @Failure 400 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Failure 400 {string} string
|
||||
// @Failure 500 {string} string
|
||||
// @Router /api/bulk [post]
|
||||
func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
|
||||
var req BulkRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
badRequest(w, "invalid request body")
|
||||
http.Error(w, "invalid request body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if len(req.PURLs) == 0 {
|
||||
badRequest(w, "purls list is required")
|
||||
http.Error(w, "purls list is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -482,15 +484,15 @@ type SearchPackageResult struct {
|
|||
// @Param q query string true "Query"
|
||||
// @Param ecosystem query string false "Ecosystem"
|
||||
// @Success 200 {object} SearchResponse
|
||||
// @Failure 400 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Failure 400 {string} string
|
||||
// @Failure 500 {string} string
|
||||
// @Router /api/search [get]
|
||||
func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query().Get("q")
|
||||
ecosystem := r.URL.Query().Get("ecosystem")
|
||||
|
||||
if query == "" {
|
||||
badRequest(w, "query parameter 'q' is required")
|
||||
http.Error(w, "query parameter 'q' is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -500,7 +502,7 @@ func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
|
|||
// Search in database
|
||||
results, err := h.db.SearchPackages(query, ecosystem, limit, (page-1)*limit)
|
||||
if err != nil {
|
||||
internalError(w, "search failed")
|
||||
http.Error(w, "search failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -544,7 +546,7 @@ func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
|
|||
func writeJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(v); err != nil {
|
||||
internalError(w, "failed to encode response")
|
||||
http.Error(w, "failed to encode response", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -579,8 +581,8 @@ type PackageListResult struct {
|
|||
// @Param ecosystem query string false "Ecosystem"
|
||||
// @Param sort query string false "Sort" Enums(hits,name,size,cached_at,ecosystem,vulns)
|
||||
// @Success 200 {object} PackagesListResponse
|
||||
// @Failure 400 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Failure 400 {string} string
|
||||
// @Failure 500 {string} string
|
||||
// @Router /api/packages [get]
|
||||
func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request) {
|
||||
ecosystem := r.URL.Query().Get("ecosystem")
|
||||
|
|
@ -598,7 +600,7 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request)
|
|||
"vulns": true,
|
||||
}
|
||||
if !validSorts[sortBy] {
|
||||
badRequest(w, "invalid sort parameter")
|
||||
http.Error(w, "invalid sort parameter", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -607,7 +609,7 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request)
|
|||
|
||||
packages, err := h.db.ListCachedPackages(ecosystem, sortBy, limit, (page-1)*limit)
|
||||
if err != nil {
|
||||
internalError(w, "failed to list packages")
|
||||
http.Error(w, "failed to list packages", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -11,46 +11,27 @@ import (
|
|||
|
||||
"github.com/git-pkgs/archives"
|
||||
"github.com/git-pkgs/archives/diff"
|
||||
"github.com/git-pkgs/magic"
|
||||
"github.com/git-pkgs/proxy/internal/database"
|
||||
"github.com/git-pkgs/proxy/internal/handler"
|
||||
"github.com/git-pkgs/purl"
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
const (
|
||||
contentTypePlainText = "text/plain; charset=utf-8"
|
||||
browseSniffSize = 512
|
||||
)
|
||||
const contentTypePlainText = "text/plain; charset=utf-8"
|
||||
|
||||
// maxBrowseArchiveSize caps how much data openArchive will buffer for
|
||||
// prefix detection. Artifacts larger than this are rejected to prevent
|
||||
// memory exhaustion from a single request.
|
||||
const maxBrowseArchiveSize = 512 << 20 // 512 MB
|
||||
|
||||
// firstBrowsableArtifact returns the first cached artifact that can be opened as
|
||||
// an archive, or nil if the version has none.
|
||||
//
|
||||
// A version's artifact list is not all archives: a PEP 658 core-metadata sidecar
|
||||
// resolves to the same name and version as the distribution it describes, so it
|
||||
// is cached under that version too. Sidecars are plain text, and because '-'
|
||||
// sorts before '.' one can even precede the real distribution in the
|
||||
// filename-ordered list, so selecting blindly would hand openArchive a file it
|
||||
// cannot parse.
|
||||
func firstBrowsableArtifact(artifacts []database.Artifact) *database.Artifact {
|
||||
for i := range artifacts {
|
||||
if artifacts[i].StoragePath.Valid && !isMetadataSidecar(artifacts[i].Filename) {
|
||||
return &artifacts[i]
|
||||
// archiveFilename returns a filename suitable for archive format detection.
|
||||
// Some ecosystems (e.g. composer) store artifacts with bare hash filenames
|
||||
// that have no extension. This adds .zip when the original has no extension
|
||||
// and the content is likely a zip archive.
|
||||
func archiveFilename(filename string) string {
|
||||
if path.Ext(filename) == "" {
|
||||
return filename + ".zip"
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isMetadataSidecar reports whether filename is a core-metadata sidecar rather
|
||||
// than a distribution archive.
|
||||
func isMetadataSidecar(filename string) bool {
|
||||
return strings.HasSuffix(filename, handler.PyPIMetadataSuffix)
|
||||
return filename
|
||||
}
|
||||
|
||||
// detectSingleRootDir returns the single top-level directory name if all files
|
||||
|
|
@ -86,6 +67,13 @@ func detectSingleRootDir(reader archives.Reader) string {
|
|||
// and stripping a single top-level directory prefix (like GitHub zipballs).
|
||||
// For npm, the hardcoded "package/" prefix takes precedence.
|
||||
func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations
|
||||
fname := archiveFilename(filename)
|
||||
|
||||
// npm always uses package/ prefix
|
||||
if ecosystem == "npm" {
|
||||
return archives.OpenWithPrefix(fname, content, "package/")
|
||||
}
|
||||
|
||||
limited := io.LimitReader(content, maxBrowseArchiveSize+1)
|
||||
data, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
|
|
@ -95,18 +83,15 @@ func openArchive(filename string, content io.Reader, ecosystem string) (archives
|
|||
return nil, fmt.Errorf("artifact too large for browsing (%d bytes)", len(data))
|
||||
}
|
||||
|
||||
if ecosystem == "npm" {
|
||||
return archives.OpenBytesWithPrefix(filename, data, "package/")
|
||||
}
|
||||
|
||||
probe, err := archives.OpenBytes(filename, data)
|
||||
// Open once to detect root prefix
|
||||
probe, err := archives.Open(fname, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prefix := detectSingleRootDir(probe)
|
||||
_ = probe.Close()
|
||||
|
||||
return archives.OpenBytesWithPrefix(filename, data, prefix)
|
||||
return archives.OpenWithPrefix(fname, bytes.NewReader(data), prefix)
|
||||
}
|
||||
|
||||
// BrowseListResponse contains the file listing for a directory in an archives.
|
||||
|
|
@ -135,9 +120,9 @@ type BrowseFileInfo struct {
|
|||
// @Param version path string true "Version"
|
||||
// @Param path query string false "Directory path inside the archive"
|
||||
// @Success 200 {object} BrowseListResponse
|
||||
// @Failure 404 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Router /ui/api/browse/{ecosystem}/{name}/{version} [get]
|
||||
// @Failure 404 {string} string
|
||||
// @Failure 500 {string} string
|
||||
// @Router /api/browse/{ecosystem}/{name}/{version} [get]
|
||||
// handleBrowsePath dispatches /api/browse/{ecosystem}/* to the appropriate browse handler.
|
||||
// It resolves namespaced package names by consulting the database.
|
||||
//
|
||||
|
|
@ -147,14 +132,15 @@ type BrowseFileInfo struct {
|
|||
// {name}/{version}/file/{path} -> browse file
|
||||
func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
||||
ecosystem := chi.URLParam(r, "ecosystem")
|
||||
segments, err := packagePathSegments(r)
|
||||
if err != nil {
|
||||
badRequest(w, err.Error())
|
||||
wildcard := chi.URLParam(r, "*")
|
||||
if err := validatePackagePath(wildcard); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
segments := splitWildcardPath(wildcard)
|
||||
|
||||
if ecosystem == "" || len(segments) < 2 {
|
||||
badRequest(w, "ecosystem, name, and version required")
|
||||
http.Error(w, "ecosystem, name, and version required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -178,7 +164,7 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
|||
rest = nameVersionSegments[len(nameVersionSegments)-1:]
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
notFound(w, "not found")
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
s.browseFile(w, r, ecosystem, name, rest[0], filePath)
|
||||
|
|
@ -192,7 +178,7 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
|||
rest = segments[len(segments)-1:]
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
notFound(w, "not found")
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
s.browseList(w, r, ecosystem, name, rest[0])
|
||||
|
|
@ -202,14 +188,15 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
|||
// Supported paths: {name}/{fromVersion}/{toVersion}
|
||||
func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) {
|
||||
ecosystem := chi.URLParam(r, "ecosystem")
|
||||
segments, err := packagePathSegments(r)
|
||||
if err != nil {
|
||||
badRequest(w, err.Error())
|
||||
wildcard := chi.URLParam(r, "*")
|
||||
if err := validatePackagePath(wildcard); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
segments := splitWildcardPath(wildcard)
|
||||
|
||||
if ecosystem == "" || len(segments) < 3 {
|
||||
badRequest(w, "ecosystem, name, fromVersion, and toVersion required")
|
||||
http.Error(w, "ecosystem, name, fromVersion, and toVersion required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -229,19 +216,26 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
|
||||
if err != nil {
|
||||
notFound(w, "version not found")
|
||||
http.Error(w, "version not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
if len(artifacts) == 0 {
|
||||
notFound(w, "no artifacts cached")
|
||||
http.Error(w, "no artifacts cached", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
cachedArtifact := firstBrowsableArtifact(artifacts)
|
||||
// Find the first cached artifact
|
||||
var cachedArtifact *database.Artifact
|
||||
for i := range artifacts {
|
||||
if artifacts[i].StoragePath.Valid {
|
||||
cachedArtifact = &artifacts[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if cachedArtifact == nil {
|
||||
notFound(w, "artifact not cached")
|
||||
http.Error(w, "artifact not cached", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -249,7 +243,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
artifactReader, err := s.storage.Open(r.Context(), cachedArtifact.StoragePath.String)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to read artifact from storage", "error", err)
|
||||
internalError(w, "failed to read artifact")
|
||||
http.Error(w, "failed to read artifact", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = artifactReader.Close() }()
|
||||
|
|
@ -258,7 +252,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
|
||||
internalError(w, "failed to open archive")
|
||||
http.Error(w, "failed to open archive", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = archiveReader.Close() }()
|
||||
|
|
@ -267,7 +261,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
files, err := archiveReader.ListDir(dirPath)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to list directory", "error", err, "path", dirPath)
|
||||
internalError(w, "failed to list directory")
|
||||
http.Error(w, "failed to list directory", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -302,13 +296,13 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
// @Param version path string true "Version"
|
||||
// @Param filepath path string true "File path inside the archive"
|
||||
// @Success 200 {file} file
|
||||
// @Failure 400 {object} ErrorResponse
|
||||
// @Failure 404 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Router /ui/api/browse/{ecosystem}/{name}/{version}/file/{filepath} [get]
|
||||
// @Failure 400 {string} string
|
||||
// @Failure 404 {string} string
|
||||
// @Failure 500 {string} string
|
||||
// @Router /api/browse/{ecosystem}/{name}/{version}/file/{filepath} [get]
|
||||
func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, name, version, filePath string) {
|
||||
if filePath == "" {
|
||||
badRequest(w, "file path required")
|
||||
http.Error(w, "file path required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -316,19 +310,26 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
|
||||
if err != nil {
|
||||
notFound(w, "version not found")
|
||||
http.Error(w, "version not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
if len(artifacts) == 0 {
|
||||
notFound(w, "no artifacts cached")
|
||||
http.Error(w, "no artifacts cached", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
cachedArtifact := firstBrowsableArtifact(artifacts)
|
||||
// Find the first cached artifact
|
||||
var cachedArtifact *database.Artifact
|
||||
for i := range artifacts {
|
||||
if artifacts[i].StoragePath.Valid {
|
||||
cachedArtifact = &artifacts[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if cachedArtifact == nil {
|
||||
notFound(w, "artifact not cached")
|
||||
http.Error(w, "artifact not cached", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -336,7 +337,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
artifactReader, err := s.storage.Open(r.Context(), cachedArtifact.StoragePath.String)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to read artifact from storage", "error", err)
|
||||
internalError(w, "failed to read artifact")
|
||||
http.Error(w, "failed to read artifact", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = artifactReader.Close() }()
|
||||
|
|
@ -345,7 +346,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
|
||||
internalError(w, "failed to open archive")
|
||||
http.Error(w, "failed to open archive", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = archiveReader.Close() }()
|
||||
|
|
@ -354,23 +355,16 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
fileReader, err := archiveReader.Extract(filePath)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
notFound(w, "file not found")
|
||||
http.Error(w, "file not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
s.logger.Error("failed to extract file", "error", err, "path", filePath)
|
||||
internalError(w, "failed to extract file")
|
||||
http.Error(w, "failed to extract file", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = fileReader.Close() }()
|
||||
|
||||
contentType, knownPath := detectContentTypeFromPath(filePath)
|
||||
var content io.Reader = fileReader
|
||||
if !knownPath {
|
||||
bufferedFile := bufio.NewReaderSize(fileReader, browseSniffSize)
|
||||
prefix, _ := bufferedFile.Peek(browseSniffSize)
|
||||
contentType = detectContentTypeFromPrefix(prefix)
|
||||
content = bufferedFile
|
||||
}
|
||||
contentType := detectContentType(filePath)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Security-Policy", "sandbox")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
|
|
@ -379,103 +373,85 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename))
|
||||
|
||||
// Stream the file
|
||||
_, _ = io.Copy(w, content)
|
||||
_, _ = io.Copy(w, fileReader)
|
||||
}
|
||||
|
||||
func detectContentTypeFromPath(filename string) (string, bool) {
|
||||
// detectContentType returns an appropriate content type based on file extension.
|
||||
func detectContentType(filename string) string {
|
||||
ext := strings.ToLower(path.Ext(filename))
|
||||
|
||||
switch ext {
|
||||
// Text formats
|
||||
case ".txt", ".md", ".markdown":
|
||||
return contentTypePlainText, true
|
||||
return contentTypePlainText
|
||||
case ".html", ".htm", ".xhtml":
|
||||
return contentTypePlainText, true
|
||||
return contentTypePlainText
|
||||
case ".css":
|
||||
return "text/css; charset=utf-8", true
|
||||
return "text/css; charset=utf-8"
|
||||
case ".js", ".mjs":
|
||||
return "application/javascript; charset=utf-8", true
|
||||
return "application/javascript; charset=utf-8"
|
||||
case ".json":
|
||||
return "application/json; charset=utf-8", true
|
||||
return "application/json; charset=utf-8"
|
||||
case ".xml":
|
||||
return "application/xml; charset=utf-8", true
|
||||
return "application/xml; charset=utf-8"
|
||||
case ".yaml", ".yml":
|
||||
return "text/yaml; charset=utf-8", true
|
||||
return "text/yaml; charset=utf-8"
|
||||
case ".toml":
|
||||
return "text/toml; charset=utf-8", true
|
||||
return "text/toml; charset=utf-8"
|
||||
|
||||
// Programming languages
|
||||
case ".go":
|
||||
return "text/x-go; charset=utf-8", true
|
||||
return "text/x-go; charset=utf-8"
|
||||
case ".rs":
|
||||
return "text/x-rust; charset=utf-8", true
|
||||
return "text/x-rust; charset=utf-8"
|
||||
case ".py":
|
||||
return "text/x-python; charset=utf-8", true
|
||||
return "text/x-python; charset=utf-8"
|
||||
case ".rb":
|
||||
return "text/x-ruby; charset=utf-8", true
|
||||
return "text/x-ruby; charset=utf-8"
|
||||
case ".java":
|
||||
return "text/x-java; charset=utf-8", true
|
||||
return "text/x-java; charset=utf-8"
|
||||
case ".c", ".h":
|
||||
return "text/x-c; charset=utf-8", true
|
||||
return "text/x-c; charset=utf-8"
|
||||
case ".cpp", ".cc", ".cxx", ".hpp":
|
||||
return "text/x-c++; charset=utf-8", true
|
||||
return "text/x-c++; charset=utf-8"
|
||||
case ".ts":
|
||||
return "text/typescript; charset=utf-8", true
|
||||
return "text/typescript; charset=utf-8"
|
||||
case ".tsx":
|
||||
return "text/tsx; charset=utf-8", true
|
||||
return "text/tsx; charset=utf-8"
|
||||
case ".jsx":
|
||||
return "text/jsx; charset=utf-8", true
|
||||
return "text/jsx; charset=utf-8"
|
||||
case ".php":
|
||||
return "text/x-php; charset=utf-8", true
|
||||
return "text/x-php; charset=utf-8"
|
||||
|
||||
// Config files
|
||||
case ".conf", ".config", ".ini":
|
||||
return contentTypePlainText, true
|
||||
return contentTypePlainText
|
||||
case ".sh", ".bash":
|
||||
return "text/x-shellscript; charset=utf-8", true
|
||||
return "text/x-shellscript; charset=utf-8"
|
||||
case ".dockerfile":
|
||||
return "text/x-dockerfile; charset=utf-8", true
|
||||
return "text/x-dockerfile; charset=utf-8"
|
||||
|
||||
// Images
|
||||
case ".png":
|
||||
return "image/png", true
|
||||
return "image/png"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg", true
|
||||
return "image/jpeg"
|
||||
case ".gif":
|
||||
return "image/gif", true
|
||||
return "image/gif"
|
||||
case ".svg":
|
||||
return contentTypePlainText, true
|
||||
return contentTypePlainText
|
||||
case ".ico":
|
||||
return "image/x-icon", true
|
||||
return "image/x-icon"
|
||||
|
||||
// Archives
|
||||
case ".zip", ".tar", ".gz", ".bz2", ".xz":
|
||||
return "application/octet-stream", true
|
||||
return "application/octet-stream"
|
||||
|
||||
default:
|
||||
// Try to detect if it looks like text
|
||||
if isLikelyText(filename) {
|
||||
return contentTypePlainText, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func detectContentTypeFromPrefix(prefix []byte) string {
|
||||
result := magic.DetectPrefix(prefix)
|
||||
if result.Kind == magic.KindText {
|
||||
return contentTypePlainText
|
||||
}
|
||||
|
||||
switch result.Format {
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "jpeg":
|
||||
return "image/jpeg"
|
||||
case "gif":
|
||||
return "image/gif"
|
||||
case "pdf":
|
||||
return "application/pdf"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
|
@ -504,16 +480,10 @@ func isLikelyText(filename string) bool {
|
|||
}
|
||||
|
||||
// BrowseSourceData contains data for the browse source page.
|
||||
//
|
||||
// Version is the decoded version, for display. EscapedVersion is the same value
|
||||
// escaped as a single URL path segment and is what the links and the browse API
|
||||
// calls must use; see database.Version.EscapedVersion.
|
||||
type BrowseSourceData struct {
|
||||
Layout
|
||||
Ecosystem string
|
||||
PackageName string
|
||||
Version string
|
||||
EscapedVersion string
|
||||
}
|
||||
|
||||
// handleBrowseSource is now showBrowseSource in server.go, dispatched via handlePackagePath.
|
||||
|
|
@ -529,9 +499,9 @@ type BrowseSourceData struct {
|
|||
// @Param fromVersion path string true "From version"
|
||||
// @Param toVersion path string true "To version"
|
||||
// @Success 200 {object} map[string]any
|
||||
// @Failure 404 {object} ErrorResponse
|
||||
// @Failure 500 {object} ErrorResponse
|
||||
// @Router /ui/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get]
|
||||
// @Failure 404 {string} string
|
||||
// @Failure 500 {string} string
|
||||
// @Router /api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get]
|
||||
func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, name, fromVersion, toVersion string) {
|
||||
// Get artifacts for both versions
|
||||
fromPURL := purl.MakePURLString(ecosystem, name, fromVersion)
|
||||
|
|
@ -539,22 +509,33 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
|
||||
fromArtifacts, err := s.db.GetArtifactsByVersionPURL(fromPURL)
|
||||
if err != nil || len(fromArtifacts) == 0 {
|
||||
notFound(w, "from version not found or not cached")
|
||||
http.Error(w, "from version not found or not cached", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
toArtifacts, err := s.db.GetArtifactsByVersionPURL(toPURL)
|
||||
if err != nil || len(toArtifacts) == 0 {
|
||||
notFound(w, "to version not found or not cached")
|
||||
http.Error(w, "to version not found or not cached", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Find cached artifacts
|
||||
fromArtifact := firstBrowsableArtifact(fromArtifacts)
|
||||
toArtifact := firstBrowsableArtifact(toArtifacts)
|
||||
var fromArtifact, toArtifact *database.Artifact
|
||||
for i := range fromArtifacts {
|
||||
if fromArtifacts[i].StoragePath.Valid {
|
||||
fromArtifact = &fromArtifacts[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
for i := range toArtifacts {
|
||||
if toArtifacts[i].StoragePath.Valid {
|
||||
toArtifact = &toArtifacts[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if fromArtifact == nil || toArtifact == nil {
|
||||
notFound(w, "one or both versions not cached")
|
||||
http.Error(w, "one or both versions not cached", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -562,7 +543,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
fromReader, err := s.storage.Open(r.Context(), fromArtifact.StoragePath.String)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open from artifact", "error", err)
|
||||
internalError(w, "failed to read from version")
|
||||
http.Error(w, "failed to read from version", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = fromReader.Close() }()
|
||||
|
|
@ -570,7 +551,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
toReader, err := s.storage.Open(r.Context(), toArtifact.StoragePath.String)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open to artifact", "error", err)
|
||||
internalError(w, "failed to read to version")
|
||||
http.Error(w, "failed to read to version", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = toReader.Close() }()
|
||||
|
|
@ -578,7 +559,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
fromArchive, err := openArchive(fromArtifact.Filename, fromReader, ecosystem)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open from archive", "error", err)
|
||||
internalError(w, "failed to open from archive")
|
||||
http.Error(w, "failed to open from archive", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = fromArchive.Close() }()
|
||||
|
|
@ -586,7 +567,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
toArchive, err := openArchive(toArtifact.Filename, toReader, ecosystem)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open to archive", "error", err)
|
||||
internalError(w, "failed to open to archive")
|
||||
http.Error(w, "failed to open to archive", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer func() { _ = toArchive.Close() }()
|
||||
|
|
@ -595,7 +576,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
result, err := diff.Compare(fromArchive, toArchive)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to generate diff", "error", err)
|
||||
internalError(w, "failed to generate diff")
|
||||
http.Error(w, "failed to generate diff", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -604,17 +585,11 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
}
|
||||
|
||||
// ComparePageData contains data for the version comparison page.
|
||||
//
|
||||
// FromVersion and ToVersion are decoded, for display; the Escaped variants are
|
||||
// the path-segment form used to build the compare API URL.
|
||||
type ComparePageData struct {
|
||||
Layout
|
||||
Ecosystem string
|
||||
PackageName string
|
||||
FromVersion string
|
||||
ToVersion string
|
||||
EscapedFromVersion string
|
||||
EscapedToVersion string
|
||||
}
|
||||
|
||||
// handleComparePage is now showComparePage in server.go, dispatched via handlePackagePath.
|
||||
|
|
|
|||
|
|
@ -1,89 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func createBenchTarGz(prefix string, fileCount, fileSize int) []byte {
|
||||
rnd := rand.New(rand.NewSource(1)) //nolint:gosec
|
||||
buf := new(bytes.Buffer)
|
||||
gw := gzip.NewWriter(buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
payload := make([]byte, fileSize)
|
||||
for i := range fileCount {
|
||||
rnd.Read(payload)
|
||||
_ = tw.WriteHeader(&tar.Header{
|
||||
Name: fmt.Sprintf("%sfile%04d.dat", prefix, i),
|
||||
Size: int64(fileSize),
|
||||
Mode: 0644,
|
||||
})
|
||||
_, _ = tw.Write(payload)
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = gw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func BenchmarkOpenArchive(b *testing.B) {
|
||||
cases := []struct {
|
||||
name string
|
||||
ecosystem string
|
||||
filename string
|
||||
data []byte
|
||||
}{
|
||||
{"npm", "npm", "pkg.tgz", createBenchTarGz("package/", 64, 16*1024)},
|
||||
{"go", "go", "v1.2.3.tar.gz", createBenchTarGz("repo-abc123/", 64, 16*1024)},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
b.Run(tc.name, func(b *testing.B) {
|
||||
b.SetBytes(int64(len(tc.data)))
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
r, err := openArchive(tc.filename, bytes.NewReader(tc.data), tc.ecosystem)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
_ = r.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDetectContentType(b *testing.B) {
|
||||
cases := []struct {
|
||||
name string
|
||||
filename string
|
||||
prefix []byte
|
||||
knownPath bool
|
||||
}{
|
||||
{"known-path", "README.md", nil, true},
|
||||
{"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize), false},
|
||||
{"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...), false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
b.Run(tc.name, func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
var contentType string
|
||||
if tc.knownPath {
|
||||
for b.Loop() {
|
||||
contentType, _ = detectContentTypeFromPath(tc.filename)
|
||||
}
|
||||
} else {
|
||||
for b.Loop() {
|
||||
contentType = detectContentTypeFromPrefix(tc.prefix)
|
||||
}
|
||||
}
|
||||
if contentType == "" {
|
||||
b.Fatal("empty content type")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -65,7 +65,7 @@ func TestHandleBrowseList(t *testing.T) {
|
|||
}
|
||||
|
||||
// Test listing root directory
|
||||
req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0", nil)
|
||||
req := httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0", nil)
|
||||
w := httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ func TestHandleBrowseList(t *testing.T) {
|
|||
}
|
||||
|
||||
// Test listing subdirectory
|
||||
req = httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0?path=lib", nil)
|
||||
req = httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0?path=lib", nil)
|
||||
w = httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
|
|
@ -137,44 +137,29 @@ func TestHandleBrowseFile(t *testing.T) {
|
|||
t.Fatalf("failed to upsert artifact: %v", err)
|
||||
}
|
||||
|
||||
files := []struct {
|
||||
path string
|
||||
content string
|
||||
contentType string
|
||||
}{
|
||||
{"README.md", "# Test Package\n", contentTypePlainText},
|
||||
{"notes.data", "short text\n", contentTypePlainText},
|
||||
{"logo", "\x89PNG\r\n\x1a\nimage data", "image/png"},
|
||||
{"page", "<!DOCTYPE html><html></html>", contentTypePlainText},
|
||||
{"misleading.txt", "\x89PNG\r\n\x1a\nimage data", contentTypePlainText},
|
||||
}
|
||||
for _, file := range files {
|
||||
t.Run(file.path, func(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/"+file.path, nil)
|
||||
// Test fetching a file
|
||||
req := httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0/file/README.md", nil)
|
||||
w := httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
if w.Body.String() != file.content {
|
||||
t.Errorf("unexpected file content: %q", w.Body.String())
|
||||
|
||||
body := w.Body.String()
|
||||
if body != "# Test Package\n" {
|
||||
t.Errorf("unexpected file content: %q", body)
|
||||
}
|
||||
if got := w.Header().Get("Content-Type"); got != file.contentType {
|
||||
t.Errorf("Content-Type = %q, want %q", got, file.contentType)
|
||||
}
|
||||
if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" {
|
||||
t.Errorf("Content-Security-Policy = %q, want sandbox", got)
|
||||
}
|
||||
if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" {
|
||||
t.Errorf("X-Content-Type-Options = %q, want nosniff", got)
|
||||
}
|
||||
})
|
||||
|
||||
// Check content type
|
||||
contentType := w.Header().Get("Content-Type")
|
||||
if contentType != contentTypePlainText {
|
||||
t.Errorf("expected text/plain content type, got %q", contentType)
|
||||
}
|
||||
|
||||
// Test fetching non-existent file
|
||||
req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil)
|
||||
w := httptest.NewRecorder()
|
||||
req = httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil)
|
||||
w = httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
|
|
@ -182,52 +167,36 @@ func TestHandleBrowseFile(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestBrowseContentTypePolicy(t *testing.T) {
|
||||
func TestDetectContentType(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
prefix []byte
|
||||
expectedCT string
|
||||
}{
|
||||
{"text extension", "file.txt", nil, contentTypePlainText},
|
||||
{"markdown extension", "file.md", nil, contentTypePlainText},
|
||||
{"JSON extension", "file.json", nil, "application/json; charset=utf-8"},
|
||||
{"JavaScript extension", "file.js", nil, "application/javascript; charset=utf-8"},
|
||||
{"Go extension", "file.go", nil, "text/x-go; charset=utf-8"},
|
||||
{"Python extension", "file.py", nil, "text/x-python; charset=utf-8"},
|
||||
{"Rust extension", "file.rs", nil, "text/x-rust; charset=utf-8"},
|
||||
{"HTML extension", "file.html", nil, contentTypePlainText},
|
||||
{"HTM extension", "file.htm", nil, contentTypePlainText},
|
||||
{"XHTML extension", "file.xhtml", nil, contentTypePlainText},
|
||||
{"SVG extension", "file.svg", nil, contentTypePlainText},
|
||||
{"PNG extension", "file.png", nil, "image/png"},
|
||||
{"JPEG extension", "file.jpg", nil, "image/jpeg"},
|
||||
{"README", "README", nil, contentTypePlainText},
|
||||
{"LICENSE", "LICENSE", nil, contentTypePlainText},
|
||||
{"Makefile", "Makefile", nil, contentTypePlainText},
|
||||
{"gitignore", ".gitignore", nil, contentTypePlainText},
|
||||
{"unknown empty", "file.bin", nil, "application/octet-stream"},
|
||||
{"extensionless PNG", "asset", []byte("\x89PNG\r\n\x1a\n"), "image/png"},
|
||||
{"extensionless JPEG", "asset", []byte("\xff\xd8\xff"), "image/jpeg"},
|
||||
{"extensionless GIF", "asset", []byte("GIF89a"), "image/gif"},
|
||||
{"extensionless PDF", "asset", []byte("%PDF-1.7"), "application/pdf"},
|
||||
{"extensionless text", "asset", []byte("plain text\n"), contentTypePlainText},
|
||||
{"extensionless HTML", "asset", []byte("<!DOCTYPE html><html></html>"), contentTypePlainText},
|
||||
{"extensionless XML", "asset", []byte("<?xml version=\"1.0\"?><root/>"), contentTypePlainText},
|
||||
{"extensionless SVG", "asset", []byte("<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>"), contentTypePlainText},
|
||||
{"extensionless ZIP", "asset", []byte("PK\x03\x04"), "application/octet-stream"},
|
||||
{"extensionless binary", "asset", []byte{0, 1, 2}, "application/octet-stream"},
|
||||
{"known path wins", "file.txt", []byte("\x89PNG\r\n\x1a\n"), contentTypePlainText},
|
||||
{"file.txt", contentTypePlainText},
|
||||
{"file.md", contentTypePlainText},
|
||||
{"file.json", "application/json; charset=utf-8"},
|
||||
{"file.js", "application/javascript; charset=utf-8"},
|
||||
{"file.go", "text/x-go; charset=utf-8"},
|
||||
{"file.py", "text/x-python; charset=utf-8"},
|
||||
{"file.rs", "text/x-rust; charset=utf-8"},
|
||||
{"file.html", contentTypePlainText},
|
||||
{"file.htm", contentTypePlainText},
|
||||
{"file.xhtml", contentTypePlainText},
|
||||
{"file.svg", contentTypePlainText},
|
||||
{"file.png", "image/png"},
|
||||
{"file.jpg", "image/jpeg"},
|
||||
{"README", contentTypePlainText},
|
||||
{"LICENSE", contentTypePlainText},
|
||||
{"Makefile", contentTypePlainText},
|
||||
{".gitignore", contentTypePlainText},
|
||||
{"file.bin", "application/octet-stream"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, knownPath := detectContentTypeFromPath(tt.filename)
|
||||
if !knownPath {
|
||||
got = detectContentTypeFromPrefix(tt.prefix)
|
||||
}
|
||||
t.Run(tt.filename, func(t *testing.T) {
|
||||
got := detectContentType(tt.filename)
|
||||
if got != tt.expectedCT {
|
||||
t.Errorf("content type for %q with prefix %q = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT)
|
||||
t.Errorf("detectContentType(%q) = %q, want %q", tt.filename, got, tt.expectedCT)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -235,14 +204,17 @@ func TestBrowseContentTypePolicy(t *testing.T) {
|
|||
|
||||
func TestOpenArchiveSizeLimit(t *testing.T) {
|
||||
huge := bytes.Repeat([]byte("x"), int(maxBrowseArchiveSize)+1)
|
||||
for _, eco := range []string{"npm", "go"} {
|
||||
_, err := openArchive("test.tar.gz", bytes.NewReader(huge), eco)
|
||||
_, err := openArchive("test.tar.gz", bytes.NewReader(huge), "npm")
|
||||
if err != nil {
|
||||
t.Log("npm path streams directly, error is acceptable:", err)
|
||||
}
|
||||
|
||||
_, err = openArchive("test.tar.gz", bytes.NewReader(huge), "go")
|
||||
if err == nil {
|
||||
t.Fatalf("%s: expected error for oversized archive, got nil", eco)
|
||||
t.Fatal("expected error for oversized archive, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "too large") {
|
||||
t.Fatalf("%s: expected 'too large' error, got: %v", eco, err)
|
||||
}
|
||||
t.Fatalf("expected 'too large' error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -286,10 +258,6 @@ func createTestArchive(t *testing.T) []byte {
|
|||
"package/lib/index.js": "module.exports = {};",
|
||||
"package/lib/helper.js": "module.exports.help = () => {};",
|
||||
"package/test/index.test.js": "// tests",
|
||||
"package/notes.data": "short text\n",
|
||||
"package/logo": "\x89PNG\r\n\x1a\nimage data",
|
||||
"package/page": "<!DOCTYPE html><html></html>",
|
||||
"package/misleading.txt": "\x89PNG\r\n\x1a\nimage data",
|
||||
}
|
||||
|
||||
for path, content := range files {
|
||||
|
|
@ -349,7 +317,7 @@ func TestBrowseNonCachedArtifact(t *testing.T) {
|
|||
}
|
||||
|
||||
// Try to browse
|
||||
req := httptest.NewRequest("GET", "/ui/api/browse/npm/not-cached/1.0.0", nil)
|
||||
req := httptest.NewRequest("GET", "/api/browse/npm/not-cached/1.0.0", nil)
|
||||
w := httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
|
|
@ -403,7 +371,7 @@ func TestHandleBrowseSourcePage(t *testing.T) {
|
|||
}
|
||||
|
||||
// Test the browse source page loads
|
||||
req := httptest.NewRequest("GET", "/ui/package/npm/test-browse/1.0.0/browse", nil)
|
||||
req := httptest.NewRequest("GET", "/package/npm/test-browse/1.0.0/browse", nil)
|
||||
w := httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
|
|
@ -430,10 +398,6 @@ func TestHandleBrowseSourcePage(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
if !strings.Contains(body, "proxy test-version (test-commit)") {
|
||||
t.Error("browse source footer should contain proxy build information, not the package version")
|
||||
}
|
||||
|
||||
// Check that the escapeHTML function is present for XSS protection
|
||||
if !strings.Contains(body, "function escapeHTML(str)") {
|
||||
t.Error("browse source page missing escapeHTML function for XSS protection")
|
||||
|
|
@ -454,10 +418,8 @@ func TestHandleBrowseSourcePage(t *testing.T) {
|
|||
if !strings.Contains(body, "const packageName = 'test-browse'") {
|
||||
t.Error("browse source page missing packageName variable")
|
||||
}
|
||||
// The version reaches the browse API as one path segment, so the page holds
|
||||
// its escaped form.
|
||||
if !strings.Contains(body, "const versionPath = '1.0.0'") {
|
||||
t.Error("browse source page missing versionPath variable")
|
||||
if !strings.Contains(body, "const version = '1.0.0'") {
|
||||
t.Error("browse source page missing version variable")
|
||||
}
|
||||
|
||||
// Verify content type
|
||||
|
|
@ -542,7 +504,7 @@ func TestHandleCompareDiff(t *testing.T) {
|
|||
}
|
||||
|
||||
// Test the compare endpoint
|
||||
req := httptest.NewRequest("GET", "/ui/api/compare/npm/test-compare/1.0.0/2.0.0", nil)
|
||||
req := httptest.NewRequest("GET", "/api/compare/npm/test-compare/1.0.0/2.0.0", nil)
|
||||
w := httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
|
|
@ -613,7 +575,7 @@ func TestHandleComparePage(t *testing.T) {
|
|||
defer ts.close()
|
||||
|
||||
// Test valid format with ... separator
|
||||
req := httptest.NewRequest("GET", "/ui/package/npm/test/compare/1.0.0...2.0.0", nil)
|
||||
req := httptest.NewRequest("GET", "/package/npm/test/compare/1.0.0...2.0.0", nil)
|
||||
w := httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
|
|
@ -623,17 +585,16 @@ func TestHandleComparePage(t *testing.T) {
|
|||
|
||||
body := w.Body.String()
|
||||
|
||||
// Check that versions are set correctly in JavaScript. The compare API takes
|
||||
// each version as a path segment, so the page holds their escaped forms.
|
||||
if !strings.Contains(body, "const fromVersionPath = '1.0.0'") {
|
||||
t.Error("page should set fromVersionPath")
|
||||
// Check that versions are set correctly in JavaScript
|
||||
if !strings.Contains(body, "const fromVersion = '1.0.0'") {
|
||||
t.Error("page should set fromVersion")
|
||||
}
|
||||
if !strings.Contains(body, "const toVersionPath = '2.0.0'") {
|
||||
t.Error("page should set toVersionPath")
|
||||
if !strings.Contains(body, "const toVersion = '2.0.0'") {
|
||||
t.Error("page should set toVersion")
|
||||
}
|
||||
|
||||
// Test invalid format (missing separator)
|
||||
req = httptest.NewRequest("GET", "/ui/package/npm/test/compare/invalid", nil)
|
||||
req = httptest.NewRequest("GET", "/package/npm/test/compare/invalid", nil)
|
||||
w = httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
|
|
@ -642,7 +603,7 @@ func TestHandleComparePage(t *testing.T) {
|
|||
}
|
||||
|
||||
// Test with only one dot (should fail)
|
||||
req = httptest.NewRequest("GET", "/ui/package/npm/test/compare/1.0.0.2.0.0", nil)
|
||||
req = httptest.NewRequest("GET", "/package/npm/test/compare/1.0.0.2.0.0", nil)
|
||||
w = httptest.NewRecorder()
|
||||
ts.handler.ServeHTTP(w, req)
|
||||
|
||||
|
|
@ -651,19 +612,25 @@ func TestHandleComparePage(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestOpenArchiveDetectsExtensionlessTarGz(t *testing.T) {
|
||||
reader, err := openArchive("artifact", bytes.NewReader(createTestArchive(t)), "npm")
|
||||
if err != nil {
|
||||
t.Fatalf("openArchive failed: %v", err)
|
||||
func TestArchiveFilename(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"package.tar.gz", "package.tar.gz"},
|
||||
{"d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856.zip"},
|
||||
{"file.zip", "file.zip"},
|
||||
{"archive.tgz", "archive.tgz"},
|
||||
{"noext", "noext.zip"},
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
files, err := reader.List()
|
||||
if err != nil {
|
||||
t.Fatalf("List failed: %v", err)
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got := archiveFilename(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("archiveFilename(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
if len(files) == 0 {
|
||||
t.Fatal("expected files in extensionless archive")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -836,70 +803,3 @@ func createTarGzArchive(t *testing.T, files map[string]string) []byte {
|
|||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// TestFirstBrowsableArtifact guards artifact selection against PEP 658
|
||||
// core-metadata sidecars. A sidecar resolves to the same version as the
|
||||
// distribution it describes, so it is cached under that version, but it is plain
|
||||
// text and openArchive cannot parse it.
|
||||
func TestFirstBrowsableArtifact(t *testing.T) {
|
||||
cached := func(filename string) database.Artifact {
|
||||
return database.Artifact{
|
||||
Filename: filename,
|
||||
StoragePath: sql.NullString{String: "pypi/" + filename, Valid: true},
|
||||
}
|
||||
}
|
||||
uncached := func(filename string) database.Artifact {
|
||||
return database.Artifact{Filename: filename}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
artifacts []database.Artifact
|
||||
want string
|
||||
}{
|
||||
{"no artifacts", nil, ""},
|
||||
{
|
||||
"sidecar only is not browsable",
|
||||
[]database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata")},
|
||||
"",
|
||||
},
|
||||
{
|
||||
// '-' (0x2D) sorts before '.' (0x2E), so the sidecar precedes the
|
||||
// sdist in the filename-ordered list the query returns.
|
||||
"sidecar sorting ahead of the sdist is skipped",
|
||||
[]database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0.tar.gz")},
|
||||
"foo-1.0.tar.gz",
|
||||
},
|
||||
{
|
||||
"sidecar skipped in favour of its own wheel",
|
||||
[]database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0-py3-none-any.whl")},
|
||||
"foo-1.0-py3-none-any.whl",
|
||||
},
|
||||
{
|
||||
"uncached archive is still not selected",
|
||||
[]database.Artifact{cached("foo-1.0.tar.gz.metadata"), uncached("foo-1.0.tar.gz")},
|
||||
"",
|
||||
},
|
||||
{"plain sdist", []database.Artifact{cached("foo-1.0.tar.gz")}, "foo-1.0.tar.gz"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := firstBrowsableArtifact(tt.artifacts)
|
||||
|
||||
if tt.want == "" {
|
||||
if got != nil {
|
||||
t.Fatalf("firstBrowsableArtifact() = %q, want nil", got.Filename)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if got == nil {
|
||||
t.Fatalf("firstBrowsableArtifact() = nil, want %q", tt.want)
|
||||
}
|
||||
if got.Filename != tt.want {
|
||||
t.Errorf("firstBrowsableArtifact() = %q, want %q", got.Filename, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import (
|
|||
|
||||
// DashboardData contains data for rendering the dashboard.
|
||||
type DashboardData struct {
|
||||
Layout
|
||||
Stats DashboardStats
|
||||
EnrichmentStats EnrichmentStatsView
|
||||
RecentPackages []PackageInfo
|
||||
|
|
@ -61,7 +60,6 @@ type RegistryConfig struct {
|
|||
|
||||
// PackageShowData contains data for rendering the package show page.
|
||||
type PackageShowData struct {
|
||||
Layout
|
||||
Package *database.Package
|
||||
Versions []database.Version
|
||||
Vulnerabilities []database.Vulnerability
|
||||
|
|
@ -70,7 +68,6 @@ type PackageShowData struct {
|
|||
|
||||
// VersionShowData contains data for rendering the version show page.
|
||||
type VersionShowData struct {
|
||||
Layout
|
||||
Package *database.Package
|
||||
Version *database.Version
|
||||
Artifacts []database.Artifact
|
||||
|
|
@ -82,7 +79,6 @@ type VersionShowData struct {
|
|||
|
||||
// SearchPageData contains data for rendering the search results page.
|
||||
type SearchPageData struct {
|
||||
Layout
|
||||
Query string
|
||||
Ecosystem string
|
||||
Results []SearchResultItem
|
||||
|
|
@ -108,7 +104,6 @@ type SearchResultItem struct {
|
|||
|
||||
// PackagesListPageData contains data for rendering the packages list page.
|
||||
type PackagesListPageData struct {
|
||||
Layout
|
||||
Ecosystem string
|
||||
SortBy string
|
||||
Results []SearchResultItem
|
||||
|
|
@ -132,7 +127,6 @@ func supportedEcosystems() []string {
|
|||
"gem",
|
||||
"golang",
|
||||
"hex",
|
||||
"julia",
|
||||
"maven",
|
||||
"npm",
|
||||
"nuget",
|
||||
|
|
@ -182,8 +176,6 @@ func ecosystemBadgeClasses(ecosystem string) string {
|
|||
return base + " bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300"
|
||||
case "cran":
|
||||
return base + " bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300"
|
||||
case "julia":
|
||||
return base + " bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300"
|
||||
case "oci":
|
||||
return base + " bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300"
|
||||
case "deb":
|
||||
|
|
@ -294,20 +286,6 @@ index-url = ` + baseURL + `/pypi/simple/</code></pre>`),
|
|||
</mirror>
|
||||
</mirrors>
|
||||
</settings></code></pre>`),
|
||||
},
|
||||
{
|
||||
ID: "gradle",
|
||||
Name: "Gradle Build Cache",
|
||||
Language: "Java/Kotlin",
|
||||
Endpoint: "/gradle/",
|
||||
Instructions: template.HTML(`<p class="config-note">Configure Gradle to use the proxy for HttpBuildCache:</p>
|
||||
<pre><code>// In settings.gradle(.kts)
|
||||
buildCache {
|
||||
remote<HttpBuildCache> {
|
||||
url = uri("` + baseURL + `/gradle/")
|
||||
push = true
|
||||
}
|
||||
}</code></pre>`),
|
||||
},
|
||||
{
|
||||
ID: "nuget",
|
||||
|
|
@ -385,17 +363,6 @@ local({
|
|||
r["CRAN"] <- "` + baseURL + `/cran"
|
||||
options(repos = r)
|
||||
})</code></pre>`),
|
||||
},
|
||||
{
|
||||
ID: "julia",
|
||||
Name: "Julia",
|
||||
Language: "Julia",
|
||||
Endpoint: "/julia/",
|
||||
Instructions: template.HTML(`<p class="config-note">Set the Pkg server before starting Julia:</p>
|
||||
<pre><code>export JULIA_PKG_SERVER=` + baseURL + `/julia</code></pre>
|
||||
<p class="config-note">Or inside a running session:</p>
|
||||
<pre><code>ENV["JULIA_PKG_SERVER"] = "` + baseURL + `/julia"
|
||||
using Pkg; Pkg.update()</code></pre>`),
|
||||
},
|
||||
{
|
||||
ID: "oci",
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Error codes returned in API error responses. These are stable identifiers
|
||||
// that clients can match on; the message text is for humans and may change.
|
||||
const (
|
||||
ErrCodeBadRequest = "BAD_REQUEST"
|
||||
ErrCodeNotFound = "NOT_FOUND"
|
||||
ErrCodeUpstream = "UPSTREAM_ERROR"
|
||||
ErrCodeInternal = "INTERNAL_ERROR"
|
||||
)
|
||||
|
||||
// ErrorResponse is the JSON body returned for API errors.
|
||||
type ErrorResponse struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// writeError sends a JSON error response with the given status, code and
|
||||
// user-facing message. Internal error details should be logged separately
|
||||
// by the caller, never passed as the message.
|
||||
func writeError(w http.ResponseWriter, status int, code, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(ErrorResponse{Code: code, Message: message})
|
||||
}
|
||||
|
||||
func badRequest(w http.ResponseWriter, message string) {
|
||||
writeError(w, http.StatusBadRequest, ErrCodeBadRequest, message)
|
||||
}
|
||||
|
||||
func notFound(w http.ResponseWriter, message string) {
|
||||
writeError(w, http.StatusNotFound, ErrCodeNotFound, message)
|
||||
}
|
||||
|
||||
func internalError(w http.ResponseWriter, message string) {
|
||||
writeError(w, http.StatusInternalServerError, ErrCodeInternal, message)
|
||||
}
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWriteError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fn func(w http.ResponseWriter)
|
||||
status int
|
||||
code string
|
||||
message string
|
||||
}{
|
||||
{
|
||||
name: "badRequest",
|
||||
fn: func(w http.ResponseWriter) { badRequest(w, "missing field") },
|
||||
status: http.StatusBadRequest,
|
||||
code: ErrCodeBadRequest,
|
||||
message: "missing field",
|
||||
},
|
||||
{
|
||||
name: "notFound",
|
||||
fn: func(w http.ResponseWriter) { notFound(w, "package not found") },
|
||||
status: http.StatusNotFound,
|
||||
code: ErrCodeNotFound,
|
||||
message: "package not found",
|
||||
},
|
||||
{
|
||||
name: "internalError",
|
||||
fn: func(w http.ResponseWriter) { internalError(w, "boom") },
|
||||
status: http.StatusInternalServerError,
|
||||
code: ErrCodeInternal,
|
||||
message: "boom",
|
||||
},
|
||||
{
|
||||
name: "upstream",
|
||||
fn: func(w http.ResponseWriter) {
|
||||
writeError(w, http.StatusBadGateway, ErrCodeUpstream, "registry unreachable")
|
||||
},
|
||||
status: http.StatusBadGateway,
|
||||
code: ErrCodeUpstream,
|
||||
message: "registry unreachable",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
tt.fn(w)
|
||||
|
||||
if w.Code != tt.status {
|
||||
t.Errorf("status = %d, want %d", w.Code, tt.status)
|
||||
}
|
||||
if ct := w.Header().Get("Content-Type"); ct != "application/json" {
|
||||
t.Errorf("Content-Type = %q, want application/json", ct)
|
||||
}
|
||||
|
||||
var resp ErrorResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("response body is not valid JSON: %v (body: %q)", err, w.Body.String())
|
||||
}
|
||||
if resp.Code != tt.code {
|
||||
t.Errorf("code = %q, want %q", resp.Code, tt.code)
|
||||
}
|
||||
if resp.Message != tt.message {
|
||||
t.Errorf("message = %q, want %q", resp.Message, tt.message)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIErrorResponseShape(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
badRequest(w, "x")
|
||||
|
||||
var raw map[string]any
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &raw); err != nil {
|
||||
t.Fatalf("invalid JSON: %v", err)
|
||||
}
|
||||
if _, ok := raw["code"]; !ok {
|
||||
t.Error("response missing 'code' field")
|
||||
}
|
||||
if _, ok := raw["message"]; !ok {
|
||||
t.Error("response missing 'message' field")
|
||||
}
|
||||
if len(raw) != 2 {
|
||||
t.Errorf("response has unexpected fields: %v", raw)
|
||||
}
|
||||
}
|
||||
|
|
@ -15,7 +15,7 @@ import (
|
|||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
)
|
||||
|
||||
func setupEvictionTest(t *testing.T) (*database.DB, *storage.Blob) {
|
||||
func setupEvictionTest(t *testing.T) (*database.DB, *storage.Filesystem) {
|
||||
t.Helper()
|
||||
|
||||
tempDir := t.TempDir()
|
||||
|
|
@ -27,7 +27,7 @@ func setupEvictionTest(t *testing.T) (*database.DB, *storage.Blob) {
|
|||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
store, err := storage.OpenBucket(context.Background(), "file://"+storagePath)
|
||||
store, err := storage.NewFilesystem(storagePath)
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
t.Fatalf("failed to create storage: %v", err)
|
||||
|
|
@ -243,7 +243,7 @@ func TestStartEvictionLoop_UnlimitedSkips(t *testing.T) {
|
|||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
store, err := storage.OpenBucket(context.Background(), "file://"+storagePath)
|
||||
store, err := storage.NewFilesystem(storagePath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create storage: %v", err)
|
||||
}
|
||||
|
|
@ -280,7 +280,7 @@ func defaultTestConfig(storagePath, dbPath string) *config.Config {
|
|||
return &config.Config{
|
||||
Listen: ":8080",
|
||||
BaseURL: "http://localhost:8080",
|
||||
Storage: config.StorageConfig{URL: "file://" + storagePath, MaxSize: ""},
|
||||
Storage: config.StorageConfig{Path: storagePath, MaxSize: ""},
|
||||
Database: config.DatabaseConfig{
|
||||
Driver: "sqlite",
|
||||
Path: dbPath,
|
||||
|
|
|
|||
|
|
@ -1,149 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
)
|
||||
|
||||
const gradleBuildCacheStoragePrefix = "_gradle/http-build-cache/"
|
||||
|
||||
type gradleBuildCacheLister interface {
|
||||
ListPrefix(ctx context.Context, prefix string) ([]storage.ObjectInfo, error)
|
||||
}
|
||||
|
||||
func (s *Server) startGradleBuildCacheEviction(ctx context.Context) {
|
||||
maxAge := s.cfg.ParseGradleBuildCacheMaxAge()
|
||||
maxSize := s.cfg.ParseGradleBuildCacheMaxSize()
|
||||
if maxAge <= 0 && maxSize <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
lister, ok := s.storage.(gradleBuildCacheLister)
|
||||
if !ok {
|
||||
s.logger.Warn("gradle cache eviction is enabled, but storage backend cannot list objects")
|
||||
return
|
||||
}
|
||||
|
||||
interval := s.cfg.ParseGradleBuildCacheSweepInterval()
|
||||
s.logger.Info("gradle cache eviction enabled",
|
||||
"max_age", maxAge,
|
||||
"max_size_bytes", maxSize,
|
||||
"interval", interval)
|
||||
|
||||
sweep := func() {
|
||||
deletedCount, freedBytes, err := sweepGradleBuildCache(ctx, s.storage, lister, maxAge, maxSize, time.Now())
|
||||
if err != nil {
|
||||
s.logger.Warn("gradle cache eviction sweep failed", "error", err)
|
||||
return
|
||||
}
|
||||
if deletedCount > 0 {
|
||||
s.logger.Info("gradle cache eviction sweep completed",
|
||||
"deleted_entries", deletedCount,
|
||||
"freed_bytes", freedBytes)
|
||||
}
|
||||
}
|
||||
|
||||
sweep()
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
sweep()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func sweepGradleBuildCache(
|
||||
ctx context.Context,
|
||||
store storage.Storage,
|
||||
lister gradleBuildCacheLister,
|
||||
maxAge time.Duration,
|
||||
maxSize int64,
|
||||
now time.Time,
|
||||
) (int, int64, error) {
|
||||
entries, err := lister.ListPrefix(ctx, gradleBuildCacheStoragePrefix)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("listing gradle cache entries: %w", err)
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
sortOldestFirst(entries)
|
||||
|
||||
deletedCount := 0
|
||||
freedBytes := int64(0)
|
||||
var firstDeleteErr error
|
||||
|
||||
deleteEntry := func(entry storage.ObjectInfo) bool {
|
||||
if err := store.Delete(ctx, entry.Path); err != nil {
|
||||
if firstDeleteErr == nil {
|
||||
firstDeleteErr = err
|
||||
}
|
||||
return false
|
||||
}
|
||||
deletedCount++
|
||||
freedBytes += entry.Size
|
||||
return true
|
||||
}
|
||||
|
||||
remaining := entries
|
||||
if maxAge > 0 {
|
||||
cutoff := now.Add(-maxAge)
|
||||
kept := make([]storage.ObjectInfo, 0, len(entries))
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.ModTime.IsZero() && entry.ModTime.Before(cutoff) {
|
||||
if deleteEntry(entry) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
kept = append(kept, entry)
|
||||
}
|
||||
|
||||
remaining = kept
|
||||
}
|
||||
|
||||
if maxSize > 0 {
|
||||
totalSize := int64(0)
|
||||
for _, entry := range remaining {
|
||||
totalSize += entry.Size
|
||||
}
|
||||
|
||||
for _, entry := range remaining {
|
||||
if totalSize <= maxSize {
|
||||
break
|
||||
}
|
||||
if deleteEntry(entry) {
|
||||
totalSize -= entry.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if firstDeleteErr != nil {
|
||||
return deletedCount, freedBytes, fmt.Errorf("deleting gradle cache entries: %w", firstDeleteErr)
|
||||
}
|
||||
|
||||
return deletedCount, freedBytes, nil
|
||||
}
|
||||
|
||||
func sortOldestFirst(entries []storage.ObjectInfo) {
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
if entries[i].ModTime.Equal(entries[j].ModTime) {
|
||||
return entries[i].Path < entries[j].Path
|
||||
}
|
||||
return entries[i].ModTime.Before(entries[j].ModTime)
|
||||
})
|
||||
}
|
||||
|
|
@ -1,138 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
)
|
||||
|
||||
type fakeGradleCacheStore struct {
|
||||
objects map[string]storage.ObjectInfo
|
||||
}
|
||||
|
||||
func newFakeGradleCacheStore(objects []storage.ObjectInfo) *fakeGradleCacheStore {
|
||||
m := make(map[string]storage.ObjectInfo, len(objects))
|
||||
for _, obj := range objects {
|
||||
m[obj.Path] = obj
|
||||
}
|
||||
return &fakeGradleCacheStore{objects: m}
|
||||
}
|
||||
|
||||
func (s *fakeGradleCacheStore) Store(_ context.Context, path string, r io.Reader) (int64, string, error) {
|
||||
data, _ := io.ReadAll(r)
|
||||
s.objects[path] = storage.ObjectInfo{Path: path, Size: int64(len(data)), ModTime: time.Now()}
|
||||
return int64(len(data)), "", nil
|
||||
}
|
||||
|
||||
func (s *fakeGradleCacheStore) Open(_ context.Context, path string) (io.ReadCloser, error) {
|
||||
obj, ok := s.objects[path]
|
||||
if !ok {
|
||||
return nil, storage.ErrNotFound
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(make([]byte, obj.Size))), nil
|
||||
}
|
||||
|
||||
func (s *fakeGradleCacheStore) Exists(_ context.Context, path string) (bool, error) {
|
||||
_, ok := s.objects[path]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (s *fakeGradleCacheStore) Delete(_ context.Context, path string) error {
|
||||
delete(s.objects, path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeGradleCacheStore) Size(_ context.Context, path string) (int64, error) {
|
||||
obj, ok := s.objects[path]
|
||||
if !ok {
|
||||
return 0, storage.ErrNotFound
|
||||
}
|
||||
return obj.Size, nil
|
||||
}
|
||||
|
||||
func (s *fakeGradleCacheStore) SignedURL(_ context.Context, _ string, _ time.Duration) (string, error) {
|
||||
return "", storage.ErrSignedURLUnsupported
|
||||
}
|
||||
|
||||
func (s *fakeGradleCacheStore) UsedSpace(_ context.Context) (int64, error) {
|
||||
var total int64
|
||||
for _, obj := range s.objects {
|
||||
total += obj.Size
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (s *fakeGradleCacheStore) URL() string { return "mem://" }
|
||||
|
||||
func (s *fakeGradleCacheStore) Close() error { return nil }
|
||||
|
||||
func (s *fakeGradleCacheStore) ListPrefix(_ context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||
objects := make([]storage.ObjectInfo, 0)
|
||||
for _, obj := range s.objects {
|
||||
if strings.HasPrefix(obj.Path, prefix) {
|
||||
objects = append(objects, obj)
|
||||
}
|
||||
}
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func TestSweepGradleBuildCache_MaxAge(t *testing.T) {
|
||||
now := time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC)
|
||||
store := newFakeGradleCacheStore([]storage.ObjectInfo{
|
||||
{Path: "_gradle/http-build-cache/old", Size: 10, ModTime: now.Add(-48 * time.Hour)},
|
||||
{Path: "_gradle/http-build-cache/new", Size: 10, ModTime: now.Add(-2 * time.Hour)},
|
||||
})
|
||||
|
||||
deleted, freed, err := sweepGradleBuildCache(context.Background(), store, store, 24*time.Hour, 0, now)
|
||||
if err != nil {
|
||||
t.Fatalf("sweepGradleBuildCache() error = %v", err)
|
||||
}
|
||||
if deleted != 1 {
|
||||
t.Fatalf("deleted entries = %d, want 1", deleted)
|
||||
}
|
||||
if freed != 10 {
|
||||
t.Fatalf("freed bytes = %d, want 10", freed)
|
||||
}
|
||||
|
||||
if _, ok := store.objects["_gradle/http-build-cache/old"]; ok {
|
||||
t.Fatal("old entry was not deleted")
|
||||
}
|
||||
if _, ok := store.objects["_gradle/http-build-cache/new"]; !ok {
|
||||
t.Fatal("new entry should remain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepGradleBuildCache_MaxSizeOldestFirst(t *testing.T) {
|
||||
now := time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC)
|
||||
store := newFakeGradleCacheStore([]storage.ObjectInfo{
|
||||
{Path: "_gradle/http-build-cache/a", Size: 5, ModTime: now.Add(-3 * time.Hour)},
|
||||
{Path: "_gradle/http-build-cache/b", Size: 5, ModTime: now.Add(-2 * time.Hour)},
|
||||
{Path: "_gradle/http-build-cache/c", Size: 5, ModTime: now.Add(-1 * time.Hour)},
|
||||
})
|
||||
|
||||
deleted, freed, err := sweepGradleBuildCache(context.Background(), store, store, 0, 10, now)
|
||||
if err != nil {
|
||||
t.Fatalf("sweepGradleBuildCache() error = %v", err)
|
||||
}
|
||||
if deleted != 1 {
|
||||
t.Fatalf("deleted entries = %d, want 1", deleted)
|
||||
}
|
||||
if freed != 5 {
|
||||
t.Fatalf("freed bytes = %d, want 5", freed)
|
||||
}
|
||||
|
||||
if _, ok := store.objects["_gradle/http-build-cache/a"]; ok {
|
||||
t.Fatal("oldest entry was not deleted")
|
||||
}
|
||||
if _, ok := store.objects["_gradle/http-build-cache/b"]; !ok {
|
||||
t.Fatal("middle entry should remain")
|
||||
}
|
||||
if _, ok := store.objects["_gradle/http-build-cache/c"]; !ok {
|
||||
t.Fatal("newest entry should remain")
|
||||
}
|
||||
}
|
||||
|
|
@ -1,182 +0,0 @@
|
|||
// Package server implements the proxy HTTP server.
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/metrics"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
probePathPrefix = ".healthcheck/"
|
||||
probeMarker = "proxy-healthcheck:"
|
||||
probeSuffixBytes = 8
|
||||
defaultProbeTTL = 30 * time.Second
|
||||
defaultProbeTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// HealthResponse is the JSON payload returned by /health.
|
||||
type HealthResponse struct {
|
||||
Status string `json:"status"`
|
||||
Checks map[string]HealthCheck `json:"checks"`
|
||||
}
|
||||
|
||||
// HealthCheck reports the status of a single subsystem check.
|
||||
type HealthCheck struct {
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
}
|
||||
|
||||
// probeError tags a storage probe failure with the step that failed.
|
||||
type probeError struct {
|
||||
step string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *probeError) Error() string { return e.step + ": " + e.err.Error() }
|
||||
func (e *probeError) Unwrap() error { return e.err }
|
||||
|
||||
// storageProbe runs a write → size-check → read → verify → delete round-trip
|
||||
// against the storage backend. Returns nil on success or a *probeError on failure.
|
||||
func storageProbe(ctx context.Context, s storage.Storage) (err error) {
|
||||
suffix, suffixErr := randomSuffix()
|
||||
if suffixErr != nil {
|
||||
return &probeError{step: "write", err: fmt.Errorf("generating random suffix: %w", suffixErr)}
|
||||
}
|
||||
path := probePathPrefix + strconv.FormatInt(time.Now().UnixNano(), 10) + "-" + suffix
|
||||
payload := []byte(probeMarker + suffix)
|
||||
|
||||
// 1. Store
|
||||
size, _, storeErr := s.Store(ctx, path, bytes.NewReader(payload))
|
||||
if storeErr != nil {
|
||||
return &probeError{step: "write", err: storeErr}
|
||||
}
|
||||
// After Store succeeds, always attempt to delete on the way out so probe
|
||||
// objects don't accumulate when a later step (size/open/read/verify) fails.
|
||||
// Delete is reported as the primary error only if no earlier failure
|
||||
// already set one.
|
||||
defer func() {
|
||||
if delErr := s.Delete(ctx, path); delErr != nil && err == nil {
|
||||
err = &probeError{step: "delete", err: delErr}
|
||||
}
|
||||
}()
|
||||
// 2. Size check
|
||||
if size != int64(len(payload)) {
|
||||
return &probeError{step: "size", err: fmt.Errorf("wrote %d bytes, expected %d", size, len(payload))}
|
||||
}
|
||||
// 3. Open
|
||||
rc, openErr := s.Open(ctx, path)
|
||||
if openErr != nil {
|
||||
return &probeError{step: "read", err: openErr}
|
||||
}
|
||||
// 4. Read all (classify mid-stream errors as read, not verify).
|
||||
// Close explicitly (not deferred) so the file handle is released before
|
||||
// Delete — on Windows, an open handle prevents deletion.
|
||||
data, readErr := io.ReadAll(rc)
|
||||
_ = rc.Close()
|
||||
if readErr != nil {
|
||||
return &probeError{step: "read", err: readErr}
|
||||
}
|
||||
// 5. Verify
|
||||
if !bytes.Equal(data, payload) {
|
||||
return &probeError{step: "verify", err: fmt.Errorf("content mismatch")}
|
||||
}
|
||||
// 6. Delete is handled via the deferred cleanup above.
|
||||
return nil
|
||||
}
|
||||
|
||||
// randomSuffix returns 8 cryptographically random bytes hex-encoded.
|
||||
func randomSuffix() (string, error) {
|
||||
b := make([]byte, probeSuffixBytes)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// healthCache memoizes the result of storageProbe for a configurable TTL.
|
||||
// It is safe for concurrent use.
|
||||
type healthCache struct {
|
||||
storage storage.Storage
|
||||
interval time.Duration
|
||||
probeTimeout time.Duration
|
||||
logger *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
lastAt time.Time
|
||||
lastErr error
|
||||
}
|
||||
|
||||
// newHealthCache builds a cache, parsing the interval from a duration string.
|
||||
// Empty interval string defaults to 30s. "0" or "0s" disables caching.
|
||||
func newHealthCache(s storage.Storage, intervalStr string, logger *slog.Logger) (*healthCache, error) {
|
||||
interval := defaultProbeTTL
|
||||
if intervalStr != "" {
|
||||
d, err := time.ParseDuration(intervalStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing storage_probe_interval %q: %w", intervalStr, err)
|
||||
}
|
||||
interval = d
|
||||
}
|
||||
return &healthCache{
|
||||
storage: s,
|
||||
interval: interval,
|
||||
probeTimeout: defaultProbeTimeout,
|
||||
logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Check returns the cached probe result if still fresh, otherwise runs a fresh probe.
|
||||
// The probe runs under a context derived from context.Background() with a fixed
|
||||
// timeout so that caller cancellation (e.g. client disconnect) cannot poison the
|
||||
// cache with context.Canceled.
|
||||
func (c *healthCache) Check() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
// Cache hit
|
||||
if c.interval > 0 && !c.lastAt.IsZero() && time.Since(c.lastAt) < c.interval {
|
||||
return c.lastErr
|
||||
}
|
||||
|
||||
// Fresh probe under a detached context
|
||||
probeCtx, cancel := context.WithTimeout(context.Background(), c.probeTimeout)
|
||||
defer cancel()
|
||||
err := storageProbe(probeCtx, c.storage)
|
||||
|
||||
// Transition logging and metric increment happen only on the fresh-probe path.
|
||||
c.logTransition(c.lastErr, err)
|
||||
if err != nil {
|
||||
var pe *probeError
|
||||
if errors.As(err, &pe) {
|
||||
metrics.RecordHealthProbeFailure(pe.step)
|
||||
} else {
|
||||
metrics.RecordHealthProbeFailure("unknown")
|
||||
}
|
||||
}
|
||||
|
||||
c.lastErr = err
|
||||
c.lastAt = time.Now()
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *healthCache) logTransition(prev, curr error) {
|
||||
switch {
|
||||
case prev != nil && curr == nil:
|
||||
c.logger.Info("storage probe recovered")
|
||||
case prev == nil && curr != nil:
|
||||
c.logger.Error("storage probe failed", "error", curr.Error())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,448 +0,0 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/metrics"
|
||||
"github.com/git-pkgs/proxy/internal/storage"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
|
||||
// fakeStorage is a minimal storage.Storage for probe tests with per-step failure injection.
|
||||
type fakeStorage struct {
|
||||
mu sync.Mutex
|
||||
|
||||
storeCalls atomic.Int64
|
||||
openCalls atomic.Int64
|
||||
closeCalls atomic.Int64
|
||||
deleteCalls atomic.Int64
|
||||
|
||||
paths []string
|
||||
payloads [][]byte
|
||||
|
||||
// Failure injection.
|
||||
storeErr error
|
||||
openErr error
|
||||
readErr error // returned by the io.ReadCloser.Read after partial bytes
|
||||
deleteErr error
|
||||
|
||||
// Misbehavior knobs.
|
||||
sizeDelta int64 // added to the reported size from Store
|
||||
readOverride []byte // if non-nil, Open returns a reader yielding these bytes instead of stored content
|
||||
|
||||
// storeBlock, if non-nil, causes Store to block until the channel is closed or ctx is done.
|
||||
storeBlock chan struct{}
|
||||
|
||||
stored map[string][]byte
|
||||
}
|
||||
|
||||
func newFakeStorage() *fakeStorage { return &fakeStorage{stored: map[string][]byte{}} }
|
||||
|
||||
func (f *fakeStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
|
||||
f.storeCalls.Add(1)
|
||||
if f.storeErr != nil {
|
||||
return 0, "", f.storeErr
|
||||
}
|
||||
if f.storeBlock != nil {
|
||||
select {
|
||||
case <-f.storeBlock:
|
||||
case <-ctx.Done():
|
||||
return 0, "", ctx.Err()
|
||||
}
|
||||
}
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.stored[path] = data
|
||||
f.paths = append(f.paths, path)
|
||||
f.payloads = append(f.payloads, data)
|
||||
f.mu.Unlock()
|
||||
return int64(len(data)) + f.sizeDelta, "fakehash", nil
|
||||
}
|
||||
|
||||
type fakeReadCloser struct {
|
||||
data []byte
|
||||
pos int
|
||||
readErr error
|
||||
closed *atomic.Int64
|
||||
}
|
||||
|
||||
func (rc *fakeReadCloser) Read(p []byte) (int, error) {
|
||||
if rc.pos >= len(rc.data) {
|
||||
if rc.readErr != nil {
|
||||
return 0, rc.readErr
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, rc.data[rc.pos:])
|
||||
rc.pos += n
|
||||
if rc.pos >= len(rc.data) && rc.readErr != nil {
|
||||
return n, rc.readErr
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (rc *fakeReadCloser) Close() error { rc.closed.Add(1); return nil }
|
||||
|
||||
func (f *fakeStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||
f.openCalls.Add(1)
|
||||
if f.openErr != nil {
|
||||
return nil, f.openErr
|
||||
}
|
||||
f.mu.Lock()
|
||||
data := f.stored[path]
|
||||
f.mu.Unlock()
|
||||
if f.readOverride != nil {
|
||||
data = f.readOverride
|
||||
}
|
||||
return &fakeReadCloser{data: data, readErr: f.readErr, closed: &f.closeCalls}, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorage) Exists(ctx context.Context, path string) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
_, ok := f.stored[path]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorage) Delete(ctx context.Context, path string) error {
|
||||
f.deleteCalls.Add(1)
|
||||
if f.deleteErr != nil {
|
||||
return f.deleteErr
|
||||
}
|
||||
f.mu.Lock()
|
||||
delete(f.stored, path)
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStorage) Size(ctx context.Context, path string) (int64, error) { return 0, nil }
|
||||
func (f *fakeStorage) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
|
||||
return "", storage.ErrSignedURLUnsupported
|
||||
}
|
||||
func (f *fakeStorage) UsedSpace(ctx context.Context) (int64, error) { return 0, nil }
|
||||
func (f *fakeStorage) URL() string { return "fake://" }
|
||||
func (f *fakeStorage) Close() error { return nil }
|
||||
|
||||
// --- Tests follow. First test: happy path ---
|
||||
|
||||
func TestStorageProbe_HappyPath(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
if err := storageProbe(context.Background(), fs); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := fs.storeCalls.Load(); got != 1 {
|
||||
t.Errorf("Store calls = %d, want 1", got)
|
||||
}
|
||||
if got := fs.openCalls.Load(); got != 1 {
|
||||
t.Errorf("Open calls = %d, want 1", got)
|
||||
}
|
||||
if got := fs.closeCalls.Load(); got != 1 {
|
||||
t.Errorf("Close calls = %d, want 1", got)
|
||||
}
|
||||
if got := fs.deleteCalls.Load(); got != 1 {
|
||||
t.Errorf("Delete calls = %d, want 1", got)
|
||||
}
|
||||
if len(fs.paths) != 1 || !strings.HasPrefix(fs.paths[0], ".healthcheck/") {
|
||||
t.Errorf("unexpected probe path: %v", fs.paths)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageProbe_WriteFails(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.storeErr = errors.New("disk full")
|
||||
err := storageProbe(context.Background(), fs)
|
||||
var pe *probeError
|
||||
if !errors.As(err, &pe) {
|
||||
t.Fatalf("expected *probeError, got %T: %v", err, err)
|
||||
}
|
||||
if pe.step != "write" {
|
||||
t.Errorf("step = %q, want write", pe.step)
|
||||
}
|
||||
if fs.openCalls.Load() != 0 {
|
||||
t.Errorf("Open should not be called after write failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageProbe_SizeMismatch(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.sizeDelta = -1 // Report 1 byte fewer than actually written
|
||||
err := storageProbe(context.Background(), fs)
|
||||
var pe *probeError
|
||||
if !errors.As(err, &pe) || pe.step != "size" {
|
||||
t.Fatalf("step = %v, want size; err = %v", pe, err)
|
||||
}
|
||||
if fs.openCalls.Load() != 0 {
|
||||
t.Errorf("Open should not be called after size mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageProbe_OpenFails(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.openErr = errors.New("access denied")
|
||||
err := storageProbe(context.Background(), fs)
|
||||
var pe *probeError
|
||||
if !errors.As(err, &pe) || pe.step != "read" {
|
||||
t.Fatalf("step = %v, want read; err = %v", pe, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageProbe_ReadMidStreamFails(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.readErr = errors.New("connection reset")
|
||||
err := storageProbe(context.Background(), fs)
|
||||
var pe *probeError
|
||||
if !errors.As(err, &pe) || pe.step != "read" {
|
||||
t.Fatalf("step = %v, want read (NOT verify); err = %v", pe, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageProbe_ContentMismatch(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.readOverride = []byte("wrong content")
|
||||
err := storageProbe(context.Background(), fs)
|
||||
var pe *probeError
|
||||
if !errors.As(err, &pe) || pe.step != "verify" {
|
||||
t.Fatalf("step = %v, want verify; err = %v", pe, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageProbe_DeleteFails(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.deleteErr = errors.New("permission denied")
|
||||
err := storageProbe(context.Background(), fs)
|
||||
var pe *probeError
|
||||
if !errors.As(err, &pe) || pe.step != "delete" {
|
||||
t.Fatalf("step = %v, want delete; err = %v", pe, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStorageProbe_CleanupOnNonDeleteFailure asserts that the probe object is
|
||||
// deleted even when a step after Store (size/open/read/verify) fails, so
|
||||
// probe artifacts don't accumulate in the storage backend.
|
||||
func TestStorageProbe_CleanupOnNonDeleteFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
inject func(*fakeStorage)
|
||||
wantErr string
|
||||
}{
|
||||
{"size mismatch", func(fs *fakeStorage) { fs.sizeDelta = -1 }, "size"},
|
||||
{"open fails", func(fs *fakeStorage) { fs.openErr = errors.New("open boom") }, "read"},
|
||||
{"read mid-stream", func(fs *fakeStorage) { fs.readErr = errors.New("mid-stream boom") }, "read"},
|
||||
{"content mismatch", func(fs *fakeStorage) { fs.readOverride = []byte("wrong") }, "verify"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
tc.inject(fs)
|
||||
err := storageProbe(context.Background(), fs)
|
||||
var pe *probeError
|
||||
if !errors.As(err, &pe) || pe.step != tc.wantErr {
|
||||
t.Fatalf("step = %v, want %q; err = %v", pe, tc.wantErr, err)
|
||||
}
|
||||
if got := fs.deleteCalls.Load(); got != 1 {
|
||||
t.Errorf("deleteCalls = %d, want 1 (cleanup should run on non-delete failures)", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageProbe_ReaderClosedOnReadFailure(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.readErr = errors.New("read error")
|
||||
_ = storageProbe(context.Background(), fs)
|
||||
if got := fs.closeCalls.Load(); got != fs.openCalls.Load() {
|
||||
t.Errorf("closeCalls = %d, openCalls = %d (should match)", got, fs.openCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageProbe_PathUniqueness(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
for i := 0; i < 100; i++ {
|
||||
if err := storageProbe(context.Background(), fs); err != nil {
|
||||
t.Fatalf("probe %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
seen := make(map[string]bool)
|
||||
for _, p := range fs.paths {
|
||||
if !strings.HasPrefix(p, ".healthcheck/") {
|
||||
t.Errorf("path missing prefix: %q", p)
|
||||
}
|
||||
if seen[p] {
|
||||
t.Errorf("duplicate path: %q", p)
|
||||
}
|
||||
seen[p] = true
|
||||
}
|
||||
}
|
||||
|
||||
// helper: a healthCache wired to a fakeStorage and a discard logger.
|
||||
func newTestCache(fs *fakeStorage, interval time.Duration) *healthCache {
|
||||
return &healthCache{
|
||||
storage: fs,
|
||||
interval: interval,
|
||||
probeTimeout: 5 * time.Second,
|
||||
logger: discardLogger(),
|
||||
}
|
||||
}
|
||||
|
||||
func discardLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
|
||||
func TestHealthCache_CacheHit(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
c := newTestCache(fs, 30*time.Second)
|
||||
if err := c.Check(); err != nil {
|
||||
t.Fatalf("first check: %v", err)
|
||||
}
|
||||
if err := c.Check(); err != nil {
|
||||
t.Fatalf("second check: %v", err)
|
||||
}
|
||||
if got := fs.storeCalls.Load(); got != 1 {
|
||||
t.Errorf("storeCalls = %d, want 1 (second call should be cached)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthCache_MissAfterTTL(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
c := newTestCache(fs, 10*time.Millisecond)
|
||||
_ = c.Check()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
_ = c.Check()
|
||||
if got := fs.storeCalls.Load(); got != 2 {
|
||||
t.Errorf("storeCalls = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthCache_Disabled(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
c := newTestCache(fs, 0) // interval = 0 means probe every call
|
||||
_ = c.Check()
|
||||
_ = c.Check()
|
||||
if got := fs.storeCalls.Load(); got != 2 {
|
||||
t.Errorf("storeCalls = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthCache_LastAtNotAdvancedOnHit(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
c := newTestCache(fs, 30*time.Second)
|
||||
for i := 0; i < 100; i++ {
|
||||
_ = c.Check()
|
||||
}
|
||||
if got := fs.storeCalls.Load(); got != 1 {
|
||||
t.Errorf("storeCalls = %d, want 1 across 100 hits", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthCache_ConcurrentSingleFlight(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
c := newTestCache(fs, 30*time.Second)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); _ = c.Check() }()
|
||||
}
|
||||
wg.Wait()
|
||||
if got := fs.storeCalls.Load(); got != 1 {
|
||||
t.Errorf("storeCalls = %d, want 1 with 20 concurrent callers", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthCache_FailureCounterIncrement(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.storeErr = errors.New("boom")
|
||||
c := newTestCache(fs, 30*time.Second)
|
||||
|
||||
before := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
|
||||
|
||||
// First call: fresh probe → counter +1
|
||||
_ = c.Check()
|
||||
afterFirst := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
|
||||
if afterFirst-before != 1 {
|
||||
t.Errorf("counter delta after first call = %v, want 1", afterFirst-before)
|
||||
}
|
||||
|
||||
// Second call: cache hit → counter NOT re-incremented
|
||||
_ = c.Check()
|
||||
afterSecond := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
|
||||
if afterSecond != afterFirst {
|
||||
t.Errorf("counter changed on cache hit: %v → %v", afterFirst, afterSecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthCache_ProbeTimeout(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
fs.storeBlock = make(chan struct{}) // Store will block until channel is closed (or never)
|
||||
t.Cleanup(func() { close(fs.storeBlock) })
|
||||
|
||||
c := &healthCache{
|
||||
storage: fs,
|
||||
interval: 30 * time.Second,
|
||||
probeTimeout: 50 * time.Millisecond,
|
||||
logger: discardLogger(),
|
||||
}
|
||||
start := time.Now()
|
||||
err := c.Check()
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout error, got nil")
|
||||
}
|
||||
if elapsed > 500*time.Millisecond {
|
||||
t.Errorf("probe took %v, expected ~50ms (timeout not respected)", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthCache_TransitionLogging(t *testing.T) {
|
||||
fs := newFakeStorage()
|
||||
var buf bytes.Buffer
|
||||
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
c := &healthCache{
|
||||
storage: fs,
|
||||
interval: 0, // probe every call
|
||||
probeTimeout: 5 * time.Second,
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
// Steady ok state — should not log
|
||||
_ = c.Check()
|
||||
_ = c.Check()
|
||||
if got := strings.Count(buf.String(), "storage probe"); got != 0 {
|
||||
t.Errorf("steady-state logs = %d, want 0; output: %s", got, buf.String())
|
||||
}
|
||||
|
||||
// ok → err transition: exactly one Error log
|
||||
buf.Reset()
|
||||
fs.storeErr = errors.New("boom")
|
||||
_ = c.Check()
|
||||
if !strings.Contains(buf.String(), "storage probe failed") {
|
||||
t.Errorf("missing failure log on transition; output: %s", buf.String())
|
||||
}
|
||||
|
||||
// err steady state — should not log again
|
||||
buf.Reset()
|
||||
_ = c.Check()
|
||||
if buf.Len() != 0 {
|
||||
t.Errorf("steady-err logs = %q, want empty", buf.String())
|
||||
}
|
||||
|
||||
// err → ok transition: exactly one Info log
|
||||
buf.Reset()
|
||||
fs.storeErr = nil
|
||||
_ = c.Check()
|
||||
if !strings.Contains(buf.String(), "storage probe recovered") {
|
||||
t.Errorf("missing recovery log on transition; output: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
package server
|
||||
|
||||
import "net/http"
|
||||
|
||||
// BuildInfo identifies the running proxy binary.
|
||||
type BuildInfo struct {
|
||||
Version string
|
||||
Commit string
|
||||
}
|
||||
|
||||
// Layout carries shared fields consumed by the base template. It is embedded
|
||||
// in every page data struct so templates can access canonical URL and build
|
||||
// information alongside the page's own fields.
|
||||
type Layout struct {
|
||||
BuildInfo BuildInfo
|
||||
UIBaseURL string
|
||||
CanonicalPath string
|
||||
}
|
||||
|
||||
func (s *Server) layoutFor(r *http.Request) Layout {
|
||||
return Layout{
|
||||
BuildInfo: s.buildInfo,
|
||||
UIBaseURL: s.cfg.UIBaseURL,
|
||||
CanonicalPath: r.URL.Path,
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue