Watch
1
0
Fork
You've already forked pkg-proxy
1
mirror of https://github.com/git-pkgs/proxy.git synced 2026-08-23 12:24:57 -04:00

Compare commits

..
Author SHA1 Message Date
Andrew Nesbitt
2e97e06766
Handle __unset sentinel in Composer minified metadata 2026-05-13 17:29:24 +01:00
119 changed files with 3167 additions and 10605 deletions

View file

@ -13,17 +13,18 @@ jobs:
strategy: strategy:
matrix: matrix:
os: [ubuntu-latest, macos-latest, windows-latest] os: [ubuntu-latest, macos-latest, windows-latest]
go-version: ['1.25']
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with: with:
persist-credentials: false persist-credentials: false
- name: Set up Go - name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with: with:
go-version-file: go.mod go-version: ${{ matrix.go-version }}
- name: Build - name: Build
run: go build -v ./... run: go build -v ./...
@ -34,14 +35,14 @@ jobs:
lint: lint:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with: with:
persist-credentials: false persist-credentials: false
- name: Set up Go - name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with: with:
go-version-file: go.mod go-version: '1.25'
- name: golangci-lint - name: golangci-lint
run: go tool golangci-lint run ./... run: go tool golangci-lint run ./...

View file

@ -18,23 +18,14 @@ jobs:
permissions: permissions:
packages: write packages: write
contents: read contents: read
id-token: write
steps: steps:
- name: Check out the repo - name: Check out the repo
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
with: with:
persist-credentials: false persist-credentials: false
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
with:
platforms: linux/amd64,linux/arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Log in to the Container registry - name: Log in to the Container registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
@ -42,57 +33,14 @@ jobs:
- name: Extract metadata (tags, labels) for Docker - name: Extract metadata (tags, labels) for Docker
id: meta id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf
with: with:
images: ghcr.io/${{ github.repository }} images: ghcr.io/${{ github.repository }}
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Build and push Docker image - name: Build and push Docker image
id: build uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
with: with:
context: . context: .
platforms: linux/amd64,linux/arm64
push: true push: true
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
provenance: mode=max
sbom: true
- name: Sign image by digest
env:
DIGEST: ${{ steps.build.outputs.digest }}
IMAGE: ghcr.io/${{ github.repository }}
run: |
set -euo pipefail
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
cosign sign --yes "${IMAGE}@${DIGEST}"
- name: Verify BuildKit attestations and extract SPDX predicates
env:
DIGEST: ${{ steps.build.outputs.digest }}
IMAGE: ghcr.io/${{ github.repository }}
run: |
set -euo pipefail
reference="${IMAGE}@${DIGEST}"
docker buildx imagetools inspect "$reference" --format '{{ json .Provenance }}' > provenance.json
docker buildx imagetools inspect "$reference" --format '{{ json .SBOM }}' > sbom.json
for platform in linux/amd64 linux/arm64; do
jq -e --arg p "$platform" '.[$p].SLSA | type == "object" and length > 0' \
provenance.json >/dev/null
jq -e --arg p "$platform" '.[$p].SPDX' sbom.json > "sbom-${platform//\//-}.spdx.json"
done
- name: Attest platform SBOMs by digest
env:
DIGEST: ${{ steps.build.outputs.digest }}
IMAGE: ghcr.io/${{ github.repository }}
run: |
set -euo pipefail
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
reference="${IMAGE}@${DIGEST}"
for predicate in sbom-linux-amd64.spdx.json sbom-linux-arm64.spdx.json; do
cosign attest --yes --type spdxjson --predicate "$predicate" "$reference"
done

View file

@ -14,20 +14,20 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with: with:
fetch-depth: 0 fetch-depth: 0
persist-credentials: false persist-credentials: false
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1
- name: Set up Go - name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with: with:
go-version-file: go.mod go-version-file: go.mod
cache: false cache: false
- uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 - uses: goreleaser/goreleaser-action@1a80836c5c9d9e5755a25cb59ec6f45a3b5f41a8 # v7.2.1
with: with:
version: "~> v2" version: "~> v2"
args: release --clean args: release --clean

View file

@ -12,14 +12,14 @@ jobs:
swagger: swagger:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with: with:
persist-credentials: false persist-credentials: false
- name: Set up Go - name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with: with:
go-version-file: go.mod go-version: '1.25'
- name: Install swag - name: Install swag
run: go install github.com/swaggo/swag/cmd/swag@latest run: go install github.com/swaggo/swag/cmd/swag@latest

View file

@ -21,9 +21,9 @@ jobs:
security-events: write security-events: write
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with: with:
persist-credentials: false persist-credentials: false
- name: Run zizmor - name: Run zizmor
uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3

4
.gitignore vendored
View file

@ -14,8 +14,8 @@ proxy
coverage.html coverage.html
coverage.txt coverage.txt
# Go vendor directory (repo root only; embedded UI vendor dirs are tracked) # Dependency directories
/vendor/ vendor/
# Go workspace file # Go workspace file
go.work go.work

View file

@ -39,7 +39,7 @@ proxy/
│ │ └── queries.go # CRUD operations │ │ └── queries.go # CRUD operations
│ ├── storage/ # Artifact file storage │ ├── storage/ # Artifact file storage
│ │ ├── storage.go # Storage interface │ │ ├── storage.go # Storage interface
│ │ └── blob.go # gocloud.dev/blob backends (file, S3, Azure) │ │ └── filesystem.go # Local filesystem impl
│ ├── upstream/ # Upstream registry clients │ ├── upstream/ # Upstream registry clients
│ │ ├── fetcher.go # HTTP artifact fetching │ │ ├── fetcher.go # HTTP artifact fetching
│ │ └── resolver.go # Download URL resolution │ │ └── resolver.go # Download URL resolution
@ -72,7 +72,7 @@ Key types:
### `internal/storage` ### `internal/storage`
Artifact file storage abstraction backed by `gocloud.dev/blob`. Supports local filesystem (`file://`), S3 (`s3://`), and Azure (`azblob://`) URLs. Artifact file storage abstraction. Currently implements local filesystem storage. Designed to allow future backends (S3, GCS).
Interface: Interface:
```go ```go

View file

@ -1,4 +1,4 @@
FROM --platform=$BUILDPLATFORM golang:1.26.6-alpine AS builder FROM golang:1.25-alpine AS builder
WORKDIR /src WORKDIR /src
@ -12,11 +12,10 @@ RUN go mod download
# Copy source code # Copy source code
COPY . . COPY . .
# Build the binary for the target platform # Build the binary
ARG TARGETARCH RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy
FROM alpine:3.24.1 FROM alpine:3.21
RUN apk add --no-cache ca-certificates RUN apk add --no-cache ca-certificates

150
README.md
View file

@ -38,7 +38,6 @@ Resolution order: package override, then ecosystem override, then global default
| Conan | C/C++ | | ✓ | | Conan | C/C++ | | ✓ |
| Conda | Python/R | Yes | ✓ | | Conda | Python/R | Yes | ✓ |
| CRAN | R | | ✓ | | CRAN | R | | ✓ |
| Julia | Julia | | ✓ |
| Container | Docker/OCI | | ✓ | | Container | Docker/OCI | | ✓ |
| Debian | Debian/Ubuntu | | ✓ | | Debian | Debian/Ubuntu | | ✓ |
| RPM | RHEL/Fedora | | ✓ | | RPM | RHEL/Fedora | | ✓ |
@ -210,18 +209,6 @@ Add to your `~/.m2/settings.xml`:
</settings> </settings>
``` ```
The `/maven/` endpoint uses Maven Central as primary upstream and falls back to the Gradle Plugin Portal for Gradle plugin marker metadata and related artifacts when the primary upstream returns not found.
For Gradle plugin resolution via the same proxy endpoint:
```kotlin
pluginManagement {
repositories {
maven(url = "http://localhost:8080/maven/")
}
}
```
### Gradle HTTP Build Cache ### Gradle HTTP Build Cache
Configure in `settings.gradle(.kts)`: Configure in `settings.gradle(.kts)`:
@ -325,21 +312,6 @@ local({
}) })
``` ```
### Julia
Set the Pkg server before starting Julia:
```bash
export JULIA_PKG_SERVER=http://localhost:8080/julia
```
Or inside a running session:
```julia
ENV["JULIA_PKG_SERVER"] = "http://localhost:8080/julia"
using Pkg; Pkg.update()
```
### Docker / Container Registry ### Docker / Container Registry
Configure Docker to use the proxy as a registry mirror in `/etc/docker/daemon.json`: Configure Docker to use the proxy as a registry mirror in `/etc/docker/daemon.json`:
@ -362,39 +334,6 @@ Or pull images directly:
docker pull localhost:8080/library/nginx:latest docker pull localhost:8080/library/nginx:latest
``` ```
### Helm
Configure each HTTP chart repository with a name, then add the matching proxy
URL to Helm:
```yaml
upstream:
helm:
bitnami: "https://charts.bitnami.com/bitnami"
```
```bash
helm repo add bitnami http://localhost:8080/helm/bitnami
helm repo update
helm pull bitnami/nginx
```
The proxy caches `index.yaml` using the normal metadata-cache settings and
caches chart archives after verifying their SHA-256 digest from the index.
For charts stored in an OCI registry, configure a named OCI upstream and add
the reserved `upstream/{name}` prefix to the chart reference:
```yaml
upstream:
oci:
ghcr: "https://ghcr.io"
```
```bash
helm pull oci://localhost:8080/upstream/ghcr/owner/charts/mychart --version 1.0.0 --plain-http
```
### Debian / APT ### Debian / APT
Configure APT to use the proxy in `/etc/apt/sources.list.d/proxy.list`: Configure APT to use the proxy in `/etc/apt/sources.list.d/proxy.list`:
@ -409,13 +348,6 @@ Replace your existing sources.list entries, then:
sudo apt update sudo apt update
``` ```
The upstream defaults to `http://deb.debian.org/debian`. To proxy a different APT repository (e.g. Ubuntu), set `upstream.debian` in the config file or `PROXY_UPSTREAM_DEBIAN` in the environment:
```yaml
upstream:
debian: "http://archive.ubuntu.com/ubuntu"
```
### RPM / Yum / DNF ### RPM / Yum / DNF
Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`: Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`:
@ -438,7 +370,6 @@ sudo dnf update
## Configuration ## Configuration
The proxy can be configured via: The proxy can be configured via:
1. Command line flags (highest priority) 1. Command line flags (highest priority)
2. Environment variables 2. Environment variables
3. Configuration file (YAML or JSON) 3. Configuration file (YAML or JSON)
@ -456,7 +387,6 @@ The proxy can be configured via:
-database-url string PostgreSQL connection URL -database-url string PostgreSQL connection URL
-log-level string Log level: debug, info, warn, error (default "info") -log-level string Log level: debug, info, warn, error (default "info")
-log-format string Log format: text, json (default "text") -log-format string Log format: text, json (default "text")
-access-log string Path to the JSONL access log
-version Print version and exit -version Print version and exit
``` ```
@ -465,14 +395,12 @@ The proxy can be configured via:
```bash ```bash
PROXY_LISTEN=:8080 PROXY_LISTEN=:8080
PROXY_BASE_URL=http://localhost:8080 PROXY_BASE_URL=http://localhost:8080
PROXY_UI_URL=http://localhost:8080 # Optional; defaults to PROXY_BASE_URL
PROXY_STORAGE_URL=file:///var/cache/proxy/artifacts PROXY_STORAGE_URL=file:///var/cache/proxy/artifacts
PROXY_DATABASE_DRIVER=sqlite PROXY_DATABASE_DRIVER=sqlite
PROXY_DATABASE_PATH=./cache/proxy.db PROXY_DATABASE_PATH=./cache/proxy.db
PROXY_DATABASE_URL=postgres://user:pass@localhost/proxy?sslmode=disable PROXY_DATABASE_URL=postgres://user:pass@localhost/proxy?sslmode=disable
PROXY_LOG_LEVEL=info PROXY_LOG_LEVEL=info
PROXY_LOG_FORMAT=text PROXY_LOG_FORMAT=text
PROXY_ACCESS_LOG_PATH=/var/log/proxy/access.jsonl
``` ```
### Configuration File ### Configuration File
@ -493,9 +421,6 @@ log:
level: "info" level: "info"
format: "text" format: "text"
access_log:
path: "/var/log/proxy/access.jsonl" # Optional JSONL activity log
# Optional: override upstream URLs # Optional: override upstream URLs
upstream: upstream:
npm: "https://registry.npmjs.org" npm: "https://registry.npmjs.org"
@ -652,7 +577,7 @@ Recently cached:
| Endpoint | Description | | Endpoint | Description |
|----------|-------------| |----------|-------------|
| `GET /` | Dashboard (web UI) | | `GET /` | Dashboard (web UI) |
| `GET /health` | Health check (JSON; HTTP 200 healthy, 503 unhealthy) | | `GET /health` | Health check (returns "ok" if healthy) |
| `GET /stats` | Cache statistics (JSON) | | `GET /stats` | Cache statistics (JSON) |
| `GET /metrics` | Prometheus metrics | | `GET /metrics` | Prometheus metrics |
| `GET /npm/*` | npm registry protocol | | `GET /npm/*` | npm registry protocol |
@ -668,8 +593,6 @@ Recently cached:
| `GET /conan/*` | Conan C/C++ protocol | | `GET /conan/*` | Conan C/C++ protocol |
| `GET /conda/*` | Conda/Anaconda protocol | | `GET /conda/*` | Conda/Anaconda protocol |
| `GET /cran/*` | CRAN (R) protocol | | `GET /cran/*` | CRAN (R) protocol |
| `GET /julia/*` | Julia Pkg server protocol |
| `GET /helm/{repository}/*` | HTTP Helm chart repository protocol |
| `GET /v2/*` | OCI/Docker registry protocol | | `GET /v2/*` | OCI/Docker registry protocol |
| `GET /debian/*` | Debian/APT repository protocol | | `GET /debian/*` | Debian/APT repository protocol |
| `GET /rpm/*` | RPM/Yum repository protocol | | `GET /rpm/*` | RPM/Yum repository protocol |
@ -866,16 +789,16 @@ Response:
## Web Interface ## Web Interface
The proxy serves a web UI under `/ui`. No separate frontend build is needed -- templates and assets are embedded in the binary. `GET /` redirects to `/ui/`. The UI is mounted under its own prefix so a reverse proxy can apply different access rules to it than to the package endpoints (for example, requiring auth for `PathPrefix(/ui)` while leaving `/npm`, `/pypi` etc. open to build machines). The proxy serves a web UI at the root URL. No separate frontend build is needed -- templates and assets are embedded in the binary.
- **Dashboard** (`/ui/`) -- cache stats, popular packages, recently cached artifacts, and vulnerability overview. - **Dashboard** (`/`) -- cache stats, popular packages, recently cached artifacts, and vulnerability overview.
- **Install guide** (`/ui/install`) -- per-ecosystem configuration instructions, so you don't have to look them up here. - **Install guide** (`/install`) -- per-ecosystem configuration instructions, so you don't have to look them up here.
- **Package browser** (`/ui/packages`) -- browse all cached packages with filtering by ecosystem and sorting by hits, size, name, or vulnerability count. - **Package browser** (`/packages`) -- browse all cached packages with filtering by ecosystem and sorting by hits, size, name, or vulnerability count.
- **Search** (`/ui/search?q=...`) -- search cached packages by name. - **Search** (`/search?q=...`) -- search cached packages by name.
- **Package detail** (`/ui/package/{ecosystem}/{name}`) -- metadata, license, vulnerabilities, and version list for a package. You can select two versions to compare. - **Package detail** (`/package/{ecosystem}/{name}`) -- metadata, license, vulnerabilities, and version list for a package. You can select two versions to compare.
- **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Version detail** (`/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts.
- **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Source browser** (`/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews.
- **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. - **Version diff** (`/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files.
## Monitoring ## Monitoring
@ -883,8 +806,6 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
| Metric | Type | Labels | Description | | Metric | Type | Labels | Description |
|--------|------|--------|-------------| |--------|------|--------|-------------|
| `proxy_requests_total` | counter | `ecosystem`, `status` | Proxy responses by package ecosystem and HTTP status |
| `proxy_request_duration_seconds` | histogram | `ecosystem`, `status` | Proxy request duration |
| `proxy_cache_hits_total` | counter | `ecosystem` | Cache hits | | `proxy_cache_hits_total` | counter | `ecosystem` | Cache hits |
| `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses | | `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses |
| `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts | | `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts |
@ -894,28 +815,9 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
| `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency | | `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency |
| `proxy_storage_errors_total` | counter | `operation` | Storage read/write failures | | `proxy_storage_errors_total` | counter | `operation` | Storage read/write failures |
| `proxy_active_requests` | gauge | | In-flight requests | | `proxy_active_requests` | gauge | | In-flight requests |
| `proxy_health_probe_failures_total` | counter | `step` | Storage health probe failures by failing step (`write`, `size`, `read`, `verify`, `delete`). |
Cache size and artifact count are refreshed every 60 seconds. The remaining metrics update on each request. Cache size and artifact count are refreshed every 60 seconds. The remaining metrics update on each request.
### Health Check
`/health` returns a structured JSON report of subsystem health. HTTP 200 if all checks pass; 503 if any fail.
```json
{
"status": "ok",
"checks": {
"database": {"status": "ok"},
"storage": {"status": "ok"}
}
}
```
Failing checks include an `"error"` field. Storage failures also include a `"step"` field identifying which probe step failed (`write`, `size`, `read`, `verify`, `delete`). When the database check fails, the storage entry reports `{"status": "skipped"}` so the response always carries the same key set.
Storage probe results are cached for `health.storage_probe_interval` (default 30s) to bound the cost of probing remote backends. A probe holds an internal mutex for up to 10 seconds (the hardcoded per-probe timeout), so `/health` is intended as a Kubernetes **readiness** probe rather than a liveness probe — a slow S3 round-trip should pull the pod from rotation, not restart it.
Scrape config for Prometheus: Scrape config for Prometheus:
```yaml ```yaml
@ -983,47 +885,22 @@ When running behind nginx, Apache, or another reverse proxy, set `base_url` to y
base_url: "https://proxy.example.com" base_url: "https://proxy.example.com"
``` ```
If the UI is reached on a different hostname than the package endpoints — for example, the UI exposed publicly on a domain while build machines hit a Docker network alias — set `ui_base_url` separately. `base_url` is the URL package managers and metadata rewriting use; `ui_base_url` is the URL advertised to humans visiting the web UI (canonical/`og:url` tags and the install guide banner): nginx example:
```yaml
base_url: "http://pkg-proxy:8080" # internal alias for build machines
ui_base_url: "https://proxy.example.com/ui" # public UI URL
```
When unset, `ui_base_url` defaults to `base_url`.
> **Warning:** the proxy serves the UI and package endpoints on the same listener. Setting `ui_base_url` only changes what URL the UI advertises to humans; it does not stop package endpoints from being reachable on the same hostname and port. When fronting the proxy with a public reverse proxy, restrict the public route to `PathPrefix(/ui)` (or your proxy's equivalent), otherwise `/npm`, `/pypi`, and the other package endpoints stay exposed alongside the UI.
nginx example, restricting the public host to the UI while leaving package endpoints reachable only on the internal listener:
```nginx ```nginx
server { server {
listen 443 ssl; listen 443 ssl;
server_name proxy.example.com; server_name proxy.example.com;
location /ui/ { location / {
proxy_pass http://127.0.0.1:8080; proxy_pass http://127.0.0.1:8080;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_buffering off; proxy_buffering off;
} }
location / {
return 404;
}
} }
``` ```
Traefik example using `PathPrefix(/ui)` so the public router only matches UI traffic:
```yaml
labels:
traefik.enable: "true"
traefik.http.services.pkg-proxy.loadbalancer.server.port: "8080"
traefik.http.routers.pkg-proxy.rule: "Host(`proxy.example.com`) && PathPrefix(`/ui`)"
traefik.http.routers.pkg-proxy.entrypoints: "websecure"
```
## Cache Management ## Cache Management
The proxy stores artifacts in the configured storage directory with this structure: The proxy stores artifacts in the configured storage directory with this structure:
@ -1064,8 +941,7 @@ The proxy will recreate the database on next start.
## Building from Source ## Building from Source
Requirements: Requirements:
- Go 1.25 or later
- Go (the project version is declared in `go.mod`)
```bash ```bash
git clone https://github.com/git-pkgs/proxy.git git clone https://github.com/git-pkgs/proxy.git

View file

@ -40,8 +40,6 @@
// Log level: debug, info, warn, error (default "info") // Log level: debug, info, warn, error (default "info")
// -log-format string // -log-format string
// Log format: text, json (default "text") // Log format: text, json (default "text")
// -access-log string
// Path to the JSONL access log (disabled by default)
// //
// Stats Flags: // Stats Flags:
// //
@ -74,15 +72,11 @@
// PROXY_DATABASE_URL - PostgreSQL connection URL // PROXY_DATABASE_URL - PostgreSQL connection URL
// PROXY_LOG_LEVEL - Log level // PROXY_LOG_LEVEL - Log level
// PROXY_LOG_FORMAT - Log format // PROXY_LOG_FORMAT - Log format
// PROXY_ACCESS_LOG_PATH - JSONL access log path
// PROXY_UPSTREAM_MAVEN - Maven repository upstream URL
// PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL - Gradle Plugin Portal upstream URL
// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads // PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads
// PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size // PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size
// PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction // PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction
// PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size // PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size
// PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval // PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval
// PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s")
// //
// Example: // Example:
// //
@ -187,7 +181,6 @@ func runServe() {
databaseURL := fs.String("database-url", "", "PostgreSQL connection URL") databaseURL := fs.String("database-url", "", "PostgreSQL connection URL")
logLevel := fs.String("log-level", "", "Log level: debug, info, warn, error") logLevel := fs.String("log-level", "", "Log level: debug, info, warn, error")
logFormat := fs.String("log-format", "", "Log format: text, json") logFormat := fs.String("log-format", "", "Log format: text, json")
accessLogPath := fs.String("access-log", "", "Path to the JSONL access log")
version := fs.Bool("version", false, "Print version and exit") version := fs.Bool("version", false, "Print version and exit")
fs.Usage = func() { fs.Usage = func() {
@ -205,15 +198,11 @@ func runServe() {
fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n") fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n")
fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n") fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n")
fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n") fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n")
fmt.Fprintf(os.Stderr, " PROXY_ACCESS_LOG_PATH JSONL access log path\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL Gradle Plugin Portal upstream URL\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n")
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL Gradle cache eviction sweep interval\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL Gradle cache eviction sweep interval\n")
fmt.Fprintf(os.Stderr, " PROXY_HEALTH_STORAGE_PROBE_INTERVAL Storage health probe cache interval\n")
} }
_ = fs.Parse(os.Args[1:]) _ = fs.Parse(os.Args[1:])
@ -261,9 +250,6 @@ func runServe() {
if *logFormat != "" { if *logFormat != "" {
cfg.Log.Format = *logFormat cfg.Log.Format = *logFormat
} }
if *accessLogPath != "" {
cfg.AccessLog.Path = *accessLogPath
}
// Validate configuration // Validate configuration
if err := cfg.Validate(); err != nil { if err := cfg.Validate(); err != nil {
@ -275,10 +261,7 @@ func runServe() {
logger := setupLogger(cfg.Log.Level, cfg.Log.Format) logger := setupLogger(cfg.Log.Level, cfg.Log.Format)
// Create and start server // Create and start server
srv, err := server.New(cfg, logger, server.BuildInfo{ srv, err := server.New(cfg, logger)
Version: Version,
Commit: Commit,
})
if err != nil { if err != nil {
logger.Error("failed to create server", "error", err) logger.Error("failed to create server", "error", err)
os.Exit(1) os.Exit(1)
@ -481,7 +464,6 @@ func runMirror() {
proxy := handler.NewProxy(db, store, fetcher, resolver, logger) proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
proxy.CacheMetadata = true // mirror always caches metadata proxy.CacheMetadata = true // mirror always caches metadata
proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataTTL = cfg.ParseMetadataTTL()
proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
m := mirror.New(proxy, db, store, logger, *concurrency) m := mirror.New(proxy, db, store, logger, *concurrency)

View file

@ -4,22 +4,10 @@
# Server listen address # Server listen address
listen: ":8080" listen: ":8080"
# Public URL where package endpoints are reachable. # Public URL where this proxy is accessible
# Used for rewriting package metadata URLs and shown in install guide snippets # Used for rewriting package metadata URLs
# so users know what to point their package manager at.
base_url: "http://localhost:8080" base_url: "http://localhost:8080"
# Timeout for individual upstream HTTP requests made by protocol handlers
# (metadata fetches, pass-through file requests). Uses Go duration syntax.
# Set to "0" to disable the timeout. Default: "30s".
# http_timeout: "30s"
# Public URL where the web UI is reached. Defaults to base_url when unset.
# Set this separately when the UI is served on a different hostname than the
# package endpoints — for example, the UI on a public domain behind auth while
# build machines hit a Docker network alias for the package endpoints.
# ui_base_url: "https://proxy.example.com/ui"
# Artifact storage configuration # Artifact storage configuration
storage: storage:
# Storage backend URL # Storage backend URL
@ -78,42 +66,19 @@ log:
# Log format: "text" or "json" # Log format: "text" or "json"
format: "text" format: "text"
# JSONL access log. Leave path empty to disable it.
access_log:
path: ""
# Upstream registry URLs and authentication # Upstream registry URLs and authentication
upstream: upstream:
# npm registry URL # npm registry URL
npm: "https://registry.npmjs.org" npm: "https://registry.npmjs.org"
# Maven repository URL (used by /maven endpoint)
maven: "https://repo1.maven.org/maven2"
# Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts)
gradle_plugin_portal: "https://plugins.gradle.org/m2"
# Cargo sparse index URL # Cargo sparse index URL
cargo: "https://index.crates.io" cargo: "https://index.crates.io"
# Cargo crate download URL # Cargo crate download URL
cargo_download: "https://static.crates.io/crates" cargo_download: "https://static.crates.io/crates"
# Debian/APT repository URL (used by /debian endpoint)
debian: "http://deb.debian.org/debian"
# Named HTTP Helm chart repositories (used by /helm/{name}/)
# helm:
# bitnami: "https://charts.bitnami.com/bitnami"
# Named OCI registries. Use the upstream/{name}/ repository prefix, e.g.
# oci://proxy.example.com/upstream/ghcr/owner/chart.
# oci:
# ghcr: "https://ghcr.io"
# Authentication for upstream registries # Authentication for upstream registries
# Keys are absolute URL scopes. Scheme, host, effective port, and path # Keys are URL prefixes matched against request URLs.
# segment boundaries must match; the longest matching scope wins.
# Values can reference environment variables using ${VAR_NAME} syntax. # Values can reference environment variables using ${VAR_NAME} syntax.
# #
# Supported auth types: # Supported auth types:
@ -163,15 +128,6 @@ gradle:
# How often eviction runs when max_age or max_size is set # How often eviction runs when max_age or max_size is set
sweep_interval: "10m" sweep_interval: "10m"
# Health endpoint configuration.
health:
# Minimum time between storage backend probes.
# The /health endpoint runs a write/read/verify/delete round-trip
# against the configured storage backend and caches the result for
# this interval. Set to "0" to probe on every request.
# Default: "30s".
storage_probe_interval: "30s"
# Version cooldown configuration # Version cooldown configuration
# Hides package versions published too recently, giving the community time # Hides package versions published too recently, giving the community time
# to spot malicious releases before they're pulled into projects. # to spot malicious releases before they're pulled into projects.
@ -185,8 +141,7 @@ cooldown:
# npm: "7d" # npm: "7d"
# cargo: "0" # cargo: "0"
# Per-package overrides (keyed by PURL). Keys are normalized, so npm scopes # Per-package overrides (keyed by PURL)
# may use either @scope or the canonical %40scope form.
# packages: # packages:
# "pkg:npm/lodash": "0" # "pkg:npm/lodash": "0"
# "pkg:npm/@babel/core": "14d" # "pkg:npm/@babel/core": "14d"

View file

@ -14,8 +14,8 @@ The proxy is a caching HTTP server that sits between package manager clients and
│ │ /npm/* -> NPMHandler /health -> healthHandler │ │ │ │ /npm/* -> NPMHandler /health -> healthHandler │ │
│ │ /cargo/* -> CargoHandler /stats -> statsHandler │ │ │ │ /cargo/* -> CargoHandler /stats -> statsHandler │ │
│ │ /gem/* -> GemHandler /metrics -> prometheus │ │ │ │ /gem/* -> GemHandler /metrics -> prometheus │ │
│ │ ...17 ecosystems /api/* -> APIHandler │ │ │ │ ...16 ecosystems /api/* -> APIHandler │ │
│ │ /ui/* -> Web UI │ │ │ │ / -> Web UI │ │
│ └──────────────────────────────────────────────────────────┘ │ │ └──────────────────────────────────────────────────────────┘ │
│ │ │ │ │ │ │ │ │ │
│ ▼ ▼ ▼ │ │ ▼ ▼ ▼ │
@ -240,8 +240,6 @@ Fetches artifacts from upstream registries.
- Exponential backoff retry on 429 (rate limit) and 5xx errors - Exponential backoff retry on 429 (rate limit) and 5xx errors
- Returns streaming reader (doesn't load into memory) - Returns streaming reader (doesn't load into memory)
- Configurable user-agent - Configurable user-agent
- Shares an authentication-aware transport with metadata requests so URL-scoped credentials apply consistently
- Discovers and caches scoped OCI Bearer tokens from registry challenges
**Resolver:** **Resolver:**
- Determines download URL for a package/version - Determines download URL for a package/version
@ -276,18 +274,18 @@ HTTP server setup, web UI, and API handlers.
- Creates and wires together all components - Creates and wires together all components
- Mounts protocol handlers at ecosystem-specific paths - Mounts protocol handlers at ecosystem-specific paths
- Middleware: request ID, real IP, logging, panic recovery, active request tracking - Middleware: request ID, real IP, logging, panic recovery, active request tracking
- Web UI under `/ui`: dashboard, package browser, source browser, version comparison - Web UI: dashboard, package browser, source browser, version comparison
- Templates are embedded in the binary via `//go:embed` - Templates are embedded in the binary via `//go:embed`
- Enrichment API for package metadata, vulnerability scanning, and outdated detection - Enrichment API for package metadata, vulnerability scanning, and outdated detection
- Health, stats, and Prometheus metrics endpoints. `/health` runs an active write → size-check → read → verify → delete probe against the storage backend and returns a structured JSON response (`HealthResponse`) with `"ok"` / `"error"` status per subsystem. Probe results are cached (default 30 s, configurable via `health.storage_probe_interval`) to avoid overwhelming remote backends. - Health, stats, and Prometheus metrics endpoints
### `internal/metrics` ### `internal/metrics`
Prometheus metrics for cache performance, upstream latency, storage operations, and active requests. See the Monitoring section of the README for the full metric list. Prometheus metrics for cache performance, upstream latency, storage operations, and active requests. See the Monitoring section of the README for the full metric list.
### Cooldown ### `internal/cooldown`
Version age filtering for supply chain attack mitigation, provided by [github.com/git-pkgs/cooldown](https://github.com/git-pkgs/cooldown). Configurable at global, ecosystem, and per-package levels. Supported by npm, PyPI, pub.dev, and Composer handlers. Version age filtering for supply chain attack mitigation. Configurable at global, ecosystem, and per-package levels. Supported by npm, PyPI, pub.dev, and Composer handlers.
### `internal/enrichment` ### `internal/enrichment`
@ -353,7 +351,6 @@ Eviction can be implemented as:
- Fresh data - new versions visible immediately - Fresh data - new versions visible immediately
- Metadata is small, upstream fetch is fast - Metadata is small, upstream fetch is fast
- Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table
- OCI manifests are the exception: they are cached automatically so previously fetched images remain pullable when the registry or token service is unavailable
**Why stream artifacts?** **Why stream artifacts?**
- Memory efficient - don't load large files into RAM - Memory efficient - don't load large files into RAM

View file

@ -17,8 +17,7 @@ See `config.example.yaml` in the repository root for a complete example.
| Config | Environment | Flag | Default | Description | | Config | Environment | Flag | Default | Description |
|--------|-------------|------|---------|-------------| |--------|-------------|------|---------|-------------|
| `listen` | `PROXY_LISTEN` | `-listen` | `:8080` | Address to listen on | | `listen` | `PROXY_LISTEN` | `-listen` | `:8080` | Address to listen on |
| `base_url` | `PROXY_BASE_URL` | `-base-url` | `http://localhost:8080` | Public URL package managers use to reach this proxy | | `base_url` | `PROXY_BASE_URL` | `-base-url` | `http://localhost:8080` | Public URL for the proxy |
| `ui_base_url` | `PROXY_UI_URL` | - | (defaults to `base_url`) | Public URL where the web UI is reached. Set separately when the UI lives behind a different hostname than package endpoints (e.g. public domain vs Docker network alias). Used for canonical/og:url tags and the install guide banner. The proxy still serves package endpoints on the same listener, so any reverse proxy fronting the UI publicly should restrict the public route to `PathPrefix(/ui)` to avoid exposing package endpoints. |
## Storage ## Storage
@ -108,30 +107,6 @@ log:
| `log.level` | `PROXY_LOG_LEVEL` | `-log-level` | `debug`, `info`, `warn`, `error` | | `log.level` | `PROXY_LOG_LEVEL` | `-log-level` | `debug`, `info`, `warn`, `error` |
| `log.format` | `PROXY_LOG_FORMAT` | `-log-format` | `text`, `json` | | `log.format` | `PROXY_LOG_FORMAT` | `-log-format` | `text`, `json` |
## Access Log
The optional access log records client requests and each HTTP exchange with an upstream registry. It is always written as JSONL, with one JSON object per line. Records for the same client request share a `request_id`.
```yaml
access_log:
path: "/var/log/proxy/access.jsonl"
```
| Config | Environment | Flag | Description |
|--------|-------------|------|-------------|
| `access_log.path` | `PROXY_ACCESS_LOG_PATH` | `-access-log` | File to append JSONL records to; empty disables the log |
The parent directory must exist and be writable when the proxy starts. A newly created log file is readable and writable only by the proxy process owner.
A request that receives a rate limit response from an upstream can produce records like these:
```json
{"time":"2026-08-16T12:00:00Z","event":"upstream","request_id":"host/example-000001","method":"GET","url":"https://registry.example/packages/example","status_code":429,"duration_ms":42}
{"time":"2026-08-16T12:00:00Z","event":"request","request_id":"host/example-000001","method":"GET","path":"/npm/example","status_code":502,"duration_ms":43,"remote_addr":"192.0.2.10:41234"}
```
Upstream retries and OCI authentication calls are separate `upstream` records, so the log preserves every status returned over the wire. Network failures have an `error` field and no `status_code`. URL credentials, query strings, and fragments are omitted from both upstream URLs and client paths.
## Upstream Registries ## Upstream Registries
Override default upstream registry URLs: Override default upstream registry URLs:
@ -139,37 +114,13 @@ Override default upstream registry URLs:
```yaml ```yaml
upstream: upstream:
npm: "https://registry.npmjs.org" npm: "https://registry.npmjs.org"
maven: "https://repo1.maven.org/maven2"
gradle_plugin_portal: "https://plugins.gradle.org/m2"
cargo: "https://index.crates.io" cargo: "https://index.crates.io"
cargo_download: "https://static.crates.io/crates" cargo_download: "https://static.crates.io/crates"
# Named HTTP Helm chart repositories, served at /helm/{name}/.
helm:
bitnami: "https://charts.bitnami.com/bitnami"
# Named OCI registries. Select one with the repository prefix
# upstream/{name}/, e.g. oci://proxy.example.com/upstream/ghcr/owner/chart.
oci:
ghcr: "https://ghcr.io"
``` ```
Helm HTTP repositories are read-only. The proxy fetches and rewrites each
repository's `index.yaml` so chart archives are downloaded through the proxy.
Chart archives are retained only when their SHA-256 digest matches the digest
listed in the index. Relative and absolute chart URLs are both supported.
Named OCI registries preserve the existing unprefixed Docker Hub mirror. A
reference such as `oci://proxy.example.com/upstream/ghcr/owner/chart` is sent
to the registry configured as `ghcr` with `owner/chart` as its repository.
When the proxy uses plain HTTP (for example `localhost:8080`), pass
`--plain-http` to Helm OCI commands.
## Authentication ## Authentication
Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. Configure authentication for private upstream registries. Auth is matched by URL prefix, and credentials can reference environment variables using `${VAR_NAME}` syntax.
OCI registries that return a Bearer challenge from a `/v2/{repository}/…` endpoint are handled automatically. The proxy discovers the token realm from `WWW-Authenticate`, applies any configured credentials for the token URL, and reuses the scoped token until shortly before it expires.
### Bearer Token ### Bearer Token
@ -218,7 +169,7 @@ upstream:
### URL Matching ### URL Matching
Auth keys must be absolute URLs. Matching compares the scheme, host, effective port, and path-segment prefix, preventing credentials for `registry.example.com` from being sent to a lookalike host such as `registry.example.com.evil.test`. The longest matching scope wins, so you can configure different credentials for different paths: Auth configs are matched by URL prefix. The longest matching prefix wins, so you can configure different credentials for different paths:
```yaml ```yaml
upstream: upstream:
@ -280,8 +231,6 @@ cooldown:
Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable. Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable.
Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins.
Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages.
Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata.
@ -292,8 +241,6 @@ Note: Hex cooldown requires disabling registry signature verification since the
By default the proxy fetches metadata fresh from upstream on every request. Enable `cache_metadata` to store metadata responses in the database and storage backend for offline fallback. When upstream is unreachable, the proxy serves the last cached copy. ETag-based revalidation avoids re-downloading unchanged metadata. By default the proxy fetches metadata fresh from upstream on every request. Enable `cache_metadata` to store metadata responses in the database and storage backend for offline fallback. When upstream is unreachable, the proxy serves the last cached copy. ETag-based revalidation avoids re-downloading unchanged metadata.
OCI manifests are always cached because cached image blobs cannot be pulled without their manifests. Digest-addressed manifests are immutable and served directly from cache. Tag-addressed manifests follow `metadata_ttl`, revalidate when stale, and fall back to the last cached response when the registry is unavailable.
```yaml ```yaml
cache_metadata: true cache_metadata: true
``` ```
@ -316,28 +263,6 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests
When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated.
### Metadata size limit
Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs.
```yaml
metadata_max_size: "100MB" # default
```
Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`.
## Upstream HTTP timeout
Protocol handlers use a shared HTTP client for upstream requests such as metadata fetches and pass-through file downloads. `http_timeout` sets that client's per-request timeout. Raise it if slow upstreams or large metadata responses cause `context deadline exceeded` errors.
```yaml
http_timeout: "30s" # default
```
Or via environment variable: `PROXY_HTTP_TIMEOUT=2m`.
Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout).
## Mirror API ## Mirror API
The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP: The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP:

View file

@ -15,6 +15,135 @@ const docTemplate = `{
"host": "{{.Host}}", "host": "{{.Host}}",
"basePath": "{{.BasePath}}", "basePath": "{{.BasePath}}",
"paths": { "paths": {
"/api/browse/{ecosystem}/{name}/{version}": {
"get": {
"description": "Lists files from the first cached artifact for a package version.",
"produces": [
"application/json"
],
"tags": [
"browse"
],
"summary": "List files inside a cached artifact",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Version",
"name": "version",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Directory path inside the archive",
"name": "path",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/server.BrowseListResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/api/browse/{ecosystem}/{name}/{version}/file/{filepath}": {
"get": {
"description": "Streams a single file from the cached artifact. The file path may contain slashes.",
"produces": [
"application/octet-stream"
],
"tags": [
"browse"
],
"summary": "Fetch a file inside a cached artifact",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Version",
"name": "version",
"in": "path",
"required": true
},
{
"type": "string",
"description": "File path inside the archive",
"name": "filepath",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/api/bulk": { "/api/bulk": {
"post": { "post": {
"consumes": [ "consumes": [
@ -60,6 +189,69 @@ const docTemplate = `{
} }
} }
}, },
"/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion}": {
"get": {
"description": "Returns a structured diff for two cached versions.",
"produces": [
"application/json"
],
"tags": [
"browse"
],
"summary": "Compare two cached versions",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "From version",
"name": "fromVersion",
"in": "path",
"required": true
},
{
"type": "string",
"description": "To version",
"name": "toVersion",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/api/outdated": { "/api/outdated": {
"post": { "post": {
"consumes": [ "consumes": [
@ -207,7 +399,7 @@ const docTemplate = `{
"/health": { "/health": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "text/plain"
], ],
"tags": [ "tags": [
"meta" "meta"
@ -217,13 +409,13 @@ const docTemplate = `{
"200": { "200": {
"description": "OK", "description": "OK",
"schema": { "schema": {
"$ref": "#/definitions/server.HealthResponse" "type": "string"
} }
}, },
"503": { "503": {
"description": "Service Unavailable", "description": "Service Unavailable",
"schema": { "schema": {
"$ref": "#/definitions/server.HealthResponse" "type": "string"
} }
} }
} }
@ -253,198 +445,6 @@ const docTemplate = `{
} }
} }
} }
},
"/ui/api/browse/{ecosystem}/{name}/{version}": {
"get": {
"description": "Lists files from the first cached artifact for a package version.",
"produces": [
"application/json"
],
"tags": [
"browse"
],
"summary": "List files inside a cached artifact",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Version",
"name": "version",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Directory path inside the archive",
"name": "path",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/server.BrowseListResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/ui/api/browse/{ecosystem}/{name}/{version}/file/{filepath}": {
"get": {
"description": "Streams a single file from the cached artifact. The file path may contain slashes.",
"produces": [
"application/octet-stream"
],
"tags": [
"browse"
],
"summary": "Fetch a file inside a cached artifact",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Version",
"name": "version",
"in": "path",
"required": true
},
{
"type": "string",
"description": "File path inside the archive",
"name": "filepath",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/ui/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion}": {
"get": {
"description": "Returns a structured diff for two cached versions.",
"produces": [
"application/json"
],
"tags": [
"browse"
],
"summary": "Compare two cached versions",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "From version",
"name": "fromVersion",
"in": "path",
"required": true
},
{
"type": "string",
"description": "To version",
"name": "toVersion",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
} }
}, },
"definitions": { "definitions": {
@ -515,34 +515,6 @@ const docTemplate = `{
} }
} }
}, },
"server.HealthCheck": {
"type": "object",
"properties": {
"error": {
"type": "string"
},
"status": {
"type": "string"
},
"step": {
"type": "string"
}
}
},
"server.HealthResponse": {
"type": "object",
"properties": {
"checks": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/server.HealthCheck"
}
},
"status": {
"type": "string"
}
}
},
"server.OutdatedPackage": { "server.OutdatedPackage": {
"type": "object", "type": "object",
"properties": { "properties": {

View file

@ -8,6 +8,135 @@
}, },
"basePath": "/", "basePath": "/",
"paths": { "paths": {
"/api/browse/{ecosystem}/{name}/{version}": {
"get": {
"description": "Lists files from the first cached artifact for a package version.",
"produces": [
"application/json"
],
"tags": [
"browse"
],
"summary": "List files inside a cached artifact",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Version",
"name": "version",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Directory path inside the archive",
"name": "path",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/server.BrowseListResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/api/browse/{ecosystem}/{name}/{version}/file/{filepath}": {
"get": {
"description": "Streams a single file from the cached artifact. The file path may contain slashes.",
"produces": [
"application/octet-stream"
],
"tags": [
"browse"
],
"summary": "Fetch a file inside a cached artifact",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Version",
"name": "version",
"in": "path",
"required": true
},
{
"type": "string",
"description": "File path inside the archive",
"name": "filepath",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/api/bulk": { "/api/bulk": {
"post": { "post": {
"consumes": [ "consumes": [
@ -53,6 +182,69 @@
} }
} }
}, },
"/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion}": {
"get": {
"description": "Returns a structured diff for two cached versions.",
"produces": [
"application/json"
],
"tags": [
"browse"
],
"summary": "Compare two cached versions",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "From version",
"name": "fromVersion",
"in": "path",
"required": true
},
{
"type": "string",
"description": "To version",
"name": "toVersion",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/api/outdated": { "/api/outdated": {
"post": { "post": {
"consumes": [ "consumes": [
@ -200,7 +392,7 @@
"/health": { "/health": {
"get": { "get": {
"produces": [ "produces": [
"application/json" "text/plain"
], ],
"tags": [ "tags": [
"meta" "meta"
@ -210,13 +402,13 @@
"200": { "200": {
"description": "OK", "description": "OK",
"schema": { "schema": {
"$ref": "#/definitions/server.HealthResponse" "type": "string"
} }
}, },
"503": { "503": {
"description": "Service Unavailable", "description": "Service Unavailable",
"schema": { "schema": {
"$ref": "#/definitions/server.HealthResponse" "type": "string"
} }
} }
} }
@ -246,198 +438,6 @@
} }
} }
} }
},
"/ui/api/browse/{ecosystem}/{name}/{version}": {
"get": {
"description": "Lists files from the first cached artifact for a package version.",
"produces": [
"application/json"
],
"tags": [
"browse"
],
"summary": "List files inside a cached artifact",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Version",
"name": "version",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Directory path inside the archive",
"name": "path",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/server.BrowseListResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/ui/api/browse/{ecosystem}/{name}/{version}/file/{filepath}": {
"get": {
"description": "Streams a single file from the cached artifact. The file path may contain slashes.",
"produces": [
"application/octet-stream"
],
"tags": [
"browse"
],
"summary": "Fetch a file inside a cached artifact",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Version",
"name": "version",
"in": "path",
"required": true
},
{
"type": "string",
"description": "File path inside the archive",
"name": "filepath",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "file"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
},
"/ui/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion}": {
"get": {
"description": "Returns a structured diff for two cached versions.",
"produces": [
"application/json"
],
"tags": [
"browse"
],
"summary": "Compare two cached versions",
"parameters": [
{
"type": "string",
"description": "Ecosystem",
"name": "ecosystem",
"in": "path",
"required": true
},
{
"type": "string",
"description": "Package name",
"name": "name",
"in": "path",
"required": true
},
{
"type": "string",
"description": "From version",
"name": "fromVersion",
"in": "path",
"required": true
},
{
"type": "string",
"description": "To version",
"name": "toVersion",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/server.ErrorResponse"
}
}
}
}
} }
}, },
"definitions": { "definitions": {
@ -508,34 +508,6 @@
} }
} }
}, },
"server.HealthCheck": {
"type": "object",
"properties": {
"error": {
"type": "string"
},
"status": {
"type": "string"
},
"step": {
"type": "string"
}
}
},
"server.HealthResponse": {
"type": "object",
"properties": {
"checks": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/server.HealthCheck"
}
},
"status": {
"type": "string"
}
}
},
"server.OutdatedPackage": { "server.OutdatedPackage": {
"type": "object", "type": "object",
"properties": { "properties": {

277
go.mod
View file

@ -1,141 +1,135 @@
module github.com/git-pkgs/proxy module github.com/git-pkgs/proxy
go 1.26.0 go 1.25.6
toolchain go1.26.6
require ( require (
github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.10.0
github.com/CycloneDX/cyclonedx-go v0.11.0 github.com/git-pkgs/archives v0.3.0
github.com/git-pkgs/archives v0.5.1 github.com/git-pkgs/enrichment v0.2.2
github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/purl v0.1.12
github.com/git-pkgs/enrichment v0.6.5 github.com/git-pkgs/registries v0.5.1
github.com/git-pkgs/integrity v0.1.1 github.com/git-pkgs/spdx v0.1.3
github.com/git-pkgs/magic v0.2.0 github.com/git-pkgs/vers v0.2.5
github.com/git-pkgs/purl v0.1.17 github.com/git-pkgs/vulns v0.1.5
github.com/git-pkgs/registries v0.8.1 github.com/go-chi/chi/v5 v5.2.5
github.com/git-pkgs/spdx v0.3.1
github.com/git-pkgs/vers v0.3.1
github.com/git-pkgs/vulns v0.2.2
github.com/go-chi/chi/v5 v5.3.1
github.com/jmoiron/sqlx v1.4.0 github.com/jmoiron/sqlx v1.4.0
github.com/lib/pq v1.12.3 github.com/lib/pq v1.12.3
github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_golang v1.23.2
github.com/prometheus/client_model v0.6.2 github.com/prometheus/client_model v0.6.2
github.com/spdx/tools-golang v0.5.7 github.com/spdx/tools-golang v0.5.7
github.com/swaggo/swag v1.16.6 github.com/swaggo/swag v1.16.6
gocloud.dev v0.46.0 gocloud.dev v0.45.0
golang.org/x/sync v0.22.0 golang.org/x/sync v0.20.0
google.golang.org/protobuf v1.36.12 google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.56.0 modernc.org/sqlite v1.50.0
) )
require ( require (
4d63.com/gocheckcompilerdirectives v1.4.0 // indirect 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect
4d63.com/gochecknoglobals v0.2.2 // indirect 4d63.com/gochecknoglobals v0.2.2 // indirect
charm.land/lipgloss/v2 v2.0.6 // indirect cloud.google.com/go/auth v0.18.2 // indirect
cloud.google.com/go/auth v0.21.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect
codeberg.org/chavacava/garif v0.2.0 // indirect codeberg.org/chavacava/garif v0.2.0 // indirect
codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect
dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect
dev.gaijin.team/go/exhaustruct/v5 v5.0.3 // indirect dev.gaijin.team/go/golib v0.6.0 // indirect
dev.gaijin.team/go/golib v0.8.1 // indirect
github.com/4meepo/tagalign v1.4.3 // indirect github.com/4meepo/tagalign v1.4.3 // indirect
github.com/Abirdcfly/dupword v0.1.8 // indirect github.com/Abirdcfly/dupword v0.1.7 // indirect
github.com/AdminBenni/iota-mixing v1.0.0 // indirect github.com/AdminBenni/iota-mixing v1.0.0 // indirect
github.com/AlwxSin/noinlineerr v1.0.6 // indirect github.com/AlwxSin/noinlineerr v1.0.5 // indirect
github.com/Antonboom/errname v1.1.2 // indirect github.com/Antonboom/errname v1.1.1 // indirect
github.com/Antonboom/nilnil v1.1.2 // indirect github.com/Antonboom/nilnil v1.1.1 // indirect
github.com/Antonboom/testifylint v1.6.4 // indirect github.com/Antonboom/testifylint v1.6.4 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/ClickHouse/clickhouse-go-linter v1.2.1 // indirect github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
github.com/BurntSushi/toml v1.6.0 // indirect
github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/Djarvur/go-err113 v0.1.1 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Masterminds/semver/v3 v3.5.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect
github.com/MirrexOne/unqueryvet v1.5.4 // indirect github.com/MirrexOne/unqueryvet v1.5.3 // indirect
github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/alecthomas/chroma/v2 v2.27.0 // indirect github.com/alecthomas/chroma/v2 v2.23.1 // indirect
github.com/alecthomas/go-check-sumtype v0.3.1 // indirect github.com/alecthomas/go-check-sumtype v0.3.1 // indirect
github.com/alexkohler/nakedret/v2 v2.0.6 // indirect github.com/alexkohler/nakedret/v2 v2.0.6 // indirect
github.com/alexkohler/prealloc v1.1.0 // indirect github.com/alexkohler/prealloc v1.0.2 // indirect
github.com/alfatraining/structtag v1.0.0 // indirect github.com/alfatraining/structtag v1.0.0 // indirect
github.com/alingse/asasalint v0.0.11 // indirect github.com/alingse/asasalint v0.0.11 // indirect
github.com/alingse/nilnesserr v0.2.0 // indirect github.com/alingse/nilnesserr v0.2.0 // indirect
github.com/anchore/go-struct-converter v0.1.0 // indirect github.com/anchore/go-struct-converter v0.1.0 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/ashanbrown/forbidigo/v2 v2.3.1 // indirect github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect
github.com/ashanbrown/makezero/v2 v2.2.1 // indirect github.com/ashanbrown/makezero/v2 v2.1.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.41.9 // indirect github.com/aws/aws-sdk-go-v2 v1.41.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.20 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.19 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.11 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.96.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
github.com/aws/smithy-go v1.26.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
github.com/aws/smithy-go v1.24.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/bkielbasa/cyclop v1.2.3 // indirect github.com/bkielbasa/cyclop v1.2.3 // indirect
github.com/blizzy78/varnamelen v0.8.0 // indirect github.com/blizzy78/varnamelen v0.8.0 // indirect
github.com/bombsimon/wsl/v4 v4.7.0 // indirect github.com/bombsimon/wsl/v4 v4.7.0 // indirect
github.com/bombsimon/wsl/v5 v5.9.0 // indirect github.com/bombsimon/wsl/v5 v5.6.0 // indirect
github.com/breml/bidichk v0.3.3 // indirect github.com/breml/bidichk v0.3.3 // indirect
github.com/breml/errchkjson v0.4.1 // indirect github.com/breml/errchkjson v0.4.1 // indirect
github.com/butuzov/ireturn v0.4.1 // indirect github.com/butuzov/ireturn v0.4.0 // indirect
github.com/butuzov/mirror v1.3.3 // indirect github.com/butuzov/mirror v1.3.0 // indirect
github.com/catenacyber/perfsprint v0.10.1 // indirect github.com/catenacyber/perfsprint v0.10.1 // indirect
github.com/ccojocar/zxcvbn-go v1.0.4 // indirect github.com/ccojocar/zxcvbn-go v1.0.4 // indirect
github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charithe/durationcheck v0.0.11 // indirect github.com/charithe/durationcheck v0.0.11 // indirect
github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect
github.com/charmbracelet/x/ansi v0.11.8 // indirect github.com/charmbracelet/x/ansi v0.10.1 // indirect
github.com/charmbracelet/x/term v0.2.2 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
github.com/charmbracelet/x/termios v0.1.1 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/charmbracelet/x/windows v0.2.2 // indirect
github.com/ckaznocha/intrange v0.3.1 // indirect github.com/ckaznocha/intrange v0.3.1 // indirect
github.com/clipperhouse/displaywidth v0.11.0 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
github.com/curioswitch/go-reassign v0.3.0 // indirect github.com/curioswitch/go-reassign v0.3.0 // indirect
github.com/daixiang0/gci v0.13.7 // indirect github.com/daixiang0/gci v0.13.7 // indirect
github.com/dave/dst v0.27.3 // indirect github.com/dave/dst v0.27.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect
github.com/dlclark/regexp2/v2 v2.2.1 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ecosyste-ms/ecosystems-go v0.4.0 // indirect github.com/ecosyste-ms/ecosystems-go v0.1.1 // indirect
github.com/ettle/strcase v0.2.0 // indirect github.com/ettle/strcase v0.2.0 // indirect
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
github.com/fatih/color v1.19.0 // indirect github.com/fatih/color v1.18.0 // indirect
github.com/fatih/structtag v1.2.0 // indirect github.com/fatih/structtag v1.2.0 // indirect
github.com/firefart/nonamedreturns v1.0.8 // indirect github.com/firefart/nonamedreturns v1.0.6 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fzipp/gocyclo v0.6.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect
github.com/ghostiam/protogetter v0.3.21 // indirect github.com/ghostiam/protogetter v0.3.20 // indirect
github.com/git-pkgs/packageurl-go v0.3.1 // indirect github.com/git-pkgs/packageurl-go v0.3.1 // indirect
github.com/git-pkgs/pom v0.1.5 // indirect github.com/git-pkgs/pom v0.1.4 // indirect
github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/github/go-spdx/v2 v2.6.0 // indirect
github.com/go-critic/go-critic v0.14.4 // indirect github.com/go-critic/go-critic v0.14.3 // indirect
github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect
@ -154,110 +148,109 @@ require (
github.com/gobwas/glob v0.2.3 // indirect github.com/gobwas/glob v0.2.3 // indirect
github.com/godoc-lint/godoc-lint v0.11.2 // indirect github.com/godoc-lint/godoc-lint v0.11.2 // indirect
github.com/gofrs/flock v0.13.0 // indirect github.com/gofrs/flock v0.13.0 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
github.com/golangci/asciicheck v0.5.0 // indirect github.com/golangci/asciicheck v0.5.0 // indirect
github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202 // indirect github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect
github.com/golangci/go-printf-func-name v0.1.1 // indirect github.com/golangci/go-printf-func-name v0.1.1 // indirect
github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792 // indirect github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d // indirect
github.com/golangci/golangci-lint/v2 v2.13.1 // indirect github.com/golangci/golangci-lint/v2 v2.10.1 // indirect
github.com/golangci/golines v0.15.0 // indirect github.com/golangci/golines v0.15.0 // indirect
github.com/golangci/misspell v0.8.0 // indirect github.com/golangci/misspell v0.8.0 // indirect
github.com/golangci/plugin-module-register v0.1.2 // indirect github.com/golangci/plugin-module-register v0.1.2 // indirect
github.com/golangci/revgrep v0.8.0 // indirect github.com/golangci/revgrep v0.8.0 // indirect
github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba // indirect
github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect
github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect
github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect
github.com/google/wire v0.7.0 // indirect github.com/google/wire v0.7.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.12 // indirect
github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/googleapis/gax-go/v2 v2.17.0 // indirect
github.com/gordonklaus/ineffassign v0.2.0 // indirect github.com/gordonklaus/ineffassign v0.2.0 // indirect
github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
github.com/gostaticanalysis/comment v1.5.0 // indirect github.com/gostaticanalysis/comment v1.5.0 // indirect
github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect
github.com/gostaticanalysis/nilerr v0.1.2 // indirect github.com/gostaticanalysis/nilerr v0.1.2 // indirect
github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect
github.com/hashicorp/go-version v1.9.0 // indirect github.com/hashicorp/go-version v1.8.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hexops/gotextdiff v1.0.3 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jgautheron/goconst v1.11.0 // indirect github.com/jgautheron/goconst v1.8.2 // indirect
github.com/jingyugao/rowserrcheck v1.1.1 // indirect
github.com/jjti/go-spancheck v0.6.5 // indirect github.com/jjti/go-spancheck v0.6.5 // indirect
github.com/josharian/intern v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect
github.com/julz/importas v0.2.0 // indirect github.com/julz/importas v0.2.0 // indirect
github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect
github.com/kisielk/errcheck v1.20.0 // indirect github.com/kisielk/errcheck v1.9.0 // indirect
github.com/kkHAIKE/contextcheck v1.1.6 // indirect github.com/kkHAIKE/contextcheck v1.1.6 // indirect
github.com/kulti/thelper v0.7.1 // indirect github.com/kulti/thelper v0.7.1 // indirect
github.com/kunwardeep/paralleltest v1.0.15 // indirect github.com/kunwardeep/paralleltest v1.0.15 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lasiar/canonicalheader v1.1.2 // indirect github.com/lasiar/canonicalheader v1.1.2 // indirect
github.com/ldez/exptostd v0.4.5 // indirect github.com/ldez/exptostd v0.4.5 // indirect
github.com/ldez/gomoddirectives v0.9.0 // indirect github.com/ldez/gomoddirectives v0.8.0 // indirect
github.com/ldez/grignotin v0.10.1 // indirect github.com/ldez/grignotin v0.10.1 // indirect
github.com/ldez/structtags v0.6.1 // indirect github.com/ldez/structtags v0.6.1 // indirect
github.com/ldez/tagliatelle v0.7.2 // indirect github.com/ldez/tagliatelle v0.7.2 // indirect
github.com/ldez/usetesting v0.5.0 // indirect github.com/ldez/usetesting v0.5.0 // indirect
github.com/leonklingele/grouper v1.1.2 // indirect github.com/leonklingele/grouper v1.1.2 // indirect
github.com/lucasb-eyer/go-colorful v1.4.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/macabu/inamedparam v0.2.0 // indirect github.com/macabu/inamedparam v0.2.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect
github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect
github.com/manuelarte/funcorder v0.6.0 // indirect github.com/manuelarte/funcorder v0.5.0 // indirect
github.com/maratori/testableexamples v1.0.1 // indirect github.com/maratori/testableexamples v1.0.1 // indirect
github.com/maratori/testpackage v1.1.2 // indirect github.com/maratori/testpackage v1.1.2 // indirect
github.com/matoous/godox v1.1.0 // indirect github.com/matoous/godox v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.24 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mgechev/revive v1.15.0 // indirect github.com/mgechev/revive v1.14.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moricho/tparallel v0.3.2 // indirect github.com/moricho/tparallel v0.3.2 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nakabonne/nestif v0.3.1 // indirect github.com/nakabonne/nestif v0.3.1 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/exhaustive v0.12.0 // indirect
github.com/nishanths/predeclared v0.2.2 // indirect github.com/nishanths/predeclared v0.2.2 // indirect
github.com/nunnatsa/ginkgolinter v0.24.0 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect
github.com/oapi-codegen/nullable v1.2.0 // indirect github.com/oapi-codegen/runtime v1.2.0 // indirect
github.com/oapi-codegen/runtime v1.6.0 // indirect
github.com/package-url/packageurl-go v0.1.6 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect
github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/prometheus/common v0.70.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/procfs v0.21.1 // indirect github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/quasilyte/go-ruleguard v0.4.5 // indirect github.com/quasilyte/go-ruleguard v0.4.5 // indirect
github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect
github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect
github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
github.com/raeperd/recvcheck v0.3.0 // indirect github.com/raeperd/recvcheck v0.2.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect
github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/ryancurrah/gomodguard v1.4.1 // indirect github.com/ryancurrah/gomodguard v1.4.1 // indirect
github.com/ryancurrah/gomodguard/v2 v2.1.3 // indirect github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
github.com/ryanrolds/sqlclosecheck v0.6.0 // indirect
github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
github.com/sashamelentyev/interfacebloat v1.1.0 // indirect github.com/sashamelentyev/interfacebloat v1.1.0 // indirect
github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect
github.com/securego/gosec/v2 v2.28.0 // indirect github.com/securego/gosec/v2 v2.23.0 // indirect
github.com/sirupsen/logrus v1.10.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect
github.com/sivchari/containedctx v1.0.3 // indirect github.com/sivchari/containedctx v1.0.3 // indirect
github.com/sonatard/noctx v0.5.1 // indirect github.com/sonatard/noctx v0.4.0 // indirect
github.com/sourcegraph/go-diff v0.8.0 // indirect github.com/sourcegraph/go-diff v0.7.0 // indirect
github.com/spf13/afero v1.15.0 // indirect github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/cobra v1.10.2 // indirect
@ -266,20 +259,20 @@ require (
github.com/spf13/viper v1.12.0 // indirect github.com/spf13/viper v1.12.0 // indirect
github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect
github.com/stretchr/objx v0.5.3 // indirect github.com/stretchr/objx v0.5.2 // indirect
github.com/stretchr/testify v1.12.1 // indirect github.com/stretchr/testify v1.11.1 // indirect
github.com/subosito/gotenv v1.4.1 // indirect github.com/subosito/gotenv v1.4.1 // indirect
github.com/tetafro/godot v1.5.6 // indirect github.com/tetafro/godot v1.5.4 // indirect
github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4 // indirect github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 // indirect
github.com/timonwong/loggercheck v0.11.0 // indirect github.com/timonwong/loggercheck v0.11.0 // indirect
github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect
github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
github.com/ulikunitz/xz v0.5.16 // indirect github.com/ulikunitz/xz v0.5.15 // indirect
github.com/ultraware/funlen v0.2.0 // indirect github.com/ultraware/funlen v0.2.0 // indirect
github.com/ultraware/whitespace v0.2.0 // indirect github.com/ultraware/whitespace v0.2.0 // indirect
github.com/urfave/cli/v2 v2.3.0 // indirect github.com/urfave/cli/v2 v2.3.0 // indirect
github.com/uudashr/gocognit v1.2.1 // indirect github.com/uudashr/gocognit v1.2.0 // indirect
github.com/uudashr/iface v1.5.0 // indirect github.com/uudashr/iface v1.4.1 // indirect
github.com/xen0n/gosmopolitan v1.3.0 // indirect github.com/xen0n/gosmopolitan v1.3.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/yagipy/maintidx v1.0.0 // indirect github.com/yagipy/maintidx v1.0.0 // indirect
@ -287,40 +280,40 @@ require (
github.com/ykadowak/zerologlint v0.1.5 // indirect github.com/ykadowak/zerologlint v0.1.5 // indirect
gitlab.com/bosi/decorder v0.4.2 // indirect gitlab.com/bosi/decorder v0.4.2 // indirect
go-simpler.org/musttag v0.14.0 // indirect go-simpler.org/musttag v0.14.0 // indirect
go-simpler.org/sloglint v0.12.0 // indirect go-simpler.org/sloglint v0.11.1 // indirect
go.augendre.info/arangolint v0.4.0 // indirect go.augendre.info/arangolint v0.4.0 // indirect
go.augendre.info/fatcontext v0.10.0 // indirect go.augendre.info/fatcontext v0.9.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel v1.41.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.41.0 // indirect
go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.41.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect go.uber.org/zap v1.27.1 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.55.0 // indirect golang.org/x/crypto v0.48.0 // indirect
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect
golang.org/x/mod v0.40.0 // indirect golang.org/x/mod v0.33.0 // indirect
golang.org/x/net v0.58.0 // indirect golang.org/x/net v0.51.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect
golang.org/x/sys v0.47.0 // indirect golang.org/x/sys v0.42.0 // indirect
golang.org/x/text v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.49.0 // indirect golang.org/x/tools v0.42.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.288.0 // indirect google.golang.org/api v0.269.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
google.golang.org/grpc v1.82.1 // indirect google.golang.org/grpc v1.79.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
honnef.co/go/tools v0.8.0 // indirect honnef.co/go/tools v0.7.0 // indirect
modernc.org/libc v1.74.4 // indirect modernc.org/libc v1.72.0 // indirect
modernc.org/mathutil v1.7.1 // indirect modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect modernc.org/memory v1.11.0 // indirect
mvdan.cc/gofumpt v0.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect
mvdan.cc/unparam v0.0.0-20260818115549-3f964bcb5673 // indirect mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect
) )

680
go.sum

File diff suppressed because it is too large Load diff

View file

@ -1,109 +0,0 @@
// Package accesslog writes proxy activity as JSON Lines.
package accesslog
import (
"context"
"encoding/json"
"fmt"
"net/url"
"os"
"sync"
"time"
)
const (
accessLogFileMode os.FileMode = 0o600
// EventRequest identifies the response sent by the proxy to a client.
EventRequest = "request"
// EventUpstream identifies one HTTP exchange with an upstream service.
EventUpstream = "upstream"
)
type requestIDKey struct{}
// Entry is one proxy activity record.
type Entry struct {
Time time.Time `json:"time"`
Event string `json:"event"`
RequestID string `json:"request_id,omitempty"`
Method string `json:"method"`
Path string `json:"path,omitempty"`
URL string `json:"url,omitempty"`
StatusCode int `json:"status_code,omitempty"`
DurationMS int64 `json:"duration_ms"`
RemoteAddr string `json:"remote_addr,omitempty"`
Error string `json:"error,omitempty"`
}
// Logger appends complete JSON objects to a file, one per line.
type Logger struct {
mu sync.Mutex
file *os.File
encoder *json.Encoder
}
// Open opens path for append, creating it with owner-only permissions when needed.
func Open(path string) (*Logger, error) {
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, accessLogFileMode)
if err != nil {
return nil, fmt.Errorf("opening access log: %w", err)
}
return &Logger{
file: file,
encoder: json.NewEncoder(file),
}, nil
}
// Write appends an entry to the log.
func (l *Logger) Write(entry Entry) error {
if entry.Time.IsZero() {
entry.Time = time.Now().UTC()
}
l.mu.Lock()
defer l.mu.Unlock()
if err := l.encoder.Encode(entry); err != nil {
return fmt.Errorf("writing access log: %w", err)
}
return nil
}
// Close closes the log file after any active writer finishes.
func (l *Logger) Close() error {
l.mu.Lock()
defer l.mu.Unlock()
if err := l.file.Close(); err != nil {
return fmt.Errorf("closing access log: %w", err)
}
return nil
}
// WithRequestID stores a proxy request ID in ctx.
func WithRequestID(ctx context.Context, requestID string) context.Context {
return context.WithValue(ctx, requestIDKey{}, requestID)
}
// RequestID returns the proxy request ID stored in ctx.
func RequestID(ctx context.Context) string {
requestID, _ := ctx.Value(requestIDKey{}).(string)
return requestID
}
// URLWithoutSecrets returns a URL without user information, query values, or fragments.
func URLWithoutSecrets(value *url.URL) string {
if value == nil {
return ""
}
clean := *value
clean.User = nil
clean.RawQuery = ""
clean.ForceQuery = false
clean.Fragment = ""
clean.RawFragment = ""
return clean.String()
}

View file

@ -1,91 +0,0 @@
package accesslog
import (
"bufio"
"context"
"encoding/json"
"net/url"
"os"
"path/filepath"
"sync"
"testing"
)
func TestLoggerWritesJSONLines(t *testing.T) {
path := filepath.Join(t.TempDir(), "access.jsonl")
logger, err := Open(path)
if err != nil {
t.Fatal(err)
}
const entries = 20
var wg sync.WaitGroup
for range entries {
wg.Add(1)
go func() {
defer wg.Done()
if err := logger.Write(Entry{
Event: EventUpstream,
RequestID: "request-id",
Method: "GET",
URL: "https://registry.example/packages/example",
StatusCode: 429,
}); err != nil {
t.Errorf("Write: %v", err)
}
}()
}
wg.Wait()
if err := logger.Close(); err != nil {
t.Fatal(err)
}
file, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = file.Close() }()
scanner := bufio.NewScanner(file)
count := 0
for scanner.Scan() {
var entry Entry
if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
t.Fatalf("line %d is not JSON: %v", count+1, err)
}
if entry.Time.IsZero() {
t.Errorf("line %d has no time", count+1)
}
if entry.StatusCode != 429 {
t.Errorf("line %d status_code = %d, want 429", count+1, entry.StatusCode)
}
count++
}
if err := scanner.Err(); err != nil {
t.Fatal(err)
}
if count != entries {
t.Errorf("lines = %d, want %d", count, entries)
}
}
func TestRequestID(t *testing.T) {
ctx := WithRequestID(context.Background(), "abc-123")
if got := RequestID(ctx); got != "abc-123" {
t.Errorf("RequestID = %q, want %q", got, "abc-123")
}
}
func TestURLWithoutSecrets(t *testing.T) {
value, err := url.Parse("https://user:password@registry.example/package.tgz?token=secret#fragment")
if err != nil {
t.Fatal(err)
}
got := URLWithoutSecrets(value)
want := "https://registry.example/package.tgz"
if got != want {
t.Errorf("URLWithoutSecrets = %q, want %q", got, want)
}
}

View file

@ -54,12 +54,10 @@ import (
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@ -68,20 +66,11 @@ type Config struct {
// Listen is the address to listen on (e.g., ":8080", "127.0.0.1:8080"). // Listen is the address to listen on (e.g., ":8080", "127.0.0.1:8080").
Listen string `json:"listen" yaml:"listen"` Listen string `json:"listen" yaml:"listen"`
// BaseURL is the public URL where package endpoints are reachable. // BaseURL is the public URL where this proxy is accessible.
// Used for rewriting package metadata URLs and shown to humans on the // Used for rewriting package metadata URLs.
// install guide so they know what to point their package manager at.
// Example: "https://proxy.example.com" or "http://localhost:8080" // Example: "https://proxy.example.com" or "http://localhost:8080"
BaseURL string `json:"base_url" yaml:"base_url"` BaseURL string `json:"base_url" yaml:"base_url"`
// UIBaseURL is the public URL where the web UI is reachable. Defaults to
// BaseURL when unset. Set this separately when the UI is served on a
// different hostname than the package endpoints — for example, the UI on a
// public domain behind auth while build machines hit a Docker network alias
// for the package endpoints.
// Example: "https://proxy.example.com/ui"
UIBaseURL string `json:"ui_base_url" yaml:"ui_base_url"`
// Storage configures artifact storage. // Storage configures artifact storage.
Storage StorageConfig `json:"storage" yaml:"storage"` Storage StorageConfig `json:"storage" yaml:"storage"`
@ -91,9 +80,6 @@ type Config struct {
// Log configures logging. // Log configures logging.
Log LogConfig `json:"log" yaml:"log"` Log LogConfig `json:"log" yaml:"log"`
// AccessLog configures the JSONL activity log.
AccessLog AccessLogConfig `json:"access_log" yaml:"access_log"`
// Upstream configures upstream registry URLs (optional overrides). // Upstream configures upstream registry URLs (optional overrides).
Upstream UpstreamConfig `json:"upstream" yaml:"upstream"` Upstream UpstreamConfig `json:"upstream" yaml:"upstream"`
@ -110,26 +96,12 @@ type Config struct {
// Default: "5m". Set to "0" to always revalidate. // Default: "5m". Set to "0" to always revalidate.
MetadataTTL string `json:"metadata_ttl" yaml:"metadata_ttl"` MetadataTTL string `json:"metadata_ttl" yaml:"metadata_ttl"`
// MetadataMaxSize is the maximum size of an upstream metadata response
// the proxy will buffer (e.g. "100MB", "250MB"). Responses over this
// size return ErrMetadataTooLarge. Default: "100MB".
MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
// HTTPTimeout is the timeout for individual upstream HTTP requests made
// by protocol handlers (metadata fetches, pass-through file requests).
// Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s".
// Set to "0" to disable the timeout entirely.
HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"`
// MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP. // MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP.
// Disabled by default to prevent unauthenticated users from triggering downloads. // Disabled by default to prevent unauthenticated users from triggering downloads.
MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
// Gradle configures Gradle HttpBuildCache behavior. // Gradle configures Gradle HttpBuildCache behavior.
Gradle GradleConfig `json:"gradle" yaml:"gradle"` Gradle GradleConfig `json:"gradle" yaml:"gradle"`
// Health configures the /health endpoint behavior.
Health HealthConfig `json:"health" yaml:"health"`
} }
// CooldownConfig configures version cooldown periods. // CooldownConfig configures version cooldown periods.
@ -142,38 +114,9 @@ type CooldownConfig struct {
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"` Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
// Packages overrides the cooldown for specific packages (keyed by PURL). // Packages overrides the cooldown for specific packages (keyed by PURL).
// Valid PURL keys are normalized to canonical form before use.
Packages map[string]string `json:"packages" yaml:"packages"` Packages map[string]string `json:"packages" yaml:"packages"`
} }
// NormalizedPackages returns a copy of the package overrides with valid PURL
// keys in canonical form. An explicitly canonical key wins over an equivalent
// noncanonical key, and invalid keys are preserved unchanged.
func (c *CooldownConfig) NormalizedPackages() map[string]string {
if c == nil || c.Packages == nil {
return nil
}
keys := make([]string, 0, len(c.Packages))
for key := range c.Packages {
keys = append(keys, key)
}
sort.Strings(keys)
normalized := make(map[string]string, len(c.Packages))
for _, key := range keys {
canonical := key
if parsed, err := purl.Parse(key); err == nil {
canonical = parsed.String()
}
if _, exists := normalized[canonical]; exists && key != canonical {
continue
}
normalized[canonical] = c.Packages[key]
}
return normalized
}
// StorageConfig configures artifact storage. // StorageConfig configures artifact storage.
type StorageConfig struct { type StorageConfig struct {
// URL is the storage backend URL. // URL is the storage backend URL.
@ -239,14 +182,6 @@ type GradleBuildCacheConfig struct {
SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"` SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"`
} }
// HealthConfig configures the /health endpoint.
type HealthConfig struct {
// StorageProbeInterval is the minimum time between storage backend probes.
// Uses Go duration syntax (e.g. "30s", "1m"). Default: "30s".
// Set to "0" to probe on every /health request (useful for low-traffic deployments).
StorageProbeInterval string `json:"storage_probe_interval" yaml:"storage_probe_interval"`
}
// DatabaseConfig configures the cache database. // DatabaseConfig configures the cache database.
type DatabaseConfig struct { type DatabaseConfig struct {
// Driver is the database driver: "sqlite" or "postgres". // Driver is the database driver: "sqlite" or "postgres".
@ -259,21 +194,6 @@ type DatabaseConfig struct {
URL string `json:"url" yaml:"url"` URL string `json:"url" yaml:"url"`
} }
// String returns a human-readable description of the configured database
// suitable for logging. For postgres the password in the connection URL is
// redacted; if the URL cannot be parsed only the driver name is returned to
// avoid leaking credentials.
func (d DatabaseConfig) String() string {
if d.Driver == "postgres" {
u, err := url.Parse(d.URL)
if err != nil || u.Host == "" {
return "postgres"
}
return u.Redacted()
}
return d.Path
}
// LogConfig configures logging. // LogConfig configures logging.
type LogConfig struct { type LogConfig struct {
// Level is the minimum log level: "debug", "info", "warn", "error". // Level is the minimum log level: "debug", "info", "warn", "error".
@ -283,12 +203,6 @@ type LogConfig struct {
Format string `json:"format" yaml:"format"` Format string `json:"format" yaml:"format"`
} }
// AccessLogConfig configures the JSONL activity log.
type AccessLogConfig struct {
// Path is the file to append activity records to. Empty disables the access log.
Path string `json:"path" yaml:"path"`
}
// UpstreamConfig configures upstream registry URLs and authentication. // UpstreamConfig configures upstream registry URLs and authentication.
// Leave empty to use defaults. // Leave empty to use defaults.
type UpstreamConfig struct { type UpstreamConfig struct {
@ -296,15 +210,6 @@ type UpstreamConfig struct {
// Default: https://registry.npmjs.org // Default: https://registry.npmjs.org
NPM string `json:"npm" yaml:"npm"` NPM string `json:"npm" yaml:"npm"`
// Maven is the upstream Maven repository URL.
// Default: https://repo1.maven.org/maven2
Maven string `json:"maven" yaml:"maven"`
// GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL.
// Used to resolve Gradle plugin marker artifacts.
// Default: https://plugins.gradle.org/m2
GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"`
// Cargo is the upstream cargo index URL. // Cargo is the upstream cargo index URL.
// Default: https://index.crates.io // Default: https://index.crates.io
Cargo string `json:"cargo" yaml:"cargo"` Cargo string `json:"cargo" yaml:"cargo"`
@ -313,45 +218,24 @@ type UpstreamConfig struct {
// Default: https://static.crates.io/crates // Default: https://static.crates.io/crates
CargoDownload string `json:"cargo_download" yaml:"cargo_download"` CargoDownload string `json:"cargo_download" yaml:"cargo_download"`
// Debian is the upstream APT repository base URL.
// Example: http://archive.ubuntu.com/ubuntu would get Ubuntu.
// Default: http://deb.debian.org/debian
Debian string `json:"debian" yaml:"debian"`
// Helm maps repository names to HTTP Helm chart repository URLs.
// Requests use /helm/{name}/index.yaml and chart URLs in the index are
// rewritten to the same named proxy endpoint.
Helm map[string]string `json:"helm" yaml:"helm"`
// OCI maps names to OCI registry URLs. Requests to a named registry use
// the repository prefix upstream/{name}/, for example
// oci://proxy.example.com/upstream/ghcr/owner/chart.
OCI map[string]string `json:"oci" yaml:"oci"`
// Auth configures authentication for upstream registries. // Auth configures authentication for upstream registries.
// Keys are absolute URL scopes matched by scheme, host, effective port, // Keys are URL prefixes that are matched against request URLs.
// and path-segment prefix.
// Example: "https://npm.pkg.github.com" matches all requests to that host. // Example: "https://npm.pkg.github.com" matches all requests to that host.
Auth map[string]AuthConfig `json:"auth" yaml:"auth"` Auth map[string]AuthConfig `json:"auth" yaml:"auth"`
} }
// AuthForURL returns the auth config that matches the given URL. // AuthForURL returns the auth config that matches the given URL.
// The longest matching URL scope wins. // Matches are based on URL prefix - the longest matching prefix wins.
func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig { func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig {
if u.Auth == nil { if u.Auth == nil {
return nil return nil
} }
target, err := parseAuthURL(url)
if err != nil {
return nil
}
var bestMatch *AuthConfig var bestMatch *AuthConfig
var bestLen int var bestLen int
for pattern, auth := range u.Auth { for pattern, auth := range u.Auth {
configured, err := parseAuthURL(pattern) if strings.HasPrefix(url, pattern) && len(pattern) > bestLen {
if err == nil && authURLMatches(configured, target) && len(pattern) > bestLen {
a := auth // copy to avoid loop variable capture a := auth // copy to avoid loop variable capture
bestMatch = &a bestMatch = &a
bestLen = len(pattern) bestLen = len(pattern)
@ -361,73 +245,6 @@ func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig {
return bestMatch return bestMatch
} }
// Validate checks upstream authentication URL scopes.
func (u *UpstreamConfig) Validate() error {
for pattern := range u.Auth {
if _, err := parseAuthURL(pattern); err != nil {
return fmt.Errorf("invalid upstream.auth URL %q: %w", pattern, err)
}
}
if err := validateNamedUpstreams("upstream.helm", u.Helm); err != nil {
return err
}
if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil {
return err
}
return nil
}
func validateNamedUpstreams(field string, upstreams map[string]string) error {
for name, upstreamURL := range upstreams {
if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\\`) {
return fmt.Errorf("invalid %s name %q", field, name)
}
if err := validateAbsoluteURL(field+"."+name, upstreamURL); err != nil {
return err
}
}
return nil
}
func parseAuthURL(value string) (*url.URL, error) {
parsed, err := url.Parse(value)
if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" {
return nil, fmt.Errorf("invalid authentication URL")
}
return parsed, nil
}
func authURLMatches(configured, target *url.URL) bool {
if !strings.EqualFold(configured.Scheme, target.Scheme) ||
!strings.EqualFold(configured.Hostname(), target.Hostname()) ||
authURLPort(configured) != authURLPort(target) {
return false
}
if configured.RawQuery != "" && configured.RawQuery != target.RawQuery {
return false
}
configuredPath := strings.TrimSuffix(configured.EscapedPath(), "/")
if configuredPath == "" {
return true
}
targetPath := strings.TrimSuffix(target.EscapedPath(), "/")
return targetPath == configuredPath || strings.HasPrefix(targetPath, configuredPath+"/")
}
func authURLPort(value *url.URL) string {
if port := value.Port(); port != "" {
return port
}
if strings.EqualFold(value.Scheme, "https") {
return "443"
}
if strings.EqualFold(value.Scheme, "http") {
return "80"
}
return ""
}
// AuthConfig configures authentication for an upstream registry. // AuthConfig configures authentication for an upstream registry.
type AuthConfig struct { type AuthConfig struct {
// Type is the authentication type: "bearer", "basic", or "header". // Type is the authentication type: "bearer", "basic", or "header".
@ -470,12 +287,9 @@ func Default() *Config {
Format: "text", Format: "text",
}, },
Upstream: UpstreamConfig{ Upstream: UpstreamConfig{
NPM: "https://registry.npmjs.org", NPM: "https://registry.npmjs.org",
Maven: "https://repo1.maven.org/maven2", Cargo: "https://index.crates.io",
GradlePluginPortal: "https://plugins.gradle.org/m2", CargoDownload: "https://static.crates.io/crates",
Cargo: "https://index.crates.io",
CargoDownload: "https://static.crates.io/crates",
Debian: "http://deb.debian.org/debian",
}, },
Gradle: GradleConfig{ Gradle: GradleConfig{
BuildCache: GradleBuildCacheConfig{ BuildCache: GradleBuildCacheConfig{
@ -520,74 +334,82 @@ func Load(path string) (*Config, error) {
return cfg, nil return cfg, nil
} }
// setEnvString sets *dst from the named environment variable, leaving it
// untouched if the variable is unset or empty.
func setEnvString(dst *string, key string) {
if v := os.Getenv(key); v != "" {
*dst = v
}
}
// setEnvBool is setEnvString for boolean fields, parsed via envBool.
func setEnvBool(dst *bool, key string) {
if v := os.Getenv(key); v != "" {
*dst = envBool(v)
}
}
// LoadFromEnv applies environment variable overrides to a Config. // LoadFromEnv applies environment variable overrides to a Config.
// Environment variables use the PROXY_ prefix: // Environment variables use the PROXY_ prefix:
// - PROXY_LISTEN // - PROXY_LISTEN
// - PROXY_BASE_URL // - PROXY_BASE_URL
// - PROXY_UI_URL
// - PROXY_STORAGE_PATH // - PROXY_STORAGE_PATH
// - PROXY_STORAGE_MAX_SIZE // - PROXY_STORAGE_MAX_SIZE
// - PROXY_DATABASE_PATH // - PROXY_DATABASE_PATH
// - PROXY_LOG_LEVEL // - PROXY_LOG_LEVEL
// - PROXY_LOG_FORMAT // - PROXY_LOG_FORMAT
// - PROXY_ACCESS_LOG_PATH
// - PROXY_HEALTH_STORAGE_PROBE_INTERVAL
func (c *Config) LoadFromEnv() { func (c *Config) LoadFromEnv() {
setEnvString(&c.Listen, "PROXY_LISTEN") if v := os.Getenv("PROXY_LISTEN"); v != "" {
setEnvString(&c.BaseURL, "PROXY_BASE_URL") c.Listen = v
setEnvString(&c.UIBaseURL, "PROXY_UI_URL") }
setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") if v := os.Getenv("PROXY_BASE_URL"); v != "" {
setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") c.BaseURL = v
setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") }
setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") if v := os.Getenv("PROXY_STORAGE_URL"); v != "" {
setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") c.Storage.URL = v
setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") }
setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") if v := os.Getenv("PROXY_STORAGE_PATH"); v != "" {
setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH") c.Storage.Path = v
setEnvString(&c.Database.URL, "PROXY_DATABASE_URL") }
setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL") if v := os.Getenv("PROXY_STORAGE_MAX_SIZE"); v != "" {
setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") c.Storage.MaxSize = v
setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH") }
setEnvString(&c.Upstream.Maven, "PROXY_UPSTREAM_MAVEN") if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE"); v != "" {
setEnvString(&c.Upstream.GradlePluginPortal, "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL") c.Storage.DirectServe = envBool(v)
setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN") }
setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT") if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_TTL"); v != "" {
setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") c.Storage.DirectServeTTL = v
setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") }
setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL"); v != "" {
setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") c.Storage.DirectServeBaseURL = v
setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") }
setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY") if v := os.Getenv("PROXY_DATABASE_DRIVER"); v != "" {
setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE") c.Database.Driver = v
setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE") }
setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE") if v := os.Getenv("PROXY_DATABASE_PATH"); v != "" {
setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL") c.Database.Path = v
setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL") }
} if v := os.Getenv("PROXY_DATABASE_URL"); v != "" {
c.Database.URL = v
// validateAbsoluteURL returns an error if value is not a parseable URL with }
// both a scheme and host. fieldName is used in the error message. if v := os.Getenv("PROXY_LOG_LEVEL"); v != "" {
func validateAbsoluteURL(fieldName, value string) error { c.Log.Level = v
u, err := url.Parse(value) }
if err != nil || u.Scheme == "" || u.Host == "" { if v := os.Getenv("PROXY_LOG_FORMAT"); v != "" {
return fmt.Errorf("invalid %s %q: must be an absolute URL", fieldName, value) c.Log.Format = v
}
if v := os.Getenv("PROXY_COOLDOWN_DEFAULT"); v != "" {
c.Cooldown.Default = v
}
if v := os.Getenv("PROXY_CACHE_METADATA"); v != "" {
c.CacheMetadata = envBool(v)
}
if v := os.Getenv("PROXY_MIRROR_API"); v != "" {
c.MirrorAPI = envBool(v)
}
if v := os.Getenv("PROXY_METADATA_TTL"); v != "" {
c.MetadataTTL = v
}
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY"); v != "" {
c.Gradle.BuildCache.ReadOnly = v == "true" || v == "1"
}
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE"); v != "" {
c.Gradle.BuildCache.MaxUploadSize = v
}
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE"); v != "" {
c.Gradle.BuildCache.MaxAge = v
}
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE"); v != "" {
c.Gradle.BuildCache.MaxSize = v
}
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL"); v != "" {
c.Gradle.BuildCache.SweepInterval = v
} }
return nil
} }
// Validate checks the configuration for errors. // Validate checks the configuration for errors.
@ -598,11 +420,6 @@ func (c *Config) Validate() error {
if c.BaseURL == "" { if c.BaseURL == "" {
return fmt.Errorf("base_url is required") return fmt.Errorf("base_url is required")
} }
if c.UIBaseURL == "" {
c.UIBaseURL = c.BaseURL
} else if err := validateAbsoluteURL("ui_base_url", c.UIBaseURL); err != nil {
return err
}
if c.Storage.URL == "" && c.Storage.Path == "" { if c.Storage.URL == "" && c.Storage.Path == "" {
return fmt.Errorf("storage.url or storage.path is required") return fmt.Errorf("storage.url or storage.path is required")
} }
@ -651,8 +468,9 @@ func (c *Config) Validate() error {
// Validate direct serve base URL if specified // Validate direct serve base URL if specified
if c.Storage.DirectServeBaseURL != "" { if c.Storage.DirectServeBaseURL != "" {
if err := validateAbsoluteURL("storage.direct_serve_base_url", c.Storage.DirectServeBaseURL); err != nil { u, err := url.Parse(c.Storage.DirectServeBaseURL)
return err if err != nil || u.Scheme == "" || u.Host == "" {
return fmt.Errorf("invalid storage.direct_serve_base_url %q: must be an absolute URL", c.Storage.DirectServeBaseURL)
} }
} }
@ -663,42 +481,10 @@ func (c *Config) Validate() error {
} }
} }
if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil { if err := c.Gradle.BuildCache.Validate(); err != nil {
return err return err
} }
if err := validateHTTPTimeout(c.HTTPTimeout); err != nil {
return err
}
return c.validateComponents()
}
func (c *Config) validateComponents() error {
if err := c.Upstream.Validate(); err != nil {
return err
}
if err := c.Health.Validate(); err != nil {
return err
}
return c.Gradle.BuildCache.Validate()
}
// Validate checks the /health configuration. An unset interval is allowed
// (the cache uses its default); explicit values must parse and be non-negative.
func (h *HealthConfig) Validate() error {
if h.StorageProbeInterval == "" || h.StorageProbeInterval == "0" {
return nil
}
d, err := time.ParseDuration(h.StorageProbeInterval)
if err != nil {
return fmt.Errorf("invalid health.storage_probe_interval %q: %w", h.StorageProbeInterval, err)
}
if d < 0 {
return fmt.Errorf("invalid health.storage_probe_interval %q: must be non-negative", h.StorageProbeInterval)
}
return nil return nil
} }
@ -744,8 +530,6 @@ func (g *GradleBuildCacheConfig) Validate() error {
const ( const (
defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default
defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default
defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default
defaultMetadataMaxSize = 100 << 20
defaultGradleBuildCacheMaxUploadSize = 100 << 20 defaultGradleBuildCacheMaxUploadSize = 100 << 20
defaultGradleBuildCacheSweepInterval = 10 * time.Minute defaultGradleBuildCacheSweepInterval = 10 * time.Minute
defaultGradleMaxUploadSizeStr = "100MB" defaultGradleMaxUploadSizeStr = "100MB"
@ -765,63 +549,6 @@ func (c *Config) ParseMaxSize() int64 {
return size return size
} }
func validateHTTPTimeout(s string) error {
if s == "" || s == "0" {
return nil
}
d, err := time.ParseDuration(s)
if err != nil {
return fmt.Errorf("invalid http_timeout %q: %w", s, err)
}
if d < 0 {
return fmt.Errorf("invalid http_timeout %q: must be non-negative", s)
}
return nil
}
func validateMetadataMaxSize(s string) error {
if s == "" {
return nil
}
size, err := ParseSize(s)
if err != nil {
return fmt.Errorf("invalid metadata_max_size: %w", err)
}
if size <= 0 {
return fmt.Errorf("invalid metadata_max_size %q: must be positive", s)
}
return nil
}
// ParseMetadataMaxSize returns the maximum metadata response size in bytes.
// Returns 100MB if unset or invalid.
func (c *Config) ParseMetadataMaxSize() int64 {
if c.MetadataMaxSize == "" {
return defaultMetadataMaxSize
}
size, err := ParseSize(c.MetadataMaxSize)
if err != nil || size <= 0 {
return defaultMetadataMaxSize
}
return size
}
// ParseHTTPTimeout returns the upstream HTTP client timeout.
// Returns 30s if unset, 0 (no timeout) if explicitly set to "0".
func (c *Config) ParseHTTPTimeout() time.Duration {
if c.HTTPTimeout == "" {
return defaultHTTPTimeout
}
if c.HTTPTimeout == "0" {
return 0
}
d, err := time.ParseDuration(c.HTTPTimeout)
if err != nil || d < 0 {
return defaultHTTPTimeout
}
return d
}
// ParseMetadataTTL returns the metadata TTL duration. // ParseMetadataTTL returns the metadata TTL duration.
// Returns 5 minutes if unset, 0 if explicitly disabled. // Returns 5 minutes if unset, 0 if explicitly disabled.
func (c *Config) ParseMetadataTTL() time.Duration { func (c *Config) ParseMetadataTTL() time.Duration {
@ -928,7 +655,8 @@ func ParseSize(s string) (int64, error) {
} }
for _, s2 := range suffixes { for _, s2 := range suffixes {
if numStr, ok := strings.CutSuffix(s, s2.suffix); ok { if strings.HasSuffix(s, s2.suffix) {
numStr := strings.TrimSuffix(s, s2.suffix)
num, err := strconv.ParseFloat(numStr, 64) num, err := strconv.ParseFloat(numStr, 64)
if err != nil { if err != nil {
return 0, fmt.Errorf("invalid number %q", numStr) return 0, fmt.Errorf("invalid number %q", numStr)

View file

@ -3,7 +3,6 @@ package config
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"time" "time"
) )
@ -26,24 +25,12 @@ func TestDefault(t *testing.T) {
if cfg.Database.Path == "" { if cfg.Database.Path == "" {
t.Error("Database.Path should not be empty") t.Error("Database.Path should not be empty")
} }
if cfg.AccessLog.Path != "" {
t.Errorf("AccessLog.Path = %q, want disabled by default", cfg.AccessLog.Path)
}
if cfg.Gradle.BuildCache.MaxUploadSize != "100MB" { if cfg.Gradle.BuildCache.MaxUploadSize != "100MB" {
t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "100MB") t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "100MB")
} }
if cfg.Gradle.BuildCache.MaxAge != "168h" { if cfg.Gradle.BuildCache.MaxAge != "168h" {
t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "168h") t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "168h")
} }
if cfg.Upstream.Maven != "https://repo1.maven.org/maven2" {
t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://repo1.maven.org/maven2")
}
if cfg.Upstream.GradlePluginPortal != "https://plugins.gradle.org/m2" {
t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.gradle.org/m2")
}
if cfg.Upstream.Debian != "http://deb.debian.org/debian" {
t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://deb.debian.org/debian")
}
} }
func TestValidate(t *testing.T) { func TestValidate(t *testing.T) {
@ -215,8 +202,6 @@ database:
log: log:
level: "debug" level: "debug"
format: "json" format: "json"
access_log:
path: "/var/log/proxy/access.jsonl"
` `
if err := os.WriteFile(path, []byte(content), 0644); err != nil { if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatalf("writing config file: %v", err) t.Fatalf("writing config file: %v", err)
@ -245,9 +230,6 @@ access_log:
if cfg.Log.Format != "json" { if cfg.Log.Format != "json" {
t.Errorf("Log.Format = %q, want %q", cfg.Log.Format, "json") t.Errorf("Log.Format = %q, want %q", cfg.Log.Format, "json")
} }
if cfg.AccessLog.Path != "/var/log/proxy/access.jsonl" {
t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/var/log/proxy/access.jsonl")
}
} }
func TestLoadJSON(t *testing.T) { func TestLoadJSON(t *testing.T) {
@ -280,13 +262,8 @@ func TestLoadFromEnv(t *testing.T) {
t.Setenv("PROXY_LISTEN", ":9000") t.Setenv("PROXY_LISTEN", ":9000")
t.Setenv("PROXY_BASE_URL", "https://env.example.com") t.Setenv("PROXY_BASE_URL", "https://env.example.com")
t.Setenv("PROXY_UI_URL", "https://ui.env.example.com/ui")
t.Setenv("PROXY_STORAGE_PATH", "/env/cache") t.Setenv("PROXY_STORAGE_PATH", "/env/cache")
t.Setenv("PROXY_LOG_LEVEL", testLevelDebug) t.Setenv("PROXY_LOG_LEVEL", testLevelDebug)
t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl")
t.Setenv("PROXY_UPSTREAM_MAVEN", "https://maven.example.com/repository/maven-public")
t.Setenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", "https://plugins.example.com/m2")
t.Setenv("PROXY_UPSTREAM_DEBIAN", "http://archive.ubuntu.com/ubuntu")
t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true")
t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB")
t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h")
@ -301,27 +278,12 @@ func TestLoadFromEnv(t *testing.T) {
if cfg.BaseURL != "https://env.example.com" { if cfg.BaseURL != "https://env.example.com" {
t.Errorf("BaseURL = %q, want %q", cfg.BaseURL, "https://env.example.com") t.Errorf("BaseURL = %q, want %q", cfg.BaseURL, "https://env.example.com")
} }
if cfg.UIBaseURL != "https://ui.env.example.com/ui" {
t.Errorf("UIBaseURL = %q, want %q", cfg.UIBaseURL, "https://ui.env.example.com/ui")
}
if cfg.Storage.Path != "/env/cache" { if cfg.Storage.Path != "/env/cache" {
t.Errorf("Storage.Path = %q, want %q", cfg.Storage.Path, "/env/cache") t.Errorf("Storage.Path = %q, want %q", cfg.Storage.Path, "/env/cache")
} }
if cfg.Log.Level != testLevelDebug { if cfg.Log.Level != testLevelDebug {
t.Errorf("Log.Level = %q, want %q", cfg.Log.Level, testLevelDebug) t.Errorf("Log.Level = %q, want %q", cfg.Log.Level, testLevelDebug)
} }
if cfg.AccessLog.Path != "/tmp/proxy-access.jsonl" {
t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/tmp/proxy-access.jsonl")
}
if cfg.Upstream.Maven != "https://maven.example.com/repository/maven-public" {
t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://maven.example.com/repository/maven-public")
}
if cfg.Upstream.GradlePluginPortal != "https://plugins.example.com/m2" {
t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.example.com/m2")
}
if cfg.Upstream.Debian != "http://archive.ubuntu.com/ubuntu" {
t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://archive.ubuntu.com/ubuntu")
}
if !cfg.Gradle.BuildCache.ReadOnly { if !cfg.Gradle.BuildCache.ReadOnly {
t.Error("Gradle.BuildCache.ReadOnly = false, want true") t.Error("Gradle.BuildCache.ReadOnly = false, want true")
} }
@ -383,34 +345,6 @@ cooldown:
if cfg.Cooldown.Packages["pkg:npm/@babel/core"] != "14d" { if cfg.Cooldown.Packages["pkg:npm/@babel/core"] != "14d" {
t.Errorf("Cooldown.Packages[@babel/core] = %q, want %q", cfg.Cooldown.Packages["pkg:npm/@babel/core"], "14d") t.Errorf("Cooldown.Packages[@babel/core] = %q, want %q", cfg.Cooldown.Packages["pkg:npm/@babel/core"], "14d")
} }
if got := cfg.Cooldown.NormalizedPackages()["pkg:npm/%40babel/core"]; got != "14d" {
t.Errorf("normalized Cooldown.Packages[@babel/core] = %q, want %q", got, "14d")
}
}
func TestCooldownConfigNormalizedPackages(t *testing.T) {
rawScoped := "pkg:npm/@typescript/typescript-darwin-arm64"
canonicalScoped := "pkg:npm/%40typescript/typescript-darwin-arm64"
cfg := CooldownConfig{Packages: map[string]string{
rawScoped: "2d",
canonicalScoped: "3d",
"not-a-purl": "4d",
}}
got := cfg.NormalizedPackages()
if got[canonicalScoped] != "3d" {
t.Errorf("canonical scoped package duration = %q, want %q", got[canonicalScoped], "3d")
}
if _, exists := got[rawScoped]; exists {
t.Errorf("raw scoped package key %q was not canonicalized", rawScoped)
}
if got["not-a-purl"] != "4d" {
t.Errorf("invalid PURL duration = %q, want preserved value %q", got["not-a-purl"], "4d")
}
if cfg.Packages[rawScoped] != "2d" {
t.Error("NormalizedPackages mutated the source map")
}
} }
func TestLoadCooldownFromEnv(t *testing.T) { func TestLoadCooldownFromEnv(t *testing.T) {
@ -480,52 +414,6 @@ func TestParseMetadataTTL(t *testing.T) {
} }
} }
func TestParseMetadataMaxSize(t *testing.T) {
tests := []struct {
name string
size string
want int64
}{
{"unset uses default", "", defaultMetadataMaxSize},
{"explicit value", "250MB", 250 << 20},
{"bytes", "1024", 1024},
{"invalid uses default", "lots", defaultMetadataMaxSize},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Default()
cfg.MetadataMaxSize = tt.size
got := cfg.ParseMetadataMaxSize()
if got != tt.want {
t.Errorf("ParseMetadataMaxSize() = %d, want %d", got, tt.want)
}
})
}
}
func TestValidateMetadataMaxSize(t *testing.T) {
cfg := Default()
cfg.MetadataMaxSize = "not-a-size"
if err := cfg.Validate(); err == nil {
t.Error("expected validation error for invalid metadata_max_size")
}
cfg.MetadataMaxSize = "0"
if err := cfg.Validate(); err == nil {
t.Error("expected validation error for zero metadata_max_size")
}
cfg.MetadataMaxSize = "250MB"
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for valid metadata_max_size: %v", err)
}
cfg.MetadataMaxSize = ""
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for unset metadata_max_size: %v", err)
}
}
func TestValidateMetadataTTL(t *testing.T) { func TestValidateMetadataTTL(t *testing.T) {
cfg := Default() cfg := Default()
cfg.MetadataTTL = "invalid" cfg.MetadataTTL = "invalid"
@ -544,97 +432,6 @@ func TestValidateMetadataTTL(t *testing.T) {
} }
} }
func TestValidateHealthStorageProbeInterval(t *testing.T) {
cfg := Default()
cfg.Health.StorageProbeInterval = "not-a-duration"
if err := cfg.Validate(); err == nil {
t.Error("expected validation error for invalid health.storage_probe_interval")
}
cfg.Health.StorageProbeInterval = "30s"
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for valid health.storage_probe_interval: %v", err)
}
cfg.Health.StorageProbeInterval = "0"
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for zero health.storage_probe_interval: %v", err)
}
cfg.Health.StorageProbeInterval = ""
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for empty health.storage_probe_interval: %v", err)
}
cfg.Health.StorageProbeInterval = "-5s"
if err := cfg.Validate(); err == nil {
t.Error("expected validation error for negative health.storage_probe_interval")
}
}
func TestParseHTTPTimeout(t *testing.T) {
tests := []struct {
name string
timeout string
want time.Duration
}{
{"empty defaults to 30s", "", 30 * time.Second},
{"explicit zero disables", "0", 0},
{"2 minutes", "2m", 2 * time.Minute},
{"90 seconds", "90s", 90 * time.Second},
{"invalid defaults to 30s", "not-a-duration", 30 * time.Second},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Default()
cfg.HTTPTimeout = tt.timeout
got := cfg.ParseHTTPTimeout()
if got != tt.want {
t.Errorf("ParseHTTPTimeout() = %v, want %v", got, tt.want)
}
})
}
}
func TestValidateHTTPTimeout(t *testing.T) {
cfg := Default()
cfg.HTTPTimeout = "not-a-duration"
if err := cfg.Validate(); err == nil {
t.Error("expected validation error for invalid http_timeout")
}
cfg.HTTPTimeout = "-5s"
if err := cfg.Validate(); err == nil {
t.Error("expected validation error for negative http_timeout")
}
cfg.HTTPTimeout = "2m"
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for valid http_timeout: %v", err)
}
cfg.HTTPTimeout = "0"
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for zero http_timeout: %v", err)
}
cfg.HTTPTimeout = ""
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for empty http_timeout: %v", err)
}
}
func TestLoadHTTPTimeoutFromEnv(t *testing.T) {
cfg := Default()
t.Setenv("PROXY_HTTP_TIMEOUT", "90s")
cfg.LoadFromEnv()
if cfg.HTTPTimeout != "90s" {
t.Errorf("HTTPTimeout = %q, want %q", cfg.HTTPTimeout, "90s")
}
}
func TestLoadMetadataTTLFromEnv(t *testing.T) { func TestLoadMetadataTTLFromEnv(t *testing.T) {
cfg := Default() cfg := Default()
t.Setenv("PROXY_METADATA_TTL", "10m") t.Setenv("PROXY_METADATA_TTL", "10m")
@ -735,40 +532,6 @@ func TestLoadDirectServeFromEnv(t *testing.T) {
} }
} }
func TestValidateUIBaseURLDefaultsToBaseURL(t *testing.T) {
cfg := Default()
cfg.BaseURL = "https://proxy.example.com"
cfg.UIBaseURL = ""
if err := cfg.Validate(); err != nil {
t.Fatalf("unexpected validation error: %v", err)
}
if cfg.UIBaseURL != "https://proxy.example.com" {
t.Errorf("UIBaseURL = %q, want it to default to BaseURL %q", cfg.UIBaseURL, "https://proxy.example.com")
}
}
func TestValidateUIBaseURL(t *testing.T) {
cfg := Default()
cfg.UIBaseURL = "not a url"
if err := cfg.Validate(); err == nil {
t.Error("expected validation error for relative ui_base_url")
}
cfg = Default()
cfg.UIBaseURL = "://bad"
if err := cfg.Validate(); err == nil {
t.Error("expected validation error for unparseable ui_base_url")
}
cfg = Default()
cfg.UIBaseURL = "https://ui.example.com/ui"
if err := cfg.Validate(); err != nil {
t.Errorf("unexpected error for valid ui_base_url: %v", err)
}
}
func TestValidateDirectServeBaseURL(t *testing.T) { func TestValidateDirectServeBaseURL(t *testing.T) {
cfg := Default() cfg := Default()
@ -787,135 +550,3 @@ func TestValidateDirectServeBaseURL(t *testing.T) {
t.Errorf("unexpected error for valid direct_serve_base_url: %v", err) t.Errorf("unexpected error for valid direct_serve_base_url: %v", err)
} }
} }
func TestDatabaseConfigString(t *testing.T) {
tests := []struct {
name string
cfg DatabaseConfig
want string
}{
{"sqlite", DatabaseConfig{Driver: "sqlite", Path: "./cache/proxy.db"}, "./cache/proxy.db"},
{"default driver", DatabaseConfig{Path: "/var/lib/proxy.db"}, "/var/lib/proxy.db"},
{"postgres no password", DatabaseConfig{Driver: "postgres", URL: "postgres://user@localhost:5432/proxy"}, "postgres://user@localhost:5432/proxy"},
{"postgres redacts password", DatabaseConfig{Driver: "postgres", URL: "postgres://user:secret@localhost:5432/proxy?sslmode=disable"}, "postgres://user:xxxxx@localhost:5432/proxy?sslmode=disable"},
{"postgres unparseable url", DatabaseConfig{Driver: "postgres", URL: "host=localhost user=foo password=bar"}, "postgres"},
{"postgres ignores sqlite path", DatabaseConfig{Driver: "postgres", URL: "postgres://localhost/db", Path: "./cache/proxy.db"}, "postgres://localhost/db"},
}
for _, tt := range tests {
if got := tt.cfg.String(); got != tt.want {
t.Errorf("%s: String() = %q, want %q", tt.name, got, tt.want)
}
}
}
func TestUpstreamAuthForURLMatchesURLComponents(t *testing.T) {
registryAuth := AuthConfig{Type: "bearer", Token: "registry-token"}
privateAuth := AuthConfig{Type: "bearer", Token: "private-token"}
config := UpstreamConfig{Auth: map[string]AuthConfig{
"https://registry.example.com": registryAuth,
"https://registry.example.com/private": privateAuth,
}}
tests := []struct {
name string
url string
wantToken string
}{
{name: "registry root", url: "https://registry.example.com/package", wantToken: "registry-token"},
{name: "host is case insensitive", url: "https://REGISTRY.EXAMPLE.COM/package", wantToken: "registry-token"},
{name: "longest path match", url: "https://registry.example.com/private/package", wantToken: "private-token"},
{name: "exact path match", url: "https://registry.example.com/private", wantToken: "private-token"},
{name: "path segment boundary", url: "https://registry.example.com/private-other/package", wantToken: "registry-token"},
{name: "lookalike host rejected", url: "https://registry.example.com.evil.test/package"},
{name: "different scheme rejected", url: "http://registry.example.com/package"},
{name: "different port rejected", url: "https://registry.example.com:8443/package"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
auth := config.AuthForURL(tt.url)
if tt.wantToken == "" {
if auth != nil {
t.Fatalf("AuthForURL() = %+v, want nil", auth)
}
return
}
if auth == nil {
t.Fatal("AuthForURL() = nil, want authentication")
}
if auth.Token != tt.wantToken {
t.Errorf("token = %q, want %q", auth.Token, tt.wantToken)
}
})
}
}
func TestValidateUpstreamAuthURLs(t *testing.T) {
t.Run("valid absolute URL", func(t *testing.T) {
cfg := Default()
cfg.Upstream.Auth = map[string]AuthConfig{
"https://registry.example.com/private": {Type: "bearer", Token: "token"},
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
t.Run("invalid URL", func(t *testing.T) {
cfg := Default()
cfg.Upstream.Auth = map[string]AuthConfig{
"registry.example.com": {Type: "bearer", Token: "token"},
}
err := cfg.Validate()
if err == nil {
t.Fatal("Validate() error = nil, want invalid upstream.auth URL error")
}
if !strings.Contains(err.Error(), "upstream.auth") || !strings.Contains(err.Error(), "registry.example.com") {
t.Errorf("Validate() error = %q, want field and URL", err)
}
})
}
func TestValidateNamedUpstreams(t *testing.T) {
tests := []struct {
name string
modify func(*Config)
wantErr bool
}{
{
name: "valid Helm and OCI upstreams",
modify: func(cfg *Config) {
cfg.Upstream.Helm = map[string]string{"bitnami": "https://charts.bitnami.com/bitnami"}
cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"}
},
},
{
name: "Helm upstream name contains path separator",
modify: func(cfg *Config) {
cfg.Upstream.Helm = map[string]string{"team/charts": "https://charts.example.com"}
},
wantErr: true,
},
{
name: "OCI upstream URL is not absolute",
modify: func(cfg *Config) {
cfg.Upstream.OCI = map[string]string{"private": "registry.example.com"}
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Default()
tt.modify(cfg)
err := cfg.Validate()
if (err != nil) != tt.wantErr {
t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr)
}
})
}
}

View file

@ -0,0 +1,125 @@
package cooldown
import (
"fmt"
"strconv"
"strings"
"time"
)
const hoursPerDay = 24
// Config holds cooldown settings for version filtering.
// Cooldown hides package versions published too recently, giving the community
// time to spot malicious releases before they're pulled into projects.
type Config struct {
// Default is the global default cooldown duration (e.g., "3d", "48h").
Default string `json:"default" yaml:"default"`
// Ecosystems overrides the default for specific ecosystems.
// Keys are ecosystem names (e.g., "npm", "pypi").
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
// Packages overrides the cooldown for specific packages.
// Keys are PURLs (e.g., "pkg:npm/lodash", "pkg:npm/@babel/core").
Packages map[string]string `json:"packages" yaml:"packages"`
defaultDuration time.Duration
ecosystemDurations map[string]time.Duration
packageDurations map[string]time.Duration
parsed bool
}
// parse resolves all string durations into time.Duration values.
// Called lazily on first use.
func (c *Config) parse() {
if c.parsed {
return
}
c.parsed = true
c.defaultDuration, _ = ParseDuration(c.Default)
c.ecosystemDurations = make(map[string]time.Duration, len(c.Ecosystems))
for k, v := range c.Ecosystems {
d, _ := ParseDuration(v)
c.ecosystemDurations[k] = d
}
c.packageDurations = make(map[string]time.Duration, len(c.Packages))
for k, v := range c.Packages {
d, _ := ParseDuration(v)
c.packageDurations[k] = d
}
}
// For returns the effective cooldown duration for a given ecosystem and package PURL.
// Resolution order: package override > ecosystem override > global default.
func (c *Config) For(ecosystem, packagePURL string) time.Duration {
c.parse()
if d, ok := c.packageDurations[packagePURL]; ok {
return d
}
if d, ok := c.ecosystemDurations[ecosystem]; ok {
return d
}
return c.defaultDuration
}
// IsAllowed returns true if a version with the given publish time has passed
// the cooldown period for this ecosystem/package.
func (c *Config) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool {
d := c.For(ecosystem, packagePURL)
if d == 0 {
return true
}
if publishedAt.IsZero() {
return true
}
return time.Since(publishedAt) >= d
}
// Enabled returns true if any cooldown is configured.
func (c *Config) Enabled() bool {
c.parse()
if c.defaultDuration > 0 {
return true
}
for _, d := range c.ecosystemDurations {
if d > 0 {
return true
}
}
for _, d := range c.packageDurations {
if d > 0 {
return true
}
}
return false
}
// ParseDuration parses a duration string supporting days (e.g., "3d"),
// in addition to Go's standard time.ParseDuration formats ("48h", "30m").
// "0" means disabled (returns 0).
func ParseDuration(s string) (time.Duration, error) {
s = strings.TrimSpace(s)
if s == "" || s == "0" {
return 0, nil
}
// Handle day suffix
if numStr, ok := strings.CutSuffix(s, "d"); ok {
days, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0, fmt.Errorf("invalid duration %q: %w", s, err)
}
return time.Duration(days * float64(hoursPerDay*time.Hour)), nil
}
d, err := time.ParseDuration(s)
if err != nil {
return 0, fmt.Errorf("invalid duration %q: %w", s, err)
}
return d, nil
}

View file

@ -0,0 +1,133 @@
package cooldown
import (
"testing"
"time"
)
func TestParseDuration(t *testing.T) {
tests := []struct {
input string
want time.Duration
wantErr bool
}{
{"", 0, false},
{"0", 0, false},
{"3d", 3 * 24 * time.Hour, false},
{"7d", 7 * 24 * time.Hour, false},
{"14d", 14 * 24 * time.Hour, false},
{"1.5d", 36 * time.Hour, false},
{"48h", 48 * time.Hour, false},
{"30m", 30 * time.Minute, false},
{"1h30m", 90 * time.Minute, false},
{"invalid", 0, true},
{"d", 0, true},
{"xd", 0, true},
}
for _, tt := range tests {
got, err := ParseDuration(tt.input)
if (err != nil) != tt.wantErr {
t.Errorf("ParseDuration(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
continue
}
if got != tt.want {
t.Errorf("ParseDuration(%q) = %v, want %v", tt.input, got, tt.want)
}
}
}
func TestConfigFor(t *testing.T) {
c := &Config{
Default: "3d",
Ecosystems: map[string]string{
"npm": "7d",
"cargo": "0",
},
Packages: map[string]string{
"pkg:npm/lodash": "0",
"pkg:npm/@babel/core": "14d",
},
}
tests := []struct {
ecosystem string
packagePURL string
want time.Duration
}{
// Package override takes priority
{"npm", "pkg:npm/lodash", 0},
{"npm", "pkg:npm/@babel/core", 14 * 24 * time.Hour},
// Ecosystem override
{"npm", "pkg:npm/express", 7 * 24 * time.Hour},
{"cargo", "pkg:cargo/serde", 0},
// Global default
{"pypi", "pkg:pypi/requests", 3 * 24 * time.Hour},
{"pub", "pkg:pub/flutter", 3 * 24 * time.Hour},
}
for _, tt := range tests {
got := c.For(tt.ecosystem, tt.packagePURL)
if got != tt.want {
t.Errorf("For(%q, %q) = %v, want %v", tt.ecosystem, tt.packagePURL, got, tt.want)
}
}
}
func TestConfigIsAllowed(t *testing.T) {
c := &Config{
Default: "3d",
Packages: map[string]string{
"pkg:npm/lodash": "0",
},
}
now := time.Now()
tests := []struct {
name string
ecosystem string
packagePURL string
publishedAt time.Time
want bool
}{
{"old enough", "npm", "pkg:npm/express", now.Add(-4 * 24 * time.Hour), true},
{"too recent", "npm", "pkg:npm/express", now.Add(-1 * 24 * time.Hour), false},
{"exactly at boundary", "npm", "pkg:npm/express", now.Add(-3 * 24 * time.Hour), true},
{"exempt package", "npm", "pkg:npm/lodash", now.Add(-1 * time.Minute), true},
{"zero time", "npm", "pkg:npm/express", time.Time{}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := c.IsAllowed(tt.ecosystem, tt.packagePURL, tt.publishedAt)
if got != tt.want {
t.Errorf("IsAllowed(%q, %q, %v) = %v, want %v",
tt.ecosystem, tt.packagePURL, tt.publishedAt, got, tt.want)
}
})
}
}
func TestConfigEnabled(t *testing.T) {
tests := []struct {
name string
cfg Config
want bool
}{
{"empty config", Config{}, false},
{"default only", Config{Default: "3d"}, true},
{"ecosystem only", Config{Ecosystems: map[string]string{"npm": "7d"}}, true},
{"package only", Config{Packages: map[string]string{"pkg:npm/x": "1d"}}, true},
{"all zero", Config{Default: "0", Ecosystems: map[string]string{"npm": "0"}}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.cfg.Enabled()
if got != tt.want {
t.Errorf("Enabled() = %v, want %v", got, tt.want)
}
})
}
}

View file

@ -8,11 +8,6 @@ import (
"time" "time"
) )
const (
testContentHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
testIntegrity = "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="
)
func TestCreateAndOpen(t *testing.T) { func TestCreateAndOpen(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db") dbPath := filepath.Join(dir, "test.db")
@ -137,7 +132,7 @@ func TestVersionCRUD(t *testing.T) {
v := &Version{ v := &Version{
PURL: "pkg:npm/lodash@4.17.21", PURL: "pkg:npm/lodash@4.17.21",
PackagePURL: "pkg:npm/lodash", PackagePURL: "pkg:npm/lodash",
Integrity: sql.NullString{String: testIntegrity, Valid: true}, Integrity: sql.NullString{String: "sha512-abc123", Valid: true},
} }
err = db.UpsertVersion(v) err = db.UpsertVersion(v)
@ -205,7 +200,7 @@ func TestArtifactCRUD(t *testing.T) {
t.Error("expected artifact to not be cached yet") t.Error("expected artifact to not be cached yet")
} }
err = db.MarkArtifactCached(versionPURL, "lodash-4.17.21.tgz", "/cache/npm/lodash-4.17.21.tgz", testContentHash, 12345, "application/gzip") err = db.MarkArtifactCached(versionPURL, "lodash-4.17.21.tgz", "/cache/npm/lodash-4.17.21.tgz", "sha256-abc", 12345, "application/gzip")
if err != nil { if err != nil {
t.Fatalf("MarkArtifactCached failed: %v", err) t.Fatalf("MarkArtifactCached failed: %v", err)
} }
@ -244,86 +239,6 @@ func TestArtifactCRUD(t *testing.T) {
}) })
} }
func TestGetCachedArtifact(t *testing.T) {
runWithBothDatabases(t, func(t *testing.T, db *DB) {
const (
packagePURL = "pkg:npm/lodash"
versionPURL = "pkg:npm/lodash@4.17.21"
filename = "lodash-4.17.21.tgz"
)
seedCachedArtifactTestData(t, db, packagePURL, versionPURL, filename)
cached, err := db.GetCachedArtifact(packagePURL, versionPURL, filename)
if err != nil {
t.Fatalf("GetCachedArtifact before cache failed: %v", err)
}
if cached != nil {
t.Fatalf("expected no cached artifact, got %+v", cached)
}
if err := db.MarkArtifactCached(versionPURL, filename, "/cache/npm/"+filename,
testContentHash, 12345, "application/gzip"); err != nil {
t.Fatalf("MarkArtifactCached failed: %v", err)
}
cached, err = db.GetCachedArtifact(packagePURL, versionPURL, filename)
if err != nil {
t.Fatalf("GetCachedArtifact failed: %v", err)
}
if cached == nil {
t.Fatal("expected cached artifact, got nil")
}
if cached.Ecosystem != "npm" {
t.Errorf("expected npm ecosystem, got %q", cached.Ecosystem)
}
if cached.StoragePath != "/cache/npm/"+filename {
t.Errorf("expected cached storage path, got %q", cached.StoragePath)
}
if cached.ContentHash.String != testContentHash {
t.Errorf("expected cached content hash, got %q", cached.ContentHash.String)
}
if cached.Size.Int64 != 12345 {
t.Errorf("expected cached size 12345, got %d", cached.Size.Int64)
}
if cached.ContentType.String != "application/gzip" {
t.Errorf("expected cached content type, got %q", cached.ContentType.String)
}
if cached.Integrity.String != testIntegrity {
t.Errorf("expected cached integrity, got %q", cached.Integrity.String)
}
cached, err = db.GetCachedArtifact("pkg:npm/other", versionPURL, filename)
if err != nil {
t.Fatalf("GetCachedArtifact with wrong package failed: %v", err)
}
if cached != nil {
t.Fatalf("expected package mismatch to miss cache, got %+v", cached)
}
})
}
func seedCachedArtifactTestData(t *testing.T, db *DB, packagePURL, versionPURL, filename string) {
t.Helper()
if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: "npm", Name: "lodash"}); err != nil {
t.Fatalf("UpsertPackage failed: %v", err)
}
if err := db.UpsertVersion(&Version{
PURL: versionPURL,
PackagePURL: packagePURL,
Integrity: sql.NullString{String: testIntegrity, Valid: true},
}); err != nil {
t.Fatalf("UpsertVersion failed: %v", err)
}
if err := db.UpsertArtifact(&Artifact{
VersionPURL: versionPURL,
Filename: filename,
UpstreamURL: "https://registry.npmjs.org/lodash/-/" + filename,
}); err != nil {
t.Fatalf("UpsertArtifact failed: %v", err)
}
}
func TestCacheManagement(t *testing.T) { func TestCacheManagement(t *testing.T) {
runWithBothDatabases(t, func(t *testing.T, db *DB) { runWithBothDatabases(t, func(t *testing.T, db *DB) {
pkg := &Package{ pkg := &Package{

View file

@ -30,12 +30,8 @@ func TestUpsertAndGetMetadataCache(t *testing.T) {
StoragePath: "_metadata/npm/lodash/metadata", StoragePath: "_metadata/npm/lodash/metadata",
ETag: sql.NullString{String: `"abc123"`, Valid: true}, ETag: sql.NullString{String: `"abc123"`, Valid: true},
ContentType: sql.NullString{String: "application/json", Valid: true}, ContentType: sql.NullString{String: "application/json", Valid: true},
ContentDigest: sql.NullString{ Size: sql.NullInt64{Int64: 1024, Valid: true},
String: "sha256:0123456789abcdef", FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
Valid: true,
},
Size: sql.NullInt64{Int64: 1024, Valid: true},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
} }
err := db.UpsertMetadataCache(entry) err := db.UpsertMetadataCache(entry)
@ -66,9 +62,6 @@ func TestUpsertAndGetMetadataCache(t *testing.T) {
if !got.ContentType.Valid || got.ContentType.String != "application/json" { if !got.ContentType.Valid || got.ContentType.String != "application/json" {
t.Errorf("content_type = %v, want %q", got.ContentType, "application/json") t.Errorf("content_type = %v, want %q", got.ContentType, "application/json")
} }
if !got.ContentDigest.Valid || got.ContentDigest.String != "sha256:0123456789abcdef" {
t.Errorf("content_digest = %v, want %q", got.ContentDigest, "sha256:0123456789abcdef")
}
if !got.Size.Valid || got.Size.Int64 != 1024 { if !got.Size.Valid || got.Size.Int64 != 1024 {
t.Errorf("size = %v, want 1024", got.Size) t.Errorf("size = %v, want 1024", got.Size)
} }
@ -185,47 +178,3 @@ func TestMetadataCacheTableCreatedByMigration(t *testing.T) {
t.Error("metadata_cache table should exist after migration") t.Error("metadata_cache table should exist after migration")
} }
} }
func TestMetadataCacheContentDigestMigrationPreservesExistingRows(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "test.db")
db, err := Create(dbPath)
if err != nil {
t.Fatalf("Create failed: %v", err)
}
defer func() { _ = db.Close() }()
if _, err := db.Exec("ALTER TABLE metadata_cache DROP COLUMN content_digest"); err != nil {
t.Fatalf("dropping content_digest: %v", err)
}
if _, err := db.Exec("DELETE FROM migrations WHERE name = ?", "006_add_metadata_content_digest"); err != nil {
t.Fatalf("resetting digest migration: %v", err)
}
if _, err := db.Exec(`
INSERT INTO metadata_cache (ecosystem, name, storage_path, content_type, size, fetched_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`, "oci-manifest", "cache-key", "_metadata/oci-manifest/cache-key/metadata", "application/json", 2, time.Now(), time.Now(), time.Now()); err != nil {
t.Fatalf("inserting legacy cache row: %v", err)
}
if err := db.MigrateSchema(); err != nil {
t.Fatalf("MigrateSchema() error = %v", err)
}
hasDigest, err := db.HasColumn("metadata_cache", "content_digest")
if err != nil {
t.Fatalf("HasColumn() error = %v", err)
}
if !hasDigest {
t.Fatal("metadata_cache.content_digest was not added")
}
entry, err := db.GetMetadataCache("oci-manifest", "cache-key")
if err != nil {
t.Fatalf("GetMetadataCache() error = %v", err)
}
if entry == nil || entry.StoragePath != "_metadata/oci-manifest/cache-key/metadata" {
t.Fatalf("existing metadata cache row was not preserved: %#v", entry)
}
if entry.ContentDigest.Valid {
t.Errorf("legacy content digest = %q, want NULL", entry.ContentDigest.String)
}
}

View file

@ -191,28 +191,6 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) {
return &a, nil return &a, nil
} }
// GetCachedArtifact returns the fields needed to serve a cached artifact.
func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*CachedArtifact, error) {
var artifact CachedArtifact
query := db.Rebind(`
SELECT packages.ecosystem, artifacts.storage_path, artifacts.content_hash, artifacts.size,
artifacts.content_type, versions.integrity
FROM artifacts
JOIN versions ON versions.purl = artifacts.version_purl
JOIN packages ON packages.purl = versions.package_purl
WHERE packages.purl = ? AND artifacts.version_purl = ? AND artifacts.filename = ?
AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL
`)
err := db.Get(&artifact, query, packagePURL, versionPURL, filename)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
return &artifact, nil
}
func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) { func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) {
var a Artifact var a Artifact
query := db.Rebind(` query := db.Rebind(`
@ -465,14 +443,11 @@ func (db *DB) GetMostPopularPackages(limit int) ([]PopularPackage, error) {
} }
type RecentPackage struct { type RecentPackage struct {
Ecosystem string `db:"ecosystem"` Ecosystem string `db:"ecosystem"`
Name string `db:"name"` Name string `db:"name"`
VersionPURL string `db:"version_purl"` Version string `db:"version"`
CachedAt time.Time `db:"fetched_at"` CachedAt time.Time `db:"fetched_at"`
Size int64 `db:"size"` Size int64 `db:"size"`
// Version is derived from VersionPURL rather than selected, so that the
// PURL percent-encoding is decoded (e.g. "%2B" back to "+").
Version string `db:"-"`
} }
func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
@ -486,10 +461,10 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
} }
var packages []RecentPackage var packages []RecentPackage
// There is no separate version column, so the full version PURL is selected // We need to extract version from the purl since there's no separate version column
// and the version is decoded from it in Go.
query := db.Rebind(` query := db.Rebind(`
SELECT p.ecosystem, p.name, v.purl as version_purl, SELECT p.ecosystem, p.name,
SUBSTR(v.purl, INSTR(v.purl, '@') + 1) as version,
a.fetched_at, COALESCE(a.size, 0) as size a.fetched_at, COALESCE(a.size, 0) as size
FROM artifacts a FROM artifacts a
JOIN versions v ON v.purl = a.version_purl JOIN versions v ON v.purl = a.version_purl
@ -499,13 +474,25 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
LIMIT ? LIMIT ?
`) `)
// For postgres, use different string function
if db.dialect == DialectPostgres {
query = db.Rebind(`
SELECT p.ecosystem, p.name,
SUBSTRING(v.purl FROM POSITION('@' IN v.purl) + 1) as version,
a.fetched_at, COALESCE(a.size, 0) as size
FROM artifacts a
JOIN versions v ON v.purl = a.version_purl
JOIN packages p ON p.purl = v.package_purl
WHERE a.storage_path IS NOT NULL AND a.fetched_at IS NOT NULL
ORDER BY a.fetched_at DESC
LIMIT ?
`)
}
err = db.Select(&packages, query, limit) err = db.Select(&packages, query, limit)
if err != nil { if err != nil {
return nil, err return nil, err
} }
for i := range packages {
packages[i].Version = VersionFromPURL(packages[i].VersionPURL)
}
return packages, nil return packages, nil
} }
@ -907,7 +894,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err
var entry MetadataCacheEntry var entry MetadataCacheEntry
query := db.Rebind(` query := db.Rebind(`
SELECT id, ecosystem, name, storage_path, etag, content_type, SELECT id, ecosystem, name, storage_path, etag, content_type,
content_digest, size, last_modified, fetched_at, created_at, updated_at size, last_modified, fetched_at, created_at, updated_at
FROM metadata_cache WHERE ecosystem = ? AND name = ? FROM metadata_cache WHERE ecosystem = ? AND name = ?
`) `)
err := db.Get(&entry, query, ecosystem, name) err := db.Get(&entry, query, ecosystem, name)
@ -927,13 +914,12 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
if db.dialect == DialectPostgres { if db.dialect == DialectPostgres {
query = ` query = `
INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type,
content_digest, size, last_modified, fetched_at, created_at, updated_at) size, last_modified, fetched_at, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT(ecosystem, name) DO UPDATE SET ON CONFLICT(ecosystem, name) DO UPDATE SET
storage_path = EXCLUDED.storage_path, storage_path = EXCLUDED.storage_path,
etag = EXCLUDED.etag, etag = EXCLUDED.etag,
content_type = EXCLUDED.content_type, content_type = EXCLUDED.content_type,
content_digest = EXCLUDED.content_digest,
size = EXCLUDED.size, size = EXCLUDED.size,
last_modified = EXCLUDED.last_modified, last_modified = EXCLUDED.last_modified,
fetched_at = EXCLUDED.fetched_at, fetched_at = EXCLUDED.fetched_at,
@ -942,13 +928,12 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
} else { } else {
query = ` query = `
INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type,
content_digest, size, last_modified, fetched_at, created_at, updated_at) size, last_modified, fetched_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(ecosystem, name) DO UPDATE SET ON CONFLICT(ecosystem, name) DO UPDATE SET
storage_path = excluded.storage_path, storage_path = excluded.storage_path,
etag = excluded.etag, etag = excluded.etag,
content_type = excluded.content_type, content_type = excluded.content_type,
content_digest = excluded.content_digest,
size = excluded.size, size = excluded.size,
last_modified = excluded.last_modified, last_modified = excluded.last_modified,
fetched_at = excluded.fetched_at, fetched_at = excluded.fetched_at,
@ -958,7 +943,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error {
_, err := db.Exec(query, _, err := db.Exec(query,
entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag,
entry.ContentType, entry.ContentDigest, entry.Size, entry.LastModified, entry.FetchedAt, now, now, entry.ContentType, entry.Size, entry.LastModified, entry.FetchedAt, now, now,
) )
if err != nil { if err != nil {
return fmt.Errorf("upserting metadata cache: %w", err) return fmt.Errorf("upserting metadata cache: %w", err)

View file

@ -102,7 +102,6 @@ CREATE TABLE IF NOT EXISTS metadata_cache (
storage_path TEXT NOT NULL, storage_path TEXT NOT NULL,
etag TEXT, etag TEXT,
content_type TEXT, content_type TEXT,
content_digest TEXT,
size INTEGER, size INTEGER,
last_modified DATETIME, last_modified DATETIME,
fetched_at DATETIME, fetched_at DATETIME,
@ -203,7 +202,6 @@ CREATE TABLE IF NOT EXISTS metadata_cache (
storage_path TEXT NOT NULL, storage_path TEXT NOT NULL,
etag TEXT, etag TEXT,
content_type TEXT, content_type TEXT,
content_digest TEXT,
size BIGINT, size BIGINT,
last_modified TIMESTAMP, last_modified TIMESTAMP,
fetched_at TIMESTAMP, fetched_at TIMESTAMP,
@ -361,7 +359,6 @@ var migrations = []migration{
{"003_ensure_artifacts_table", migrateEnsureArtifactsTable}, {"003_ensure_artifacts_table", migrateEnsureArtifactsTable},
{"004_ensure_vulnerabilities_table", migrateEnsureVulnerabilitiesTable}, {"004_ensure_vulnerabilities_table", migrateEnsureVulnerabilitiesTable},
{"005_ensure_metadata_cache_table", migrateEnsureMetadataCacheTable}, {"005_ensure_metadata_cache_table", migrateEnsureMetadataCacheTable},
{"006_add_metadata_content_digest", migrateAddMetadataContentDigest},
} }
// isTableNotFound returns true if the error indicates a missing table. // isTableNotFound returns true if the error indicates a missing table.
@ -584,20 +581,6 @@ func migrateEnsureMetadataCacheTable(db *DB) error {
return db.EnsureMetadataCacheTable() return db.EnsureMetadataCacheTable()
} }
func migrateAddMetadataContentDigest(db *DB) error {
hasColumn, err := db.HasColumn("metadata_cache", "content_digest")
if err != nil {
return fmt.Errorf("checking metadata_cache content_digest column: %w", err)
}
if hasColumn {
return nil
}
if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN content_digest TEXT"); err != nil {
return fmt.Errorf("adding metadata_cache content_digest column: %w", err)
}
return nil
}
// EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist.
func (db *DB) EnsureMetadataCacheTable() error { func (db *DB) EnsureMetadataCacheTable() error {
has, err := db.HasTable("metadata_cache") has, err := db.HasTable("metadata_cache")
@ -618,7 +601,6 @@ func (db *DB) EnsureMetadataCacheTable() error {
storage_path TEXT NOT NULL, storage_path TEXT NOT NULL,
etag TEXT, etag TEXT,
content_type TEXT, content_type TEXT,
content_digest TEXT,
size BIGINT, size BIGINT,
last_modified TIMESTAMP, last_modified TIMESTAMP,
fetched_at TIMESTAMP, fetched_at TIMESTAMP,
@ -636,7 +618,6 @@ func (db *DB) EnsureMetadataCacheTable() error {
storage_path TEXT NOT NULL, storage_path TEXT NOT NULL,
etag TEXT, etag TEXT,
content_type TEXT, content_type TEXT,
content_digest TEXT,
size INTEGER, size INTEGER,
last_modified DATETIME, last_modified DATETIME,
fetched_at DATETIME, fetched_at DATETIME,

View file

@ -2,7 +2,6 @@ package database
import ( import (
"database/sql" "database/sql"
"net/url"
"strings" "strings"
"time" "time"
) )
@ -48,79 +47,10 @@ type Version struct {
// Version extracts the version string from the PURL. // Version extracts the version string from the PURL.
// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21" // e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21"
func (v *Version) Version() string { func (v *Version) Version() string {
return VersionFromPURL(v.PURL) if idx := strings.LastIndex(v.PURL, "@"); idx >= 0 {
} return v.PURL[idx+1:]
// EscapedVersion returns the version escaped for use as a single URL path
// segment.
//
// Version returns decoded text, which is what should be shown to a user but is
// not safe to drop into a link: html/template preserves reserved characters and
// existing escapes in a URL, so "release/1" would split into two path segments,
// "v1?build" would start a query string, and a literal "%2B" would be read back
// as "+". Escaping here and decoding in splitWildcardPath round-trips the value,
// so the link resolves to the version that was stored.
func (v *Version) EscapedVersion() string {
return url.PathEscape(v.Version())
}
// DisplayPURL returns the PURL with its path components percent-decoded, for
// showing in the UI. The stored PURL keeps the canonical encoding (which is
// what the API and all lookups use); this is only a readable rendering, so that
// a version like "7.91+dfsg1-2ubuntu0.1" is not shown as "7.91%2Bdfsg1-2ubuntu0.1"
// and an npm scope is shown as "@babel" rather than "%40babel". Qualifiers and
// subpath keep their encoding, since decoding those would be ambiguous.
func (v *Version) DisplayPURL() string {
base, suffix := v.PURL, ""
if i := strings.IndexAny(base, "?#"); i >= 0 {
base, suffix = base[:i], base[i:]
} }
return ""
name, version := base, ""
if idx := strings.LastIndex(base, "@"); idx >= 0 {
name, version = base[:idx], "@"+decodePURLComponent(base[idx+1:])
}
parts := strings.Split(name, "/")
for i, part := range parts {
parts[i] = decodePURLComponent(part)
}
return strings.Join(parts, "/") + version + suffix
}
// VersionFromPURL extracts the decoded version string from a PURL.
//
// PURL percent-encodes characters that are not safe in a path component, so a
// Debian version like "7.91+dfsg1-2ubuntu0.1" is stored as
// "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1". The raw substring after "@" is
// therefore not the version: it must be percent-decoded before being displayed
// or used to build a URL, otherwise "%2B" leaks into the UI and round-tripping
// the value back into a PURL double-encodes it.
//
// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21"
func VersionFromPURL(p string) string {
// Qualifiers ("?key=value") and subpath ("#path") follow the version.
if i := strings.IndexAny(p, "?#"); i >= 0 {
p = p[:i]
}
idx := strings.LastIndex(p, "@")
if idx < 0 {
return ""
}
return decodePURLComponent(p[idx+1:])
}
// decodePURLComponent percent-decodes a single PURL path component, returning
// the input unchanged if it is not valid percent-encoding.
func decodePURLComponent(s string) string {
if !strings.Contains(s, "%") {
return s
}
decoded, err := url.PathUnescape(s)
if err != nil {
return s
}
return decoded
} }
// Artifact represents a cached artifact in the database. // Artifact represents a cached artifact in the database.
@ -146,30 +76,19 @@ func (a *Artifact) IsCached() bool {
return a.StoragePath.Valid && a.FetchedAt.Valid return a.StoragePath.Valid && a.FetchedAt.Valid
} }
// CachedArtifact contains the fields needed to serve a cached artifact.
type CachedArtifact struct {
Ecosystem string `db:"ecosystem"`
StoragePath string `db:"storage_path"`
ContentHash sql.NullString `db:"content_hash"`
Size sql.NullInt64 `db:"size"`
ContentType sql.NullString `db:"content_type"`
Integrity sql.NullString `db:"integrity"`
}
// MetadataCacheEntry represents a cached metadata blob for offline serving. // MetadataCacheEntry represents a cached metadata blob for offline serving.
type MetadataCacheEntry struct { type MetadataCacheEntry struct {
ID int64 `db:"id" json:"id"` ID int64 `db:"id" json:"id"`
Ecosystem string `db:"ecosystem" json:"ecosystem"` Ecosystem string `db:"ecosystem" json:"ecosystem"`
Name string `db:"name" json:"name"` Name string `db:"name" json:"name"`
StoragePath string `db:"storage_path" json:"storage_path"` StoragePath string `db:"storage_path" json:"storage_path"`
ETag sql.NullString `db:"etag" json:"etag,omitempty"` ETag sql.NullString `db:"etag" json:"etag,omitempty"`
ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"`
ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"` Size sql.NullInt64 `db:"size" json:"size,omitempty"`
Size sql.NullInt64 `db:"size" json:"size,omitempty"` LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"`
LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"`
FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` CreatedAt time.Time `db:"created_at" json:"created_at"`
CreatedAt time.Time `db:"created_at" json:"created_at"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
} }
// Vulnerability represents a cached vulnerability record. // Vulnerability represents a cached vulnerability record.

View file

@ -1,159 +0,0 @@
package database
import (
"database/sql"
"net/url"
"testing"
"time"
)
func TestVersionFromPURL(t *testing.T) {
tests := []struct {
name string
purl string
want string
}{
{"simple", "pkg:npm/lodash@4.17.21", "4.17.21"},
{"namespaced", "pkg:composer/symfony/console@6.0.0", "6.0.0"},
// Debian/Ubuntu versions routinely contain "+", which PURL encodes.
{"encoded plus", "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"},
{"encoded epoch", "pkg:deb/curl@1%3A7.81.0-1", "1:7.81.0-1"},
{"encoded plus with qualifier", "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", "7.91+dfsg1"},
{"tilde is not encoded", "pkg:deb/foo@1.0~rc1", "1.0~rc1"},
{"no version", "pkg:npm/lodash", ""},
{"invalid escape passed through", "pkg:npm/lodash@1.0%zz", "1.0%zz"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := VersionFromPURL(tt.purl); got != tt.want {
t.Errorf("VersionFromPURL(%q) = %q, want %q", tt.purl, got, tt.want)
}
v := &Version{PURL: tt.purl}
if got := v.Version(); got != tt.want {
t.Errorf("Version.Version() for %q = %q, want %q", tt.purl, got, tt.want)
}
})
}
}
// TestVersionEscapedVersion checks the value the templates put in a URL. It
// must survive the round trip back through the router: escaping here and
// decoding per path segment on the way in has to yield the original version.
func TestVersionEscapedVersion(t *testing.T) {
tests := []struct {
name string
purl string
want string
}{
{"simple", "pkg:npm/lodash@4.17.21", "4.17.21"},
// "+" is legal in a path segment, so it stays literal and the UI keeps
// showing the version the way Debian writes it.
{"plus stays literal", "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1-2ubuntu0.1"},
// A slash would otherwise split the version into two path segments.
{"slash", "pkg:golang/example@release%2F1", "release%2F1"},
// A question mark would otherwise start the query string.
{"question mark", "pkg:npm/example@v1%3Fbuild", "v1%3Fbuild"},
// A version containing a literal "%2B" is stored double-encoded; the
// link must re-encode it or it decodes back to "+" instead.
{"literal percent escape", "pkg:npm/example@1.0%252B", "1.0%252B"},
{"space", "pkg:npm/example@1.0%20beta", "1.0%20beta"},
{"no version", "pkg:npm/lodash", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
v := &Version{PURL: tt.purl}
got := v.EscapedVersion()
if got != tt.want {
t.Errorf("EscapedVersion() for %q = %q, want %q", tt.purl, got, tt.want)
}
// The router decodes each path segment, which must give back the
// version the page displays.
decoded, err := url.PathUnescape(got)
if err != nil {
t.Fatalf("PathUnescape(%q) failed: %v", got, err)
}
if decoded != v.Version() {
t.Errorf("round trip for %q = %q, want %q", tt.purl, decoded, v.Version())
}
})
}
}
func TestVersionDisplayPURL(t *testing.T) {
tests := []struct {
name string
purl string
want string
}{
{"simple", "pkg:npm/lodash@4.17.21", "pkg:npm/lodash@4.17.21"},
{
"encoded plus",
"pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1",
"pkg:deb/nmap@7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1",
},
{
"qualifier preserved",
"pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com",
"pkg:deb/nmap@7.91+dfsg1?repository_url=http%3A%2F%2Fexample.com",
},
// The namespace is encoded too: MakePURLString("npm", "@babel/core", …)
// produces "pkg:npm/%40babel/core@…".
{"encoded npm scope", "pkg:npm/%40babel/core@7.0.0", "pkg:npm/@babel/core@7.0.0"},
{"encoded scope without version", "pkg:npm/%40babel/core", "pkg:npm/@babel/core"},
{"no version", "pkg:npm/lodash", "pkg:npm/lodash"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
v := &Version{PURL: tt.purl}
if got := v.DisplayPURL(); got != tt.want {
t.Errorf("DisplayPURL() for %q = %q, want %q", tt.purl, got, tt.want)
}
})
}
}
// TestGetRecentlyCachedPackagesDecodesVersion guards the dashboard's "recently
// cached" list, which derives the version from the version PURL.
func TestGetRecentlyCachedPackagesDecodesVersion(t *testing.T) {
runWithBothDatabases(t, func(t *testing.T, db *DB) {
const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1"
if err := db.UpsertPackage(&Package{
PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap",
}); err != nil {
t.Fatalf("UpsertPackage failed: %v", err)
}
if err := db.UpsertVersion(&Version{
PURL: versionPURL, PackagePURL: "pkg:deb/nmap",
}); err != nil {
t.Fatalf("UpsertVersion failed: %v", err)
}
if err := db.UpsertArtifact(&Artifact{
VersionPURL: versionPURL,
Filename: "nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb",
UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb",
StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
}); err != nil {
t.Fatalf("UpsertArtifact failed: %v", err)
}
recent, err := db.GetRecentlyCachedPackages(10)
if err != nil {
t.Fatalf("GetRecentlyCachedPackages failed: %v", err)
}
if len(recent) != 1 {
t.Fatalf("expected 1 recent package, got %d", len(recent))
}
const want = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"
if recent[0].Version != want {
t.Errorf("Version = %q, want %q", recent[0].Version, want)
}
if recent[0].VersionPURL != versionPURL {
t.Errorf("VersionPURL = %q, want %q", recent[0].VersionPURL, versionPURL)
}
})
}

View file

@ -201,6 +201,43 @@ func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, ver
return results, nil return results, nil
} }
// BulkCheckVulnerabilities queries vulnerabilities for multiple package versions.
func (s *Service) BulkCheckVulnerabilities(ctx context.Context, packages []struct{ Ecosystem, Name, Version string }) (map[string][]VulnInfo, error) {
purls := make([]*purl.PURL, len(packages))
for i, pkg := range packages {
purls[i] = purl.MakePURL(pkg.Ecosystem, pkg.Name, pkg.Version)
}
vulnResults, err := s.vulnSource.QueryBatch(ctx, purls)
if err != nil {
return nil, err
}
result := make(map[string][]VulnInfo, len(packages))
for i, vulnList := range vulnResults {
pkg := packages[i]
key := purl.MakePURLString(pkg.Ecosystem, pkg.Name, pkg.Version)
var infos []VulnInfo
for _, v := range vulnList {
info := VulnInfo{
ID: v.ID,
Summary: v.Summary,
Severity: v.SeverityLevel(),
CVSSScore: v.CVSSScore(),
FixedVersion: v.FixedVersion(pkg.Ecosystem, pkg.Name),
}
for _, ref := range v.References {
info.References = append(info.References, ref.URL)
}
infos = append(infos, info)
}
result[key] = infos
}
return result, nil
}
// IsOutdated checks if a version is older than the latest version. // IsOutdated checks if a version is older than the latest version.
func (s *Service) IsOutdated(currentVersion, latestVersion string) bool { func (s *Service) IsOutdated(currentVersion, latestVersion string) bool {
if latestVersion == "" || currentVersion == "" { if latestVersion == "" || currentVersion == "" {
@ -251,6 +288,19 @@ func (s *Service) CategorizeLicense(license string) LicenseCategory {
return LicenseUnknown return LicenseUnknown
} }
// NormalizeLicense normalizes a license string to SPDX format.
func (s *Service) NormalizeLicense(license string) string {
if license == "" {
return ""
}
if normalized, err := spdx.NormalizeExpressionLax(license); err == nil {
return normalized
}
return license
}
// EnrichmentResult contains all enrichment data for a package version. // EnrichmentResult contains all enrichment data for a package version.
type EnrichmentResult struct { type EnrichmentResult struct {
Package *PackageInfo Package *PackageInfo

View file

@ -74,3 +74,25 @@ func TestCategorizeLicense(t *testing.T) {
} }
} }
} }
func TestNormalizeLicense(t *testing.T) {
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
svc := New(logger)
tests := []struct {
input string
expected string
}{
{"MIT", "MIT"},
{"Apache 2", "Apache-2.0"},
{"Apache-2.0", "Apache-2.0"},
{"", ""},
}
for _, tc := range tests {
result := svc.NormalizeLicense(tc.input)
if result != tc.expected {
t.Errorf("NormalizeLicense(%q) = %q, want %q", tc.input, result, tc.expected)
}
}
}

View file

@ -6,9 +6,10 @@ import (
"errors" "errors"
"fmt" "fmt"
"net/http" "net/http"
"net/url"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
) )
const ( const (
@ -29,18 +30,11 @@ type CargoHandler struct {
} }
// NewCargoHandler creates a new cargo protocol handler. // NewCargoHandler creates a new cargo protocol handler.
func NewCargoHandler(proxy *Proxy, proxyURL, indexURL, downloadURL string) *CargoHandler { func NewCargoHandler(proxy *Proxy, proxyURL string) *CargoHandler {
if strings.TrimSpace(indexURL) == "" {
indexURL = cargoUpstream
}
if strings.TrimSpace(downloadURL) == "" {
downloadURL = cargoDownloadBase
}
return &CargoHandler{ return &CargoHandler{
proxy: proxy, proxy: proxy,
indexURL: strings.TrimSuffix(indexURL, "/"), indexURL: cargoUpstream,
downloadURL: strings.TrimSuffix(downloadURL, "/"), downloadURL: cargoDownloadBase,
proxyURL: strings.TrimSuffix(proxyURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"),
} }
} }
@ -149,7 +143,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr
continue continue
} }
cratePURL := canonicalPackagePURL("cargo", crate.Name) cratePURL := purl.MakePURLString("cargo", crate.Name, "")
if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) { if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) {
h.proxy.Logger.Info("cooldown: filtering cargo version", h.proxy.Logger.Info("cooldown: filtering cargo version",
@ -197,17 +191,10 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
h.proxy.Logger.Info("cargo download request", h.proxy.Logger.Info("cargo download request",
"crate", name, "version", version, "filename", filename) "crate", name, "version", version, "filename", filename)
downloadURL := fmt.Sprintf( result, err := h.proxy.GetOrFetchArtifact(r.Context(), "cargo", name, version, filename)
"%s/%s/%s",
h.downloadURL,
url.PathEscape(name),
url.PathEscape(filename),
)
result, err := h.proxy.GetOrFetchArtifactFromURL(
r.Context(), "cargo", name, version, filename, downloadURL,
)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch crate") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch crate", http.StatusBadGateway)
return return
} }

View file

@ -2,7 +2,6 @@ package handler
import ( import (
"encoding/json" "encoding/json"
"io"
"log/slog" "log/slog"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -10,8 +9,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
"github.com/git-pkgs/registries/fetch"
) )
func cargoTestProxy() *Proxy { func cargoTestProxy() *Proxy {
@ -72,75 +70,6 @@ func TestCargoConfigEndpoint(t *testing.T) {
} }
} }
func TestCargoHandlerUsesConfiguredUpstreams(t *testing.T) {
t.Run("index", func(t *testing.T) {
var requestPath, authHeader string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestPath = r.URL.Path
authHeader = r.Header.Get("Authorization")
if authHeader != "Bearer cargo-token" {
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "text/plain")
_, _ = io.WriteString(w, `{"name":"serde","vers":"1.0.0"}`)
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
proxy.AuthForURL = func(string) (string, string) {
return "Authorization", "Bearer cargo-token"
}
h := NewCargoHandler(
proxy,
"http://proxy.test",
upstream.URL+"/index/",
"https://crates.example.test/files/",
)
req := httptest.NewRequest(http.MethodGet, "/se/rd/serde", nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
}
if requestPath != "/index/se/rd/serde" {
t.Errorf("upstream path = %q, want %q", requestPath, "/index/se/rd/serde")
}
if authHeader != "Bearer cargo-token" {
t.Errorf("Authorization = %q, want %q", authHeader, "Bearer cargo-token")
}
})
t.Run("download", func(t *testing.T) {
proxy, _, _, artifactFetcher := setupTestProxy(t)
artifactFetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("crate")),
ContentType: "application/gzip",
}
h := NewCargoHandler(
proxy,
"http://proxy.test",
"https://index.example.test/root/",
"https://crates.example.test/files/",
)
req := httptest.NewRequest(http.MethodGet, "/crates/serde/1.0.0/download", nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
}
want := "https://crates.example.test/files/serde/serde-1.0.0.crate"
if artifactFetcher.fetchedURL != want {
t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want)
}
})
}
func TestCargoIndexProxy(t *testing.T) { func TestCargoIndexProxy(t *testing.T) {
// Create a mock upstream index server // Create a mock upstream index server
indexContent := `{"name":"serde","vers":"1.0.0","deps":[],"cksum":"abc123"} indexContent := `{"name":"serde","vers":"1.0.0","deps":[],"cksum":"abc123"}

View file

@ -1,7 +1,6 @@
package handler package handler
import ( import (
"context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -10,6 +9,8 @@ import (
"path" "path"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
) )
const ( const (
@ -214,7 +215,7 @@ func deepCopyValue(v any) any {
// filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs // filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs
// for a single package's version list. // for a single package's version list.
func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any { func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any {
packagePURL := canonicalPackagePURL("composer", packageName) packagePURL := purl.MakePURLString("composer", packageName, "")
filtered := versionList[:0] filtered := versionList[:0]
for _, v := range versionList { for _, v := range versionList {
@ -306,127 +307,50 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request)
h.proxy.Logger.Info("composer download request", h.proxy.Logger.Info("composer download request",
"package", packageName, "version", version, "filename", filename) "package", packageName, "version", version, "filename", filename)
// We need to fetch the metadata to get the actual download URL since // We need to fetch the metadata to get the actual download URL
// Packagist URLs include a hash. Packagist serves dev versions (e.g. // since Packagist URLs include a hash
// "3.x-dev", "dev-master") from a separate "~dev" metadata file, while metaURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg)
// tagged releases live in the regular file. Try the file most likely to
// contain this version first, then fall back to the other so that both
// stable and dev versions resolve correctly.
metaURLs := h.metadataURLsForVersion(vendor, pkg, version)
h.proxy.Logger.Debug("resolving download URL", req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, metaURL, nil)
"package", packageName, "version", version,
"metadata_urls", metaURLs)
var downloadURL string
for _, metaURL := range metaURLs {
url, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version)
if err != nil {
h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL)
http.Error(w, "failed to fetch metadata", http.StatusBadGateway)
return
}
if url != "" {
downloadURL = url
break
}
}
if downloadURL == "" {
h.proxy.Logger.Debug("version not found in any metadata source",
"package", packageName, "version", version,
"tried_urls", metaURLs)
http.Error(w, "version not found", http.StatusNotFound)
return
}
h.proxy.Logger.Debug("resolved download URL",
"package", packageName, "version", version,
"download_url", downloadURL)
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") http.Error(w, "failed to create request", http.StatusInternalServerError)
return return
} }
ServeArtifact(w, result)
}
// isDevVersion reports whether a Composer version string refers to a
// development (unstable, branch) version rather than a tagged release.
// Composer formats these as either "dev-<branch>" (e.g. "dev-master") or
// "<alias>-dev" (e.g. "3.x-dev").
func isDevVersion(version string) bool {
return strings.HasPrefix(version, "dev-") || strings.HasSuffix(version, "-dev")
}
// metadataURLsForVersion returns the upstream metadata URLs to consult for a
// given version, in priority order. Dev versions are served from the "~dev"
// file, tagged releases from the regular file; the other file is included as a
// fallback so an unexpected classification still resolves.
func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string {
stable := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg)
dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", h.repoURL, vendor, pkg)
if isDevVersion(version) {
return []string{dev, stable}
}
return []string{stable, dev}
}
// findDownloadURLFromMetadata fetches a metadata document and returns the dist
// URL for the given version, or an empty string if the version is not present.
// An error is returned only on transport failure; a missing document (non-200)
// or a missing version both yield an empty string so the caller can fall back.
func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, error) {
h.proxy.Logger.Debug("fetching upstream metadata for download lookup",
"url", metaURL, "package", packageName, "version", version)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil)
if err != nil {
return "", err
}
resp, err := h.proxy.HTTPClient.Do(req) resp, err := h.proxy.HTTPClient.Do(req)
if err != nil { if err != nil {
return "", err h.proxy.Logger.Error("failed to fetch metadata", "error", err)
http.Error(w, "failed to fetch metadata", http.StatusBadGateway)
return
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
h.proxy.Logger.Debug("upstream metadata response",
"url", metaURL, "status", resp.StatusCode)
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return "", nil http.Error(w, "package not found", http.StatusNotFound)
return
} }
var metadata map[string]any var metadata map[string]any
if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
return "", err http.Error(w, "failed to parse metadata", http.StatusInternalServerError)
return
} }
// Expand minified Composer v2 format so that inherited fields (including // Find the download URL for this version
// dist) are present on every version entry. Without this, versions that downloadURL := h.findDownloadURL(metadata, packageName, version)
// inherit dist from a previous entry will appear to have no download URL. if downloadURL == "" {
if metadata["minified"] == "composer/2.0" { http.Error(w, "version not found", http.StatusNotFound)
h.proxy.Logger.Debug("expanding minified metadata", "url", metaURL) return
if packages, ok := metadata["packages"].(map[string]any); ok {
for pkgName, versions := range packages {
versionList, ok := versions.([]any)
if !ok {
continue
}
packages[pkgName] = expandMinifiedVersions(versionList)
}
}
} }
url := h.findDownloadURL(metadata, packageName, version) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL)
h.proxy.Logger.Debug("download URL lookup result", if err != nil {
"url", metaURL, "package", packageName, "version", version, h.proxy.Logger.Error("failed to get artifact", "error", err)
"download_url", url) http.Error(w, "failed to fetch package", http.StatusBadGateway)
return url, nil return
}
ServeArtifact(w, result)
} }
// findDownloadURL finds the dist URL for a specific version in metadata. // findDownloadURL finds the dist URL for a specific version in metadata.

View file

@ -1,16 +1,13 @@
package handler package handler
import ( import (
"context"
"encoding/json" "encoding/json"
"log/slog" "log/slog"
"net/http"
"net/http/httptest"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
) )
func TestComposerRewriteMetadata(t *testing.T) { func TestComposerRewriteMetadata(t *testing.T) {
@ -468,100 +465,6 @@ func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) {
} }
} }
// TestComposerDownloadDevVersionUsesDevMetadata is a regression test for the
// bug that made it impossible to install a *-dev dependency from dist.
//
// Packagist serves development versions (e.g. "3.x-dev", "dev-master") from a
// separate "{package}~dev.json" metadata file; the regular "{package}.json"
// file contains only tagged releases. The download handler used to fetch only
// the regular file, so it could never find the dist URL for a dev version and
// returned 404 — causing Composer to silently fall back to a git clone.
//
// This test serves both files from a mock upstream and asserts that:
// - the OLD behavior (regular file only) cannot resolve the dev version, and
// - the FIXED behavior (consulting the ~dev file) does.
func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) {
const (
pkg = "phpmd/phpmd"
vendor = "phpmd"
name = "phpmd"
version = "3.x-dev"
distURL = "https://api.github.com/repos/phpmd/phpmd/zipball/2a9217f60aaf27bf6ddad9188f254d020ab70745"
)
// Regular metadata: tagged releases only — no dev versions.
stableBody := `{
"packages": {
"phpmd/phpmd": [
{"version": "2.15.0", "dist": {"url": "https://example.com/2.15.0.zip", "type": "zip"}}
]
}
}`
// ~dev metadata: where the 3.x-dev version actually lives.
devBody := `{
"packages": {
"phpmd/phpmd": [
{"version": "3.x-dev", "dist": {"url": "` + distURL + `", "type": "zip"}}
]
}
}`
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/p2/phpmd/phpmd.json":
_, _ = w.Write([]byte(stableBody))
case "/p2/phpmd/phpmd~dev.json":
_, _ = w.Write([]byte(devBody))
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
h := &ComposerHandler{
proxy: testProxy(),
repoURL: srv.URL,
proxyURL: "http://localhost:8080",
}
ctx := context.Background()
// OLD behavior: fetching only the regular file fails to resolve the dev
// version, which is what produced the 404 before the fix.
stableURL := srv.URL + "/p2/phpmd/phpmd.json"
got, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version)
if err != nil {
t.Fatalf("unexpected error fetching regular metadata: %v", err)
}
if got != "" {
t.Fatalf("regular metadata unexpectedly contained dev version %q (got %q); "+
"the test no longer reproduces the original bug", version, got)
}
// FIXED behavior: the handler consults the ~dev file (it is first in the
// candidate list for dev versions) and resolves the dist URL.
urls := h.metadataURLsForVersion(vendor, name, version)
if len(urls) == 0 || !strings.HasSuffix(urls[0], "/p2/phpmd/phpmd~dev.json") {
t.Fatalf("dev version should consult the ~dev metadata file first, got %v", urls)
}
var resolved string
for _, u := range urls {
resolved, err = h.findDownloadURLFromMetadata(ctx, u, pkg, version)
if err != nil {
t.Fatalf("unexpected error fetching metadata %q: %v", u, err)
}
if resolved != "" {
break
}
}
if resolved != distURL {
t.Errorf("dev version dist URL = %q, want %q", resolved, distURL)
}
}
func TestComposerRewriteMetadataCooldown(t *testing.T) { func TestComposerRewriteMetadataCooldown(t *testing.T) {
now := time.Now() now := time.Now()
old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339) old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339)

View file

@ -84,7 +84,8 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request)
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch file") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch file", http.StatusBadGateway)
return return
} }
@ -121,7 +122,8 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request)
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch file") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch file", http.StatusBadGateway)
return return
} }

View file

@ -6,6 +6,8 @@ import (
"net/http" "net/http"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
) )
const ( const (
@ -72,7 +74,8 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, filename, upstreamURL) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, filename, upstreamURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return return
} }
@ -158,7 +161,7 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) {
return return
} }
body, err := h.proxy.ReadMetadata(resp.Body) body, err := ReadMetadata(resp.Body)
if err != nil { if err != nil {
http.Error(w, "failed to read response", http.StatusInternalServerError) http.Error(w, "failed to read response", http.StatusInternalServerError)
return return
@ -215,7 +218,7 @@ func (h *CondaHandler) applyCooldownFiltering(body []byte) ([]byte, error) {
continue continue
} }
packagePURL := canonicalPackagePURL("conda", name) packagePURL := purl.MakePURLString("conda", name, "")
if !h.proxy.Cooldown.IsAllowed("conda", packagePURL, publishedAt) { if !h.proxy.Cooldown.IsAllowed("conda", packagePURL, publishedAt) {
version, _ := entryMap["version"].(string) version, _ := entryMap["version"].(string)

View file

@ -8,7 +8,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
) )
func TestCondaParseFilename(t *testing.T) { func TestCondaParseFilename(t *testing.T) {

View file

@ -2,7 +2,6 @@ package handler
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -11,40 +10,31 @@ import (
) )
const ( const (
dockerHubRegistry = "https://registry-1.docker.io" dockerHubRegistry = "https://registry-1.docker.io"
blobMatchCount = 3 // full match + name + digest dockerHubAuth = "https://auth.docker.io"
manifestMatchCount = 3 // full match + name + reference blobMatchCount = 3 // full match + name + digest
tagsListMatchCount = 2 // full match + name manifestMatchCount = 3 // full match + name + reference
registrySelectorParts = 3 // upstream + name + repository tagsListMatchCount = 2 // full match + name
) )
// ContainerHandler handles OCI/Docker container registry protocol requests. // ContainerHandler handles OCI/Docker container registry protocol requests.
// It implements the OCI Distribution Spec for pulling images. // It implements the OCI Distribution Spec for pulling images.
// Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md // Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md
type ContainerHandler struct { type ContainerHandler struct {
proxy *Proxy proxy *Proxy
registryURL string registryURL string
proxyURL string authURL string
namedRegistries map[string]string proxyURL string
} }
// NewContainerHandler creates a new container registry protocol handler. // NewContainerHandler creates a new container registry protocol handler.
// Named registries are selected with the repository prefix func NewContainerHandler(proxy *Proxy, proxyURL string) *ContainerHandler {
// upstream/{name}/, leaving unprefixed requests compatible with the Docker Hub return &ContainerHandler{
// mirror behavior.
func NewContainerHandler(proxy *Proxy, proxyURL string, namedRegistries ...map[string]string) *ContainerHandler {
h := &ContainerHandler{
proxy: proxy, proxy: proxy,
registryURL: dockerHubRegistry, registryURL: dockerHubRegistry,
authURL: dockerHubAuth,
proxyURL: strings.TrimSuffix(proxyURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"),
} }
if len(namedRegistries) > 0 {
h.namedRegistries = make(map[string]string, len(namedRegistries[0]))
for name, registryURL := range namedRegistries[0] {
h.namedRegistries[name] = strings.TrimSuffix(registryURL, "/")
}
}
return h
} }
// Routes returns the HTTP handler for container registry requests. // Routes returns the HTTP handler for container registry requests.
@ -97,68 +87,48 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req
return return
} }
registryURL, upstreamName, cacheName, ok := h.registryForName(name) h.proxy.Logger.Info("container blob request", "name", name, "digest", digest)
if !ok {
h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry")
return
}
h.proxy.Logger.Info("container blob request", "name", upstreamName, "digest", digest) // Get auth token for upstream
token, err := h.getAuthToken(r.Context(), name, "pull")
filename := digest
cached, err := h.proxy.GetCachedArtifact(r.Context(), "oci", cacheName, digest, filename)
if err != nil { if err != nil {
h.proxy.Logger.Error("failed to check blob cache", "error", err) h.proxy.Logger.Error("failed to get auth token", "error", err)
h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to check blob cache") h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate")
return
}
if cached != nil {
w.Header().Set("Docker-Content-Digest", digest)
if cached.ContentType != "" {
w.Header().Set("Content-Type", cached.ContentType)
} else {
w.Header().Set("Content-Type", "application/octet-stream")
}
serveArtifact(w, r.Method, cached)
return return
} }
// For HEAD requests, just proxy to upstream // For HEAD requests, just proxy to upstream
if r.Method == http.MethodHead { if r.Method == http.MethodHead {
h.proxyBlobHead(w, r, registryURL, upstreamName, digest) h.proxyBlobHead(w, r, name, digest, token)
return return
} }
// Try to get from cache, or fetch from the authentication-aware upstream client. // Try to get from cache, or fetch from upstream with auth
result, err := h.proxy.GetOrFetchArtifactFromURL( filename := digest
headers := http.Header{"Authorization": {"Bearer " + token}}
result, err := h.proxy.GetOrFetchArtifactFromURLWithHeaders(
r.Context(), r.Context(),
"oci", "oci",
cacheName, name,
digest, // use digest as version digest, // use digest as version
filename, filename,
fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, upstreamName, digest), fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest),
headers,
) )
if err != nil { if err != nil {
if errors.Is(err, ErrUpstreamNotFound) {
h.containerError(w, http.StatusNotFound, "BLOB_UNKNOWN", "blob unknown to registry")
return
}
h.proxy.Logger.Error("failed to fetch blob", "error", err) h.proxy.Logger.Error("failed to fetch blob", "error", err)
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch blob") h.containerError(w, http.StatusBadGateway, "BLOB_UNKNOWN", "failed to fetch blob")
return return
} }
w.Header().Set("Docker-Content-Digest", digest) w.Header().Set("Docker-Content-Digest", digest)
if result.ContentType != "" { w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Type", result.ContentType)
} else {
w.Header().Set("Content-Type", "application/octet-stream")
}
ServeArtifact(w, result) ServeArtifact(w, result)
} }
// handleManifest serves immutable manifests from cache and revalidates mutable tags. // handleManifest proxies manifest requests to upstream.
// Manifests change when tags are updated, so we proxy these directly.
// Path format: {name}/manifests/{reference} // Path format: {name}/manifests/{reference}
func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request, path string) { func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request, path string) {
if r.Method != http.MethodGet && r.Method != http.MethodHead { if r.Method != http.MethodGet && r.Method != http.MethodHead {
@ -172,14 +142,58 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request
return return
} }
registryURL, upstreamName, _, ok := h.registryForName(name) h.proxy.Logger.Info("container manifest request", "name", name, "reference", reference)
if !ok {
h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") // Get auth token
token, err := h.getAuthToken(r.Context(), name, "pull")
if err != nil {
h.proxy.Logger.Error("failed to get auth token", "error", err)
h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate")
return return
} }
h.proxy.Logger.Info("container manifest request", "name", upstreamName, "reference", reference) // Proxy to upstream
h.serveManifest(w, r, registryURL, upstreamName, reference) upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", h.registryURL, name, reference)
req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil)
if err != nil {
h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request")
return
}
req.Header.Set("Authorization", "Bearer "+token)
// Forward Accept header for content negotiation
if accept := r.Header.Get("Accept"); accept != "" {
req.Header.Set("Accept", accept)
} else {
// Default accept headers for manifests
req.Header.Set("Accept", strings.Join([]string{
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.oci.image.index.v1+json",
"application/vnd.docker.distribution.manifest.v2+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v1+prettyjws",
}, ", "))
}
resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
h.proxy.Logger.Error("failed to fetch manifest", "error", err)
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
return
}
defer func() { _ = resp.Body.Close() }()
// Copy relevant headers
for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag"} {
if v := resp.Header.Get(header); v != "" {
w.Header().Set(header, v)
}
}
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
} }
// handleTagsList proxies tag list requests to upstream. // handleTagsList proxies tag list requests to upstream.
@ -195,13 +209,14 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request
return return
} }
registryURL, upstreamName, _, ok := h.registryForName(name) // Get auth token
if !ok { token, err := h.getAuthToken(r.Context(), name, "pull")
h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") if err != nil {
h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate")
return return
} }
upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", registryURL, upstreamName) upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", h.registryURL, name)
if r.URL.RawQuery != "" { if r.URL.RawQuery != "" {
upstreamURL += "?" + r.URL.RawQuery upstreamURL += "?" + r.URL.RawQuery
} }
@ -212,6 +227,8 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request
return return
} }
req.Header.Set("Authorization", "Bearer "+token)
resp, err := h.proxy.HTTPClient.Do(req) resp, err := h.proxy.HTTPClient.Do(req)
if err != nil { if err != nil {
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
@ -224,9 +241,46 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request
_, _ = io.Copy(w, resp.Body) _, _ = io.Copy(w, resp.Body)
} }
// getAuthToken gets a bearer token for the specified repository.
// Docker Hub requires auth even for public images.
func (h *ContainerHandler) getAuthToken(_ interface{ Done() <-chan struct{} }, repository, action string) (string, error) {
// For Docker Hub: https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull
authURL := fmt.Sprintf("%s/token?service=registry.docker.io&scope=repository:%s:%s",
h.authURL, repository, action)
req, err := http.NewRequest(http.MethodGet, authURL, nil)
if err != nil {
return "", err
}
resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
return "", err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("auth failed with status %d", resp.StatusCode)
}
var tokenResp struct {
Token string `json:"token"`
AccessToken string `json:"access_token"`
}
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
return "", err
}
if tokenResp.Token != "" {
return tokenResp.Token, nil
}
return tokenResp.AccessToken, nil
}
// proxyBlobHead handles HEAD requests for blobs. // proxyBlobHead handles HEAD requests for blobs.
func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, registryURL, name, digest string) { func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, name, digest, token string) {
upstreamURL := fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, name, digest) upstreamURL := fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest)
req, err := http.NewRequestWithContext(r.Context(), http.MethodHead, upstreamURL, nil) req, err := http.NewRequestWithContext(r.Context(), http.MethodHead, upstreamURL, nil)
if err != nil { if err != nil {
@ -234,6 +288,8 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request,
return return
} }
req.Header.Set("Authorization", "Bearer "+token)
resp, err := h.proxy.HTTPClient.Do(req) resp, err := h.proxy.HTTPClient.Do(req)
if err != nil { if err != nil {
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
@ -250,24 +306,6 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request,
w.WriteHeader(resp.StatusCode) w.WriteHeader(resp.StatusCode)
} }
// registryForName resolves a client-visible OCI repository name to an upstream
// registry and its repository name. Named upstreams use upstream/{name}/ as a
// reserved prefix; all other names continue to target Docker Hub.
func (h *ContainerHandler) registryForName(name string) (registryURL, upstreamName, cacheName string, ok bool) {
parts := strings.SplitN(name, "/", registrySelectorParts)
if len(parts) >= 2 && parts[0] == "upstream" {
if len(parts) != registrySelectorParts || parts[2] == "" {
return "", "", "", false
}
registryURL, ok = h.namedRegistries[parts[1]]
if !ok || registryURL == "" {
return "", "", "", false
}
return registryURL, parts[2], name, true
}
return h.registryURL, name, name, true
}
// containerError writes an OCI-compliant error response. // containerError writes an OCI-compliant error response.
func (h *ContainerHandler) containerError(w http.ResponseWriter, status int, code, message string) { func (h *ContainerHandler) containerError(w http.ResponseWriter, status int, code, message string) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")

View file

@ -1,251 +0,0 @@
package handler
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"io"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/git-pkgs/proxy/internal/database"
)
const (
containerManifestCacheEcosystem = "oci-manifest"
containerStaleWarning = `110 - "Response is Stale"`
)
var manifestDigestReferencePattern = regexp.MustCompile(`^[a-z0-9]+:[a-f0-9]+$`)
type cachedContainerManifest struct {
body []byte
contentType string
contentDigest string
etag string
size int64
fetchedAt time.Time
}
func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, reference string) {
accept := containerManifestAccept(r)
cacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept)
cached, err := h.loadContainerManifest(r.Context(), cacheKey)
if err != nil {
h.proxy.Logger.Warn("failed to read cached container manifest", "error", err)
cached = nil
}
immutable := manifestDigestReferencePattern.MatchString(reference)
if cached != nil && (immutable || h.containerManifestFresh(cached)) {
writeContainerManifest(w, r.Method, cached, false)
return
}
upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", registryURL, name, reference)
req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil)
if err != nil {
h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request")
return
}
req.Header.Set("Accept", accept)
if cached != nil && cached.etag != "" {
req.Header.Set("If-None-Match", cached.etag)
}
resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
h.serveStaleManifestOrError(w, r, cached, err)
return
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotModified && cached != nil {
cached.fetchedAt = time.Now()
if err := h.storeContainerManifest(r.Context(), cacheKey, cached); err != nil {
h.proxy.Logger.Warn("failed to refresh cached container manifest", "error", err)
}
writeContainerManifest(w, r.Method, cached, false)
return
}
if resp.StatusCode != http.StatusOK {
if cached != nil && shouldServeStaleManifest(resp.StatusCode) {
writeContainerManifest(w, r.Method, cached, true)
return
}
copyContainerManifestHeaders(w.Header(), resp.Header)
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
return
}
if r.Method == http.MethodHead {
copyContainerManifestHeaders(w.Header(), resp.Header)
w.WriteHeader(http.StatusOK)
return
}
body, err := h.proxy.ReadMetadata(resp.Body)
if err != nil {
h.serveStaleManifestOrError(w, r, cached, fmt.Errorf("reading manifest: %w", err))
return
}
manifest := &cachedContainerManifest{
body: body,
contentType: resp.Header.Get("Content-Type"),
contentDigest: resp.Header.Get("Docker-Content-Digest"),
etag: resp.Header.Get("ETag"),
size: int64(len(body)),
fetchedAt: time.Now(),
}
if manifest.contentDigest == "" {
manifest.contentDigest = sha256Digest(body)
}
if err := h.storeContainerManifest(r.Context(), cacheKey, manifest); err != nil {
h.proxy.Logger.Warn("failed to cache container manifest", "error", err)
}
if manifest.contentDigest != reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) {
digestKey := h.containerManifestCacheKey(registryURL, name, manifest.contentDigest, accept)
if err := h.storeContainerManifest(r.Context(), digestKey, manifest); err != nil {
h.proxy.Logger.Warn("failed to cache container manifest by digest", "error", err)
}
}
writeContainerManifest(w, r.Method, manifest, false)
}
func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, cached *cachedContainerManifest, err error) {
if cached != nil {
h.proxy.Logger.Warn("upstream manifest fetch failed, serving stale cache", "error", err)
writeContainerManifest(w, r.Method, cached, true)
return
}
h.proxy.Logger.Error("failed to fetch manifest", "error", err)
h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
}
func (h *ContainerHandler) containerManifestFresh(manifest *cachedContainerManifest) bool {
return h.proxy.MetadataTTL > 0 && !manifest.fetchedAt.IsZero() && time.Since(manifest.fetchedAt) < h.proxy.MetadataTTL
}
func (h *ContainerHandler) containerManifestCacheKey(registryURL, name, reference, accept string) string {
identity := strings.Join([]string{registryURL, name, reference, accept}, "\x00")
sum := sha256.Sum256([]byte(identity))
return hex.EncodeToString(sum[:])
}
func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, error) {
if h.proxy.DB == nil || h.proxy.Storage == nil {
return nil, nil
}
entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, cacheKey)
if err != nil || entry == nil {
return nil, err
}
reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath)
if err != nil {
return nil, nil
}
defer func() { _ = reader.Close() }()
body, err := h.proxy.ReadMetadata(reader)
if err != nil {
return nil, err
}
manifest := &cachedContainerManifest{body: body, size: int64(len(body))}
if entry.ContentType.Valid {
manifest.contentType = entry.ContentType.String
}
if entry.ContentDigest.Valid {
manifest.contentDigest = entry.ContentDigest.String
} else {
manifest.contentDigest = sha256Digest(body)
}
if entry.ETag.Valid {
manifest.etag = entry.ETag.String
}
if entry.Size.Valid {
manifest.size = entry.Size.Int64
}
if entry.FetchedAt.Valid {
manifest.fetchedAt = entry.FetchedAt.Time
}
return manifest, nil
}
func (h *ContainerHandler) storeContainerManifest(ctx context.Context, cacheKey string, manifest *cachedContainerManifest) error {
if h.proxy.DB == nil || h.proxy.Storage == nil {
return nil
}
storagePath := metadataStoragePath(containerManifestCacheEcosystem, cacheKey)
size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(manifest.body))
if err != nil {
return fmt.Errorf("storing manifest: %w", err)
}
manifest.size = size
return h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{
Ecosystem: containerManifestCacheEcosystem,
Name: cacheKey,
StoragePath: storagePath,
ETag: sql.NullString{String: manifest.etag, Valid: manifest.etag != ""},
ContentType: sql.NullString{String: manifest.contentType, Valid: manifest.contentType != ""},
ContentDigest: sql.NullString{String: manifest.contentDigest, Valid: manifest.contentDigest != ""},
Size: sql.NullInt64{Int64: size, Valid: true},
FetchedAt: sql.NullTime{Time: manifest.fetchedAt, Valid: !manifest.fetchedAt.IsZero()},
})
}
func writeContainerManifest(w http.ResponseWriter, method string, manifest *cachedContainerManifest, stale bool) {
if manifest.contentType != "" {
w.Header().Set("Content-Type", manifest.contentType)
}
w.Header().Set("Content-Length", strconv.FormatInt(manifest.size, 10))
if manifest.contentDigest != "" {
w.Header().Set("Docker-Content-Digest", manifest.contentDigest)
}
if manifest.etag != "" {
w.Header().Set("ETag", manifest.etag)
}
if stale {
w.Header().Set("Warning", containerStaleWarning)
}
w.WriteHeader(http.StatusOK)
if method != http.MethodHead {
_, _ = w.Write(manifest.body)
}
}
func containerManifestAccept(r *http.Request) string {
if accept := r.Header.Get("Accept"); accept != "" {
return accept
}
return strings.Join([]string{
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.oci.image.index.v1+json",
"application/vnd.docker.distribution.manifest.v2+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v1+prettyjws",
}, ", ")
}
func copyContainerManifestHeaders(destination, source http.Header) {
for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag", "WWW-Authenticate"} {
if value := source.Get(header); value != "" {
destination.Set(header, value)
}
}
}
func shouldServeStaleManifest(status int) bool {
return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError
}
func sha256Digest(body []byte) string {
digest := sha256.Sum256(body)
return "sha256:" + hex.EncodeToString(digest[:])
}

View file

@ -1,15 +1,16 @@
package handler package handler
import ( import (
"bytes"
"context"
"encoding/json" "encoding/json"
"io" "io"
"log/slog"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strconv"
"testing" "testing"
"time"
upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/registries/fetch" "github.com/git-pkgs/registries/fetch"
) )
@ -134,568 +135,90 @@ func TestContainerHandler_parseTagsListPath(t *testing.T) {
} }
} }
func TestContainerHandler_NamedOCIRegistryServesHelmArtifacts(t *testing.T) { func TestContainerHandler_BlobDownload_CachesWithAuth(t *testing.T) {
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" // Set up a mock auth server that returns a token
manifest := `{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.helm.config.v1+json"},"layers":[{"mediaType":"application/vnd.cncf.helm.chart.content.v1.tar+gzip","digest":"` + digest + `"}]}` authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v2/owner/demo/manifests/1.0.0":
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
w.Header().Set("Docker-Content-Digest", digest)
_, _ = io.WriteString(w, manifest)
case "/v2/owner/demo/blobs/" + digest:
w.Header().Set("Content-Type", "application/vnd.cncf.helm.chart.content.v1.tar+gzip")
_, _ = io.WriteString(w, "chart archive")
default:
http.NotFound(w, r)
}
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
proxy.Fetcher = fetcher
t.Cleanup(func() { _ = fetcher.Close() })
h := NewContainerHandler(proxy, "http://proxy.example", map[string]string{"ghcr": upstream.URL})
manifestResponse := httptest.NewRecorder()
h.Routes().ServeHTTP(manifestResponse,
httptest.NewRequest(http.MethodGet, "/upstream/ghcr/owner/demo/manifests/1.0.0", nil))
if manifestResponse.Code != http.StatusOK {
t.Fatalf("manifest status = %d, want 200: %s", manifestResponse.Code, manifestResponse.Body.String())
}
if got := manifestResponse.Header().Get("Content-Type"); got != "application/vnd.oci.image.manifest.v1+json" {
t.Errorf("manifest Content-Type = %q", got)
}
blobResponse := httptest.NewRecorder()
h.Routes().ServeHTTP(blobResponse,
httptest.NewRequest(http.MethodGet, "/upstream/ghcr/owner/demo/blobs/"+digest, nil))
if blobResponse.Code != http.StatusOK {
t.Fatalf("blob status = %d, want 200: %s", blobResponse.Code, blobResponse.Body.String())
}
if got := blobResponse.Header().Get("Content-Type"); got != "application/vnd.cncf.helm.chart.content.v1.tar+gzip" {
t.Errorf("blob Content-Type = %q", got)
}
}
func TestContainerHandler_BlobDownload_DiscoversBearerChallenge(t *testing.T) {
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
registryRequests := 0
tokenRequests := 0
var upstream *httptest.Server
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/token":
tokenRequests++
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"token": "discovered-token",
"expires_in": 3600,
})
case "/v2/library/nginx/blobs/" + digest:
registryRequests++
if r.Header.Get("Authorization") != "Bearer discovered-token" {
w.Header().Set("WWW-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="registry.test",scope="repository:library/nginx:pull"`)
http.Error(w, "authentication required", http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = io.WriteString(w, "upstream blob")
default:
http.NotFound(w, r)
}
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
authTransport := upstreamhttp.NewTransport(http.DefaultTransport, nil)
client := &http.Client{Transport: authTransport}
artifactFetcher := fetch.NewFetcher(
fetch.WithHTTPClient(client),
fetch.WithMaxRetries(0),
)
t.Cleanup(func() { _ = artifactFetcher.Close() })
proxy.Fetcher = artifactFetcher
proxy.HTTPClient = client
h := &ContainerHandler{
proxy: proxy,
registryURL: upstream.URL,
proxyURL: "http://localhost:8080",
}
for range 2 {
req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
}
if got := w.Body.String(); got != "upstream blob" {
t.Errorf("body = %q, want %q", got, "upstream blob")
}
}
if tokenRequests != 1 {
t.Errorf("token requests = %d, want 1", tokenRequests)
}
if registryRequests != 2 {
t.Errorf("registry requests = %d, want 2", registryRequests)
}
}
func TestContainerHandler_CachedImagePullSurvivesRegistryAndTokenOutages(t *testing.T) {
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`
blob := "cached image blob"
registryAvailable := true
tokenAvailable := true
registryRequests := 0
tokenRequests := 0
tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
tokenRequests++
if !tokenAvailable {
http.Error(w, "token service unavailable", http.StatusServiceUnavailable)
return
}
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{ _ = json.NewEncoder(w).Encode(map[string]string{"token": "test-token-123"})
"token": "discovered-token",
"expires_in": 3600,
})
})) }))
defer tokenServer.Close() defer authServer.Close()
registryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Set up mock fetcher that captures headers
registryRequests++ var capturedHeaders http.Header
if !registryAvailable { mf := &mockFetcherWithHeaders{
http.Error(w, "registry unavailable", http.StatusServiceUnavailable) fetchFn: func(_ context.Context, _ string, headers http.Header) (*fetch.Artifact, error) {
return capturedHeaders = headers
} return &fetch.Artifact{
if r.Header.Get("Authorization") != "Bearer discovered-token" { Body: io.NopCloser(bytes.NewReader([]byte("blob-content"))),
w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenServer.URL+`",service="registry.test",scope="repository:library/nginx:pull"`) Size: 12,
http.Error(w, "authentication required", http.StatusUnauthorized) ContentType: "application/octet-stream",
return }, nil
} },
switch r.URL.Path {
case "/v2/library/nginx/manifests/latest":
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
w.Header().Set("Docker-Content-Digest", digest)
_, _ = io.WriteString(w, manifest)
case "/v2/library/nginx/blobs/" + digest:
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = io.WriteString(w, blob)
default:
http.NotFound(w, r)
}
}))
defer registryServer.Close()
warmProxy, db, store, _ := setupTestProxy(t)
warmClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)}
warmFetcher := fetch.NewFetcher(
fetch.WithHTTPClient(warmClient),
fetch.WithMaxRetries(0),
)
t.Cleanup(func() { _ = warmFetcher.Close() })
warmProxy.Fetcher = warmFetcher
warmProxy.HTTPClient = warmClient
warmProxy.MetadataTTL = time.Hour
warmHandler := (&ContainerHandler{
proxy: warmProxy,
registryURL: registryServer.URL,
proxyURL: "http://localhost:8080",
}).Routes()
for _, request := range []struct {
path string
body string
}{
{path: "/library/nginx/manifests/latest", body: manifest},
{path: "/library/nginx/blobs/" + digest, body: blob},
} {
response := httptest.NewRecorder()
warmHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil))
if response.Code != http.StatusOK {
t.Fatalf("warming %s: status = %d, want %d; body: %s", request.path, response.Code, http.StatusOK, response.Body.String())
}
if got := response.Body.String(); got != request.body {
t.Fatalf("warming %s: body = %q, want %q", request.path, got, request.body)
}
} }
warmRegistryRequests := registryRequests dir := t.TempDir()
warmTokenRequests := tokenRequests db, err := database.Create(dir + "/test.db")
registryAvailable = false if err != nil {
tokenAvailable = false t.Fatalf("failed to create test database: %v", err)
offlineClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)}
offlineFetcher := fetch.NewFetcher(
fetch.WithHTTPClient(offlineClient),
fetch.WithMaxRetries(0),
)
t.Cleanup(func() { _ = offlineFetcher.Close() })
offlineProxy := NewProxy(db, store, offlineFetcher, fetch.NewResolver(), warmProxy.Logger)
offlineProxy.HTTPClient = offlineClient
offlineProxy.MetadataTTL = time.Hour
offlineHandler := (&ContainerHandler{
proxy: offlineProxy,
registryURL: registryServer.URL,
proxyURL: "http://localhost:8080",
}).Routes()
for _, request := range []struct {
name string
path string
body string
}{
{name: "tag manifest", path: "/library/nginx/manifests/latest", body: manifest},
{name: "digest manifest", path: "/library/nginx/manifests/" + digest, body: manifest},
{name: "blob", path: "/library/nginx/blobs/" + digest, body: blob},
} {
t.Run(request.name, func(t *testing.T) {
response := httptest.NewRecorder()
offlineHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil))
if response.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", response.Code, http.StatusOK, response.Body.String())
}
if got := response.Body.String(); got != request.body {
t.Errorf("body = %q, want %q", got, request.body)
}
if got := response.Header().Get("Docker-Content-Digest"); got != digest {
t.Errorf("Docker-Content-Digest = %q, want %q", got, digest)
}
})
} }
t.Cleanup(func() { _ = db.Close() })
if registryRequests != warmRegistryRequests { store := newMockStorage()
t.Errorf("offline registry requests = %d, want 0", registryRequests-warmRegistryRequests) logger := slog.New(slog.NewTextHandler(io.Discard, nil))
proxy := &Proxy{
DB: db,
Storage: store,
Fetcher: mf,
Logger: logger,
HTTPClient: &http.Client{},
} }
if tokenRequests != warmTokenRequests {
t.Errorf("offline token requests = %d, want 0", tokenRequests-warmTokenRequests)
}
}
func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) {
proxy, db, store, fetcher := setupTestProxy(t)
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob")
upstreamRequests := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
upstreamRequests++
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
}))
defer upstream.Close()
h := &ContainerHandler{ h := &ContainerHandler{
proxy: proxy, proxy: proxy,
registryURL: upstream.URL, registryURL: "https://registry-1.docker.io",
authURL: authServer.URL,
proxyURL: "http://localhost:8080", proxyURL: "http://localhost:8080",
} }
req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) handler := h.Routes()
req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req) handler.ServeHTTP(w, req)
if w.Code != http.StatusOK { if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) t.Errorf("got status %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
} }
if got := w.Body.String(); got != "cached blob" {
t.Errorf("body = %q, want %q", got, "cached blob") // Verify auth header was passed to the fetcher
if capturedHeaders == nil {
t.Fatal("expected headers to be passed to fetcher, got nil")
} }
if upstreamRequests != 0 { auth := capturedHeaders.Get("Authorization")
t.Errorf("upstream requests = %d, want 0", upstreamRequests) if auth != "Bearer test-token-123" {
t.Errorf("Authorization = %q, want %q", auth, "Bearer test-token-123")
} }
if fetcher.fetchCalled {
t.Error("fetcher should not be called on cache hit") // Verify response headers
if got := w.Header().Get("Docker-Content-Digest"); got != "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" {
t.Errorf("Docker-Content-Digest = %q, want digest", got)
} }
} }
func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) { // mockFetcherWithHeaders captures headers passed to FetchWithHeaders.
proxy, db, store, fetcher := setupTestProxy(t) type mockFetcherWithHeaders struct {
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" fetchFn func(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error)
seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob")
upstreamRequests := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
upstreamRequests++
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
}))
defer upstream.Close()
proxy.HTTPClient = upstream.Client()
h := &ContainerHandler{
proxy: proxy,
registryURL: upstream.URL,
proxyURL: "http://localhost:8080",
}
req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
}
if got := w.Header().Get("Docker-Content-Digest"); got != digest {
t.Errorf("Docker-Content-Digest = %q, want %q", got, digest)
}
if got := w.Header().Get("Content-Length"); got != "11" {
t.Errorf("Content-Length = %q, want %q", got, "11")
}
if w.Body.Len() != 0 {
t.Errorf("HEAD response body length = %d, want 0", w.Body.Len())
}
if upstreamRequests != 0 {
t.Errorf("upstream requests = %d, want 0", upstreamRequests)
}
if fetcher.fetchCalled {
t.Error("fetcher should not be called on cache hit")
}
} }
func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) { func (f *mockFetcherWithHeaders) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) {
proxy, db, store, fetcher := setupTestProxy(t) return f.FetchWithHeaders(ctx, url, nil)
digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd"
seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob")
store.signedURL = "https://storage.example.test/cached-blob?signature=test"
proxy.DirectServe = true
h := &ContainerHandler{
proxy: proxy,
registryURL: "https://registry.example.test",
proxyURL: "http://localhost:8080",
}
req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusFound {
t.Fatalf("status = %d, want %d", w.Code, http.StatusFound)
}
if got := w.Header().Get("Location"); got != store.signedURL {
t.Errorf("Location = %q, want %q", got, store.signedURL)
}
wantETag := `"` + sha256Hex("cached blob") + `"`
if got := w.Header().Get("ETag"); got != wantETag {
t.Errorf("ETag = %q, want %q", got, wantETag)
}
if w.Body.Len() != 0 {
t.Errorf("HEAD response body length = %d, want 0", w.Body.Len())
}
if fetcher.fetchCalled {
t.Error("fetcher should not be called on cache hit")
}
} }
func TestContainerHandler_ManifestByDigest_CacheHitSkipsUpstream(t *testing.T) { func (f *mockFetcherWithHeaders) FetchWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) {
digest := "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" return f.fetchFn(ctx, url, headers)
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`
upstreamAvailable := true
upstreamRequests := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstreamRequests++
if !upstreamAvailable {
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
return
}
if r.URL.Path != "/v2/library/nginx/manifests/"+digest {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
w.Header().Set("Docker-Content-Digest", digest)
w.Header().Set("ETag", `"manifest-etag"`)
if r.Method != http.MethodHead {
_, _ = io.WriteString(w, manifest)
}
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"}
first := httptest.NewRecorder()
h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil))
if first.Code != http.StatusOK {
t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String())
}
if first.Body.String() != manifest {
t.Fatalf("initial body = %q, want %q", first.Body.String(), manifest)
}
upstreamAvailable = false
second := httptest.NewRecorder()
h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil))
if second.Code != http.StatusOK {
t.Fatalf("cached status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String())
}
if second.Body.String() != manifest {
t.Errorf("cached body = %q, want %q", second.Body.String(), manifest)
}
if got := second.Header().Get("Docker-Content-Digest"); got != digest {
t.Errorf("cached Docker-Content-Digest = %q, want %q", got, digest)
}
head := httptest.NewRecorder()
h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/"+digest, nil))
if head.Code != http.StatusOK {
t.Fatalf("cached HEAD status = %d, want %d", head.Code, http.StatusOK)
}
wantLength := strconv.Itoa(len(manifest))
if got := head.Header().Get("Content-Length"); got != wantLength {
t.Errorf("cached HEAD Content-Length = %q, want %q", got, wantLength)
}
if head.Body.Len() != 0 {
t.Errorf("cached HEAD body length = %d, want 0", head.Body.Len())
}
if upstreamRequests != 1 {
t.Errorf("upstream requests = %d, want 1", upstreamRequests)
}
} }
func TestContainerHandler_ManifestByTag_UsesStaleCacheOnUpstreamFailure(t *testing.T) { func (f *mockFetcherWithHeaders) Head(_ context.Context, _ string) (int64, string, error) {
digest := "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" return 0, "", nil
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json"}`
upstreamAvailable := true
upstreamRequests := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
upstreamRequests++
if !upstreamAvailable {
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
return
}
w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
w.Header().Set("Docker-Content-Digest", digest)
_, _ = io.WriteString(w, manifest)
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
proxy.MetadataTTL = 0
h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"}
first := httptest.NewRecorder()
h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil))
if first.Code != http.StatusOK {
t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String())
}
upstreamAvailable = false
second := httptest.NewRecorder()
h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil))
if second.Code != http.StatusOK {
t.Fatalf("stale status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String())
}
if second.Body.String() != manifest {
t.Errorf("stale body = %q, want %q", second.Body.String(), manifest)
}
if got := second.Header().Get("Warning"); got != `110 - "Response is Stale"` {
t.Errorf("Warning = %q, want stale warning", got)
}
if got := second.Header().Get("Docker-Content-Digest"); got != digest {
t.Errorf("stale Docker-Content-Digest = %q, want %q", got, digest)
}
if upstreamRequests != 2 {
t.Errorf("upstream requests = %d, want 2", upstreamRequests)
}
}
func TestContainerHandler_ManifestByTag_CachesDigestAlias(t *testing.T) {
digest := "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`
upstreamAvailable := true
upstreamRequests := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstreamRequests++
if !upstreamAvailable {
http.Error(w, "upstream unavailable", http.StatusServiceUnavailable)
return
}
if r.URL.Path != "/v2/library/nginx/manifests/latest" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
w.Header().Set("Docker-Content-Digest", digest)
_, _ = io.WriteString(w, manifest)
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"}
first := httptest.NewRecorder()
h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil))
if first.Code != http.StatusOK {
t.Fatalf("tag status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String())
}
upstreamAvailable = false
byDigest := httptest.NewRecorder()
h.Routes().ServeHTTP(byDigest, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil))
if byDigest.Code != http.StatusOK {
t.Fatalf("digest status = %d, want %d; body: %s", byDigest.Code, http.StatusOK, byDigest.Body.String())
}
if byDigest.Body.String() != manifest {
t.Errorf("digest body = %q, want %q", byDigest.Body.String(), manifest)
}
if got := byDigest.Header().Get("Docker-Content-Digest"); got != digest {
t.Errorf("Docker-Content-Digest = %q, want %q", got, digest)
}
if upstreamRequests != 1 {
t.Errorf("upstream requests = %d, want 1", upstreamRequests)
}
}
func TestContainerHandler_ManifestByTag_StaleHeadChecksUpstream(t *testing.T) {
oldDigest := "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
newDigest := "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
currentDigest := oldDigest
upstreamRequests := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstreamRequests++
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
w.Header().Set("Docker-Content-Digest", currentDigest)
w.Header().Set("ETag", `"`+currentDigest+`"`)
if r.Method != http.MethodHead {
_, _ = io.WriteString(w, `{"schemaVersion":2}`)
}
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
proxy.MetadataTTL = 0
h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"}
first := httptest.NewRecorder()
h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil))
if first.Code != http.StatusOK {
t.Fatalf("initial status = %d, want %d", first.Code, http.StatusOK)
}
currentDigest = newDigest
head := httptest.NewRecorder()
h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/latest", nil))
if head.Code != http.StatusOK {
t.Fatalf("HEAD status = %d, want %d", head.Code, http.StatusOK)
}
if got := head.Header().Get("Docker-Content-Digest"); got != newDigest {
t.Errorf("Docker-Content-Digest = %q, want %q", got, newDigest)
}
if upstreamRequests != 2 {
t.Errorf("upstream requests = %d, want 2", upstreamRequests)
}
} }
func TestContainerHandler_Routes_VersionCheck(t *testing.T) { func TestContainerHandler_Routes_VersionCheck(t *testing.T) {

View file

@ -72,7 +72,8 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, version, filename, upstreamURL) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, version, filename, upstreamURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return return
} }
@ -106,7 +107,8 @@ func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Reques
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, storageVersion, filename, upstreamURL) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, storageVersion, filename, upstreamURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return return
} }

View file

@ -21,13 +21,10 @@ type DebianHandler struct {
} }
// NewDebianHandler creates a new Debian/APT protocol handler. // NewDebianHandler creates a new Debian/APT protocol handler.
func NewDebianHandler(proxy *Proxy, proxyURL string, upstreamURL string) *DebianHandler { func NewDebianHandler(proxy *Proxy, proxyURL string) *DebianHandler {
if upstreamURL == "" {
upstreamURL = debianUpstream
}
return &DebianHandler{ return &DebianHandler{
proxy: proxy, proxy: proxy,
upstreamURL: strings.TrimSuffix(upstreamURL, "/"), upstreamURL: debianUpstream,
proxyURL: strings.TrimSuffix(proxyURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"),
} }
} }
@ -84,7 +81,8 @@ func (h *DebianHandler) handlePackageDownload(w http.ResponseWriter, r *http.Req
result, err := h.proxy.GetOrFetchArtifactFromURL( result, err := h.proxy.GetOrFetchArtifactFromURL(
r.Context(), "deb", name, version, filename, downloadURL) r.Context(), "deb", name, version, filename, downloadURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") h.proxy.Logger.Error("failed to get debian package", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return return
} }

View file

@ -12,17 +12,12 @@ func TestDebianHandler_parsePoolPath(t *testing.T) {
{"pool/main/libn/libncurses/libncurses6_6.2-1_amd64.deb", "libncurses6", "6.2-1", "amd64"}, {"pool/main/libn/libncurses/libncurses6_6.2-1_amd64.deb", "libncurses6", "6.2-1", "amd64"},
{"pool/contrib/v/virtualbox/virtualbox_6.1.38-1_amd64.deb", "virtualbox", "6.1.38-1", "amd64"}, {"pool/contrib/v/virtualbox/virtualbox_6.1.38-1_amd64.deb", "virtualbox", "6.1.38-1", "amd64"},
{"pool/main/g/git/git_2.39.2-1_arm64.deb", "git", "2.39.2-1", "arm64"}, {"pool/main/g/git/git_2.39.2-1_arm64.deb", "git", "2.39.2-1", "arm64"},
{
"pool/universe/n/nmap/nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb",
"nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64",
},
{"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"},
{"invalid/path", "", "", ""}, {"invalid/path", "", "", ""},
{"pool/main/n/nginx/nginx.deb", "", "", ""}, {"pool/main/n/nginx/nginx.deb", "", "", ""},
}) })
} }
func TestDebianHandler_Routes(t *testing.T) { func TestDebianHandler_Routes(t *testing.T) {
h := NewDebianHandler(nil, "http://localhost:8080", "") h := NewDebianHandler(nil, "http://localhost:8080")
assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd") assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd")
} }

View file

@ -49,7 +49,7 @@ func seedPackageWithPURL(t *testing.T, db *database.DB, store *mockStorage, ecos
Filename: filename, Filename: filename,
UpstreamURL: "https://example.com/" + filename, UpstreamURL: "https://example.com/" + filename,
StoragePath: sql.NullString{String: storagePath, Valid: true}, StoragePath: sql.NullString{String: storagePath, Valid: true},
ContentHash: sql.NullString{String: sha256Hex(content), Valid: true}, ContentHash: sql.NullString{String: "abc123", Valid: true},
Size: sql.NullInt64{Int64: int64(len(content)), Valid: true}, Size: sql.NullInt64{Int64: int64(len(content)), Valid: true},
ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
@ -673,7 +673,7 @@ func TestMavenHandler_DownloadCacheHit(t *testing.T) {
proxy, db, store, _ := setupTestProxy(t) proxy, db, store, _ := setupTestProxy(t)
seedPackageWithPURL(t, db, store, "maven", "com.google.guava:guava", "32.1.3-jre", "guava-32.1.3-jre.jar", "jar content") seedPackageWithPURL(t, db, store, "maven", "com.google.guava:guava", "32.1.3-jre", "guava-32.1.3-jre.jar", "jar content")
h := NewMavenHandler(proxy, "http://localhost", "", "") h := NewMavenHandler(proxy, "http://localhost")
srv := httptest.NewServer(h.Routes()) srv := httptest.NewServer(h.Routes())
defer srv.Close() defer srv.Close()
@ -730,7 +730,7 @@ func TestMavenHandler_MetadataProxied(t *testing.T) {
func TestMavenHandler_EmptyPathNotFound(t *testing.T) { func TestMavenHandler_EmptyPathNotFound(t *testing.T) {
proxy, _, _, _ := setupTestProxy(t) proxy, _, _, _ := setupTestProxy(t)
h := NewMavenHandler(proxy, "http://localhost", "", "") h := NewMavenHandler(proxy, "http://localhost")
srv := httptest.NewServer(h.Routes()) srv := httptest.NewServer(h.Routes())
defer srv.Close() defer srv.Close()
@ -748,7 +748,7 @@ func TestMavenHandler_EmptyPathNotFound(t *testing.T) {
func TestMavenHandler_ArtifactExtensions(t *testing.T) { func TestMavenHandler_ArtifactExtensions(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t) proxy, _, _, fetcher := setupTestProxy(t)
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"} extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib"}
for _, ext := range extensions { for _, ext := range extensions {
fetcher.artifact = &fetch.Artifact{ fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("artifact")), Body: io.NopCloser(strings.NewReader("artifact")),
@ -756,7 +756,7 @@ func TestMavenHandler_ArtifactExtensions(t *testing.T) {
} }
fetcher.fetchCalled = false fetcher.fetchCalled = false
h := NewMavenHandler(proxy, "http://localhost", "", "") h := NewMavenHandler(proxy, "http://localhost")
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Errorf("should not proxy artifact file %s to upstream", ext) t.Errorf("should not proxy artifact file %s to upstream", ext)
@ -789,7 +789,7 @@ func TestMavenHandler_CacheMiss(t *testing.T) {
ContentType: "application/java-archive", ContentType: "application/java-archive",
} }
h := NewMavenHandler(proxy, "http://localhost", "", "") h := NewMavenHandler(proxy, "http://localhost")
srv := httptest.NewServer(h.Routes()) srv := httptest.NewServer(h.Routes())
defer srv.Close() defer srv.Close()
@ -809,274 +809,6 @@ func TestMavenHandler_CacheMiss(t *testing.T) {
} }
} }
func TestMavenHandler_GradlePluginMarkerFallbackAndCache(t *testing.T) {
tests := []struct {
name string
markerPath string
}{
{
name: "Spotless",
markerPath: "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom",
},
{
name: "BenManes",
markerPath: "/com/github/ben-manes/versions/com.github.ben-manes.versions.gradle.plugin/0.54.0/com.github.ben-manes.versions.gradle.plugin-0.54.0.pom",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
primaryUpstream := "https://repo1.maven.org/maven2"
pluginPortalUpstream := "https://plugins.gradle.org/m2"
primaryURL := primaryUpstream + tt.markerPath
fetcher.fetchErrByURL = map[string]error{
primaryURL: ErrUpstreamNotFound,
}
fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("<project/>")),
ContentType: "application/xml",
}
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL + tt.markerPath)
if err != nil {
t.Fatalf("request failed: %v", err)
}
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
if string(body) != "<project/>" {
t.Fatalf("body = %q, want %q", body, "<project/>")
}
wantFallbackURL := pluginPortalUpstream + tt.markerPath
if fetcher.fetchedURL != wantFallbackURL {
t.Fatalf("fallback URL = %q, want %q", fetcher.fetchedURL, wantFallbackURL)
}
fetcher.fetchCalled = false
resp, err = http.Get(srv.URL + tt.markerPath)
if err != nil {
t.Fatalf("second request failed: %v", err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("second status = %d, want %d", resp.StatusCode, http.StatusOK)
}
if fetcher.fetchCalled {
t.Fatal("expected plugin marker POM to be served from cache on second request")
}
})
}
}
func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) {
paths := map[string]string{
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1": "sha1",
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha256": "sha256",
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.md5": "md5",
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "<metadata/>",
}
primaryHits := map[string]int{}
pluginHits := map[string]int{}
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
primaryHits[r.URL.Path]++
if _, ok := paths[r.URL.Path]; ok {
http.NotFound(w, r)
return
}
t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path)
}))
defer primary.Close()
pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
pluginHits[r.URL.Path]++
body, ok := paths[r.URL.Path]
if !ok {
http.NotFound(w, r)
return
}
_, _ = io.WriteString(w, body)
}))
defer pluginPortal.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = primary.Client()
h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL)
srv := httptest.NewServer(h.Routes())
defer srv.Close()
for reqPath, wantBody := range paths {
resp, err := http.Get(srv.URL + reqPath)
if err != nil {
t.Fatalf("GET %s failed: %v", reqPath, err)
}
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK)
}
if string(body) != wantBody {
t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody)
}
if primaryHits[reqPath] == 0 {
t.Fatalf("GET %s did not hit primary upstream", reqPath)
}
if pluginHits[reqPath] == 0 {
t.Fatalf("GET %s did not hit plugin portal fallback", reqPath)
}
}
}
func TestMavenHandler_GradlePluginImplementationMetadataFallback(t *testing.T) {
paths := map[string]string{
"/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha1": "impl-sha1",
"/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha256": "impl-sha256",
}
primaryHits := map[string]int{}
pluginHits := map[string]int{}
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
primaryHits[r.URL.Path]++
if _, ok := paths[r.URL.Path]; ok {
http.NotFound(w, r)
return
}
t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path)
}))
defer primary.Close()
pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
pluginHits[r.URL.Path]++
body, ok := paths[r.URL.Path]
if !ok {
http.NotFound(w, r)
return
}
_, _ = io.WriteString(w, body)
}))
defer pluginPortal.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = primary.Client()
h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL)
srv := httptest.NewServer(h.Routes())
defer srv.Close()
for reqPath, wantBody := range paths {
resp, err := http.Get(srv.URL + reqPath)
if err != nil {
t.Fatalf("GET %s failed: %v", reqPath, err)
}
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK)
}
if string(body) != wantBody {
t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody)
}
if primaryHits[reqPath] == 0 {
t.Fatalf("GET %s did not hit primary upstream", reqPath)
}
if pluginHits[reqPath] == 0 {
t.Fatalf("GET %s did not hit plugin portal fallback", reqPath)
}
}
}
func TestMavenHandler_GradlePluginImplementation_FallbackToPluginPortal(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
primaryUpstream := "https://repo1.maven.org/maven2"
pluginPortalUpstream := "https://plugins.gradle.org/m2"
implPath := "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar"
primaryURL := primaryUpstream + implPath
pluginPortalURL := pluginPortalUpstream + implPath
fetcher.fetchErrByURL = map[string]error{
primaryURL: ErrUpstreamNotFound,
}
fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("plugin impl jar")),
ContentType: "application/java-archive",
}
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL + implPath)
if err != nil {
t.Fatalf("request failed: %v", err)
}
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
if string(body) != "plugin impl jar" {
t.Fatalf("body = %q, want %q", body, "plugin impl jar")
}
if fetcher.fetchedURL != pluginPortalURL {
t.Fatalf("implementation artifact should fallback to plugin portal; fetched URL = %q, want %q", fetcher.fetchedURL, pluginPortalURL)
}
}
func TestMavenHandler_GradlePluginImplementation_NotFoundInBothUpstreams(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
primaryUpstream := "https://repo1.maven.org/maven2"
pluginPortalUpstream := "https://plugins.gradle.org/m2"
implPath := "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar"
primaryURL := primaryUpstream + implPath
pluginPortalURL := pluginPortalUpstream + implPath
fetcher.fetchErrByURL = map[string]error{
primaryURL: ErrUpstreamNotFound,
pluginPortalURL: ErrUpstreamNotFound,
}
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL + implPath)
if err != nil {
t.Fatalf("request failed: %v", err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotFound)
}
if fetcher.fetchedURL != pluginPortalURL {
t.Fatalf("expected fallback attempt to plugin portal; fetched URL = %q, want %q", fetcher.fetchedURL, pluginPortalURL)
}
}
func TestNuGetHandler_DownloadCacheMiss(t *testing.T) { func TestNuGetHandler_DownloadCacheMiss(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t) proxy, _, _, fetcher := setupTestProxy(t)
fetcher.artifact = &fetch.Artifact{ fetcher.artifact = &fetch.Artifact{
@ -1165,7 +897,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) {
ContentType: "application/vnd.debian.binary-package", ContentType: "application/vnd.debian.binary-package",
} }
h := NewDebianHandler(proxy, "http://localhost", "") h := NewDebianHandler(proxy, "http://localhost")
srv := httptest.NewServer(h.Routes()) srv := httptest.NewServer(h.Routes())
defer srv.Close() defer srv.Close()

View file

@ -1,39 +0,0 @@
package handler
import (
"net/http"
"strings"
)
type filenameDownload struct {
ecosystem string
suffix string
parseErr string
fetchErr string
parse func(string) (name, version string)
}
func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) {
filename := r.PathValue("filename")
if filename == "" || !strings.HasSuffix(filename, d.suffix) {
http.Error(w, "invalid filename", http.StatusBadRequest)
return
}
name, version := d.parse(filename)
if name == "" || version == "" {
http.Error(w, d.parseErr, http.StatusBadRequest)
return
}
p.Logger.Info(d.ecosystem+" download request",
"name", name, "version", version, "filename", filename)
result, err := p.GetOrFetchArtifact(r.Context(), d.ecosystem, name, version, filename)
if err != nil {
p.serveArtifactError(w, err, d.fetchErr)
return
}
ServeArtifact(w, result)
}

View file

@ -8,6 +8,8 @@ import (
"net/http" "net/http"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
) )
const ( const (
@ -58,13 +60,30 @@ func (h *GemHandler) Routes() http.Handler {
// handleDownload serves a gem file, fetching and caching from upstream if needed. // handleDownload serves a gem file, fetching and caching from upstream if needed.
func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
h.proxy.handleFilenameDownload(w, r, filenameDownload{ filename := r.PathValue("filename")
ecosystem: "gem", if filename == "" || !strings.HasSuffix(filename, ".gem") {
suffix: ".gem", http.Error(w, "invalid filename", http.StatusBadRequest)
parseErr: "could not parse gem filename", return
fetchErr: "failed to fetch gem", }
parse: h.parseGemFilename,
}) // Extract name and version from filename (e.g., "rails-7.1.0.gem")
name, version := h.parseGemFilename(filename)
if name == "" || version == "" {
http.Error(w, "could not parse gem filename", http.StatusBadRequest)
return
}
h.proxy.Logger.Info("gem download request",
"name", name, "version", version, "filename", filename)
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "gem", name, version, filename)
if err != nil {
h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch gem", http.StatusBadGateway)
return
}
ServeArtifact(w, result)
} }
// parseGemFilename extracts name and version from a gem filename. // parseGemFilename extracts name and version from a gem filename.
@ -247,7 +266,7 @@ func (h *GemHandler) fetchFilteredVersions(r *http.Request, name string) (map[st
return nil, err return nil, err
} }
packagePURL := canonicalPackagePURL("gem", name) packagePURL := purl.MakePURLString("gem", name, "")
filtered := make(map[string]bool) filtered := make(map[string]bool)
for _, v := range versions { for _, v := range versions {

View file

@ -10,7 +10,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
) )
func TestGemParseFilename(t *testing.T) { func TestGemParseFilename(t *testing.T) {

View file

@ -1,12 +1,9 @@
package handler package handler
import ( import (
"errors"
"fmt" "fmt"
"net/http" "net/http"
"strings" "strings"
"github.com/git-pkgs/registries/fetch"
) )
const ( const (
@ -103,10 +100,6 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "golang", decodedModule, version, filename) result, err := h.proxy.GetOrFetchArtifact(r.Context(), "golang", decodedModule, version, filename)
if err != nil { if err != nil {
if errors.Is(err, fetch.ErrNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
}
h.proxy.Logger.Error("failed to get artifact", "error", err) h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch module", http.StatusBadGateway) http.Error(w, "failed to fetch module", http.StatusBadGateway)
return return

View file

@ -1,49 +1,9 @@
package handler package handler
import ( import (
"errors"
"net/http"
"net/http/httptest"
"testing" "testing"
"github.com/git-pkgs/registries/fetch"
) )
func TestGoModuleDownloadUpstreamErrors(t *testing.T) {
tests := []struct {
name string
fetchErr error
wantStatus int
}{
{
name: "module not found",
fetchErr: fetch.ErrNotFound,
wantStatus: http.StatusNotFound,
},
{
name: "upstream failure",
fetchErr: errors.New("connection refused"),
wantStatus: http.StatusBadGateway,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
fetcher.fetchErr = tt.fetchErr
handler := NewGoHandler(proxy, "http://localhost:8080")
req := httptest.NewRequest(http.MethodGet, "/example.com/mod/@v/v1.0.0.zip", nil)
resp := httptest.NewRecorder()
handler.Routes().ServeHTTP(resp, req)
if resp.Code != tt.wantStatus {
t.Fatalf("status = %d, want %d", resp.Code, tt.wantStatus)
}
})
}
}
func TestDecodeGoModule(t *testing.T) { func TestDecodeGoModule(t *testing.T) {
tests := []struct { tests := []struct {
encoded string encoded string

View file

@ -7,9 +7,7 @@ import (
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"time"
"github.com/git-pkgs/proxy/internal/metrics"
"github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/proxy/internal/storage"
) )
@ -96,28 +94,18 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http
w.Header().Set("Content-Type", gradleBuildCacheContentType) w.Header().Set("Content-Type", gradleBuildCacheContentType)
if r.Method == http.MethodHead { if r.Method == http.MethodHead {
existsStart := time.Now()
exists, err := h.proxy.Storage.Exists(r.Context(), storagePath) exists, err := h.proxy.Storage.Exists(r.Context(), storagePath)
metrics.RecordStorageOperation("read", time.Since(existsStart))
if err != nil { if err != nil {
metrics.RecordStorageError("read")
h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err) h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err)
http.Error(w, "failed to read cache entry", http.StatusInternalServerError) http.Error(w, "failed to read cache entry", http.StatusInternalServerError)
return return
} }
if !exists { if !exists {
metrics.RecordCacheMiss("gradle")
http.NotFound(w, r) http.NotFound(w, r)
return return
} }
metrics.RecordCacheHit("gradle")
sizeStart := time.Now() if size, err := h.proxy.Storage.Size(r.Context(), storagePath); err == nil && size >= 0 {
size, err := h.proxy.Storage.Size(r.Context(), storagePath)
metrics.RecordStorageOperation("read", time.Since(sizeStart))
if err != nil {
metrics.RecordStorageError("read")
} else if size >= 0 {
w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
} }
@ -125,22 +113,17 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http
return return
} }
readStart := time.Now()
reader, err := h.proxy.Storage.Open(r.Context(), storagePath) reader, err := h.proxy.Storage.Open(r.Context(), storagePath)
metrics.RecordStorageOperation("read", time.Since(readStart))
if err != nil { if err != nil {
if errors.Is(err, storage.ErrNotFound) { if errors.Is(err, storage.ErrNotFound) {
metrics.RecordCacheMiss("gradle")
http.NotFound(w, r) http.NotFound(w, r)
return return
} }
metrics.RecordStorageError("read")
h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err) h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err)
http.Error(w, "failed to read cache entry", http.StatusInternalServerError) http.Error(w, "failed to read cache entry", http.StatusInternalServerError)
return return
} }
defer func() { _ = reader.Close() }() defer func() { _ = reader.Close() }()
metrics.RecordCacheHit("gradle")
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
_, _ = io.Copy(w, reader) _, _ = io.Copy(w, reader)
@ -155,9 +138,7 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque
r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize) r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize)
storeStart := time.Now()
_, hash, err := h.proxy.Storage.Store(r.Context(), storagePath, r.Body) _, hash, err := h.proxy.Storage.Store(r.Context(), storagePath, r.Body)
metrics.RecordStorageOperation("write", time.Since(storeStart))
if err != nil { if err != nil {
var maxBytesErr *http.MaxBytesError var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) { if errors.As(err, &maxBytesErr) {
@ -165,7 +146,6 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque
return return
} }
metrics.RecordStorageError("write")
h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err) h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err)
http.Error(w, "failed to write cache entry", http.StatusInternalServerError) http.Error(w, "failed to write cache entry", http.StatusInternalServerError)
return return

View file

@ -6,9 +6,6 @@ import (
"net/http/httptest" "net/http/httptest"
"strings" "strings"
"testing" "testing"
"github.com/git-pkgs/proxy/internal/metrics"
"github.com/prometheus/client_golang/prometheus/testutil"
) )
func TestGradleBuildCacheHandler_PutGetHead(t *testing.T) { func TestGradleBuildCacheHandler_PutGetHead(t *testing.T) {
@ -230,56 +227,3 @@ func TestGradleBuildCacheHandler_PutTooLarge(t *testing.T) {
t.Fatalf("PUT status = %d, want %d", resp.StatusCode, http.StatusRequestEntityTooLarge) t.Fatalf("PUT status = %d, want %d", resp.StatusCode, http.StatusRequestEntityTooLarge)
} }
} }
func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) {
proxy, _, _, _ := setupTestProxy(t)
h := NewGradleBuildCacheHandler(proxy)
srv := httptest.NewServer(h.Routes())
defer srv.Close()
hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle"))
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle"))
key := "metrics-key"
putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("payload"))
if err != nil {
t.Fatalf("failed to create PUT request: %v", err)
}
putResp, err := http.DefaultClient.Do(putReq)
if err != nil {
t.Fatalf("PUT request failed: %v", err)
}
_ = putResp.Body.Close()
getResp, err := http.Get(srv.URL + "/" + key)
if err != nil {
t.Fatalf("GET request failed: %v", err)
}
_ = getResp.Body.Close()
headReq, err := http.NewRequest(http.MethodHead, srv.URL+"/"+key, nil)
if err != nil {
t.Fatalf("failed to create HEAD request: %v", err)
}
headResp, err := http.DefaultClient.Do(headReq)
if err != nil {
t.Fatalf("HEAD request failed: %v", err)
}
_ = headResp.Body.Close()
missResp, err := http.Get(srv.URL + "/missing-key")
if err != nil {
t.Fatalf("GET miss request failed: %v", err)
}
_ = missResp.Body.Close()
hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle"))
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle"))
if diff := hitsAfter - hitsBefore; diff != 2 {
t.Fatalf("cache hits delta = %.0f, want 2", diff)
}
if diff := missesAfter - missesBefore; diff != 1 {
t.Fatalf("cache misses delta = %.0f, want 1", diff)
}
}

View file

@ -13,10 +13,9 @@ import (
"net/url" "net/url"
"strconv" "strconv"
"strings" "strings"
"sync"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
"github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/metrics"
"github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/proxy/internal/storage"
@ -49,44 +48,27 @@ func hasDotDotSegment(path string) bool {
const defaultHTTPTimeout = 30 * time.Second const defaultHTTPTimeout = 30 * time.Second
const artifactCopyBufferSize = 32 << 10
var artifactCopyBufferPool = sync.Pool{ //nolint:gochecknoglobals // shared across artifact responses
New: func() any {
buffer := make([]byte, artifactCopyBufferSize)
return &buffer
},
}
// canonicalPackagePURL returns a versionless PURL in canonical form so cooldown
// lookups match keys produced by config.CooldownConfig.NormalizedPackages.
func canonicalPackagePURL(ecosystem, name string) string {
return purl.MakePURLString(ecosystem, name, "")
}
const contentTypeJSON = "application/json" const contentTypeJSON = "application/json"
const headerAcceptEncoding = "Accept-Encoding" const headerAcceptEncoding = "Accept-Encoding"
// defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset. // maxMetadataSize is the maximum size of upstream metadata responses (100 MB).
const defaultMetadataMaxSize = 100 << 20 // Package metadata (e.g. npm with many versions) can be large, but unbounded
// reads risk OOM if an upstream misbehaves.
const maxMetadataSize = 100 << 20
// ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit. // ErrMetadataTooLarge is returned when upstream metadata exceeds maxMetadataSize.
var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit") var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit")
// ReadMetadata reads an upstream response body with a size limit to prevent OOM // ReadMetadata reads an upstream response body with a size limit to prevent OOM
// from unexpectedly large responses. Returns ErrMetadataTooLarge if the response // from unexpectedly large responses. Returns ErrMetadataTooLarge if the response
// is truncated by the limit. // is truncated by the limit.
func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { func ReadMetadata(r io.Reader) ([]byte, error) {
limit := p.MetadataMaxSize data, err := io.ReadAll(io.LimitReader(r, maxMetadataSize+1))
if limit <= 0 {
limit = defaultMetadataMaxSize
}
data, err := io.ReadAll(io.LimitReader(r, limit+1))
if err != nil { if err != nil {
return nil, err return nil, err
} }
if int64(len(data)) > limit { if int64(len(data)) > maxMetadataSize {
return nil, ErrMetadataTooLarge return nil, ErrMetadataTooLarge
} }
return data, nil return data, nil
@ -102,7 +84,6 @@ type Proxy struct {
Cooldown *cooldown.Config Cooldown *cooldown.Config
CacheMetadata bool CacheMetadata bool
MetadataTTL time.Duration MetadataTTL time.Duration
MetadataMaxSize int64
GradleReadOnly bool GradleReadOnly bool
GradleMaxUploadSize int64 GradleMaxUploadSize int64
DirectServe bool DirectServe bool
@ -112,7 +93,6 @@ type Proxy struct {
// storage at an internal one. // storage at an internal one.
DirectServeBaseURL string DirectServeBaseURL string
HTTPClient *http.Client HTTPClient *http.Client
AuthForURL func(string) (headerName, headerValue string)
} }
// NewProxy creates a new Proxy with the given dependencies. // NewProxy creates a new Proxy with the given dependencies.
@ -144,59 +124,41 @@ type CacheResult struct {
// GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream. // GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream.
func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) {
if cached, err := p.GetCachedArtifact(ctx, ecosystem, name, version, filename); err != nil { pkgPURL := purl.MakePURLString(ecosystem, name, "")
versionPURL := purl.MakePURLString(ecosystem, name, version)
if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil {
return nil, err return nil, err
} else if cached != nil { } else if cached != nil {
return cached, nil return cached, nil
} }
metrics.RecordCacheMiss(ecosystem)
pkgPURL := purl.MakePURLString(ecosystem, name, "")
versionPURL := purl.MakePURLString(ecosystem, name, version)
return p.fetchAndCache(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL) return p.fetchAndCache(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL)
} }
// GetCachedArtifact retrieves an artifact from cache without contacting an upstream.
// It returns nil when no usable cache entry exists.
func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) {
pkgPURL := purl.MakePURLString(ecosystem, name, "")
versionPURL := purl.MakePURLString(ecosystem, name, version)
return p.checkCache(ctx, pkgPURL, versionPURL, filename)
}
// ClearCachedArtifact removes both an artifact cache record and its stored
// bytes after an external integrity check fails.
func (p *Proxy) ClearCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) error {
if p.DB == nil || p.Storage == nil {
return nil
}
pkgPURL := purl.MakePURLString(ecosystem, name, "")
versionPURL := purl.MakePURLString(ecosystem, name, version)
cached, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename)
if err != nil {
return fmt.Errorf("looking up cached artifact: %w", err)
}
if cached == nil {
return nil
}
if err := p.Storage.Delete(ctx, cached.StoragePath); err != nil {
return fmt.Errorf("deleting cached artifact: %w", err)
}
return p.DB.ClearArtifactCache(versionPURL, filename)
}
// checkCache looks up an artifact in the cache. Returns nil if not cached. // checkCache looks up an artifact in the cache. Returns nil if not cached.
func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) {
artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) pkg, err := p.DB.GetPackageByPURL(pkgPURL)
if err != nil {
return nil, fmt.Errorf("checking package cache: %w", err)
}
if pkg == nil {
return nil, nil
}
ver, err := p.DB.GetVersionByPURL(versionPURL)
if err != nil {
return nil, fmt.Errorf("checking version cache: %w", err)
}
if ver == nil {
return nil, nil
}
artifact, err := p.DB.GetArtifact(versionPURL, filename)
if err != nil { if err != nil {
return nil, fmt.Errorf("checking artifact cache: %w", err) return nil, fmt.Errorf("checking artifact cache: %w", err)
} }
if artifact == nil { if artifact == nil || !artifact.IsCached() {
return nil, nil
}
checks, err := newIntegrityChecks(artifact.ContentHash.String, artifact.Integrity.String)
if err != nil {
p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err)
return nil, nil return nil, nil
} }
@ -208,44 +170,39 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
} }
if p.DirectServe { if p.DirectServe {
signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL)
if err == nil { if err == nil {
result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL) result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL)
p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) p.recordCacheHit(pkgPURL, versionPURL, filename)
return result, nil return result, nil
} }
if !errors.Is(err, storage.ErrSignedURLUnsupported) { if !errors.Is(err, storage.ErrSignedURLUnsupported) {
p.Logger.Warn("failed to sign storage URL, falling back to streaming", p.Logger.Warn("failed to sign storage URL, falling back to streaming",
"path", artifact.StoragePath, "error", err) "path", artifact.StoragePath.String, "error", err)
} }
} }
start := time.Now() start := time.Now()
reader, err := p.Storage.Open(ctx, artifact.StoragePath) reader, err := p.Storage.Open(ctx, artifact.StoragePath.String)
metrics.RecordStorageOperation("read", time.Since(start)) metrics.RecordStorageOperation("read", time.Since(start))
if err != nil { if err != nil {
metrics.RecordStorageError("read") metrics.RecordStorageError("read")
p.Logger.Warn("cached artifact missing from storage, will refetch", p.Logger.Warn("cached artifact missing from storage, will refetch",
"path", artifact.StoragePath, "error", err) "path", artifact.StoragePath.String, "error", err)
return nil, nil return nil, nil
} }
result.Reader, err = checks.wrap(reader, result.Reader = newVerifyingReader(reader, artifact.ContentHash.String, ver.Integrity.String,
func(reason string) { func(reason string) {
p.Logger.Error("cached artifact failed integrity check", p.Logger.Error("cached artifact failed integrity check",
"purl", versionPURL, "filename", filename, "purl", versionPURL, "filename", filename,
"path", artifact.StoragePath, "reason", reason) "path", artifact.StoragePath.String, "reason", reason)
metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem)) metrics.RecordIntegrityFailure(pkg.Ecosystem)
if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil { if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil {
p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err) p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err)
} }
}) })
if err != nil { p.recordCacheHit(pkgPURL, versionPURL, filename)
_ = reader.Close()
p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err)
return nil, nil
}
p.recordCacheHit(artifact.Ecosystem, versionPURL, filename)
return result, nil return result, nil
} }
@ -269,28 +226,20 @@ func rewriteSignedURLHost(signed, baseURL string) string {
return s.String() return s.String()
} }
func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { func (p *Proxy) recordCacheHit(pkgPURL, versionPURL, filename string) {
_ = p.DB.RecordArtifactHit(versionPURL, filename) _ = p.DB.RecordArtifactHit(versionPURL, filename)
metrics.RecordCacheHit(ecosystem) if parsed, err := purl.Parse(pkgPURL); err == nil {
} metrics.RecordCacheHit(purl.PURLTypeToEcosystem(parsed.Type))
func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, versionPURL, filename string, cause error) {
p.Logger.Warn("cached artifact has unusable integrity metadata",
"purl", versionPURL, "filename", filename,
"path", artifact.StoragePath, "error", cause)
metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem))
if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil {
p.Logger.Warn("failed to clear unusable artifact from cache", "error", err)
} }
} }
func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) { func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) {
// Record cache miss
metrics.RecordCacheMiss(ecosystem)
// Resolve download URL // Resolve download URL
info, err := p.Resolver.Resolve(ctx, ecosystem, name, version) info, err := p.Resolver.Resolve(ctx, ecosystem, name, version)
if err != nil { if err != nil {
if errors.Is(err, fetch.ErrNotFound) {
return nil, ErrUpstreamNotFound
}
return nil, fmt.Errorf("resolving download URL: %w", err) return nil, fmt.Errorf("resolving download URL: %w", err)
} }
@ -310,9 +259,6 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil
if err != nil { if err != nil {
metrics.RecordUpstreamFetch(ecosystem, fetchDuration) metrics.RecordUpstreamFetch(ecosystem, fetchDuration)
metrics.RecordUpstreamError(ecosystem, "fetch_failed") metrics.RecordUpstreamError(ecosystem, "fetch_failed")
if errors.Is(err, fetch.ErrNotFound) {
return nil, ErrUpstreamNotFound
}
return nil, fmt.Errorf("fetching from upstream: %w", err) return nil, fmt.Errorf("fetching from upstream: %w", err)
} }
metrics.RecordUpstreamFetch(ecosystem, fetchDuration) metrics.RecordUpstreamFetch(ecosystem, fetchDuration)
@ -399,40 +345,29 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u
// ServeArtifact writes a CacheResult to an HTTP response. // ServeArtifact writes a CacheResult to an HTTP response.
func ServeArtifact(w http.ResponseWriter, result *CacheResult) { func ServeArtifact(w http.ResponseWriter, result *CacheResult) {
serveArtifact(w, http.MethodGet, result)
}
func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) {
if result.RedirectURL != "" { if result.RedirectURL != "" {
if result.Hash != "" { if result.Hash != "" {
w.Header().Set("ETag", `"`+result.Hash+`"`) w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash))
} }
w.Header().Set("Location", result.RedirectURL) w.Header().Set("Location", result.RedirectURL)
w.WriteHeader(http.StatusFound) w.WriteHeader(http.StatusFound)
return return
} }
if result.Reader != nil { defer func() { _ = result.Reader.Close() }()
defer func() { _ = result.Reader.Close() }()
}
if result.ContentType != "" { if result.ContentType != "" {
w.Header().Set("Content-Type", result.ContentType) w.Header().Set("Content-Type", result.ContentType)
} }
if result.Size > 0 || (method == http.MethodHead && result.Size == 0) { if result.Size > 0 {
w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10)) w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
} }
if result.Hash != "" { if result.Hash != "" {
w.Header().Set("ETag", `"`+result.Hash+`"`) w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash))
} }
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
if method != http.MethodHead && result.Reader != nil { _, _ = io.Copy(w, result.Reader)
buffer := artifactCopyBufferPool.Get().(*[]byte)
defer artifactCopyBufferPool.Put(buffer)
// Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer.
_, _ = io.CopyBuffer(struct{ io.Writer }{w}, result.Reader, *buffer)
}
} }
// ProxyUpstream forwards a request to an upstream URL without caching. // ProxyUpstream forwards a request to an upstream URL without caching.
@ -453,7 +388,6 @@ func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamUR
req.Header.Set(header, v) req.Header.Set(header, v)
} }
} }
p.applyUpstreamAuth(req)
resp, err := p.HTTPClient.Do(req) resp, err := p.HTTPClient.Do(req)
if err != nil { if err != nil {
@ -480,7 +414,6 @@ func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL st
http.Error(w, "failed to create request", http.StatusInternalServerError) http.Error(w, "failed to create request", http.StatusInternalServerError)
return return
} }
p.applyUpstreamAuth(req)
resp, err := p.HTTPClient.Do(req) resp, err := p.HTTPClient.Do(req)
if err != nil { if err != nil {
@ -507,18 +440,7 @@ func JSONError(w http.ResponseWriter, status int, message string) {
} }
// ErrUpstreamNotFound indicates the upstream returned 404. // ErrUpstreamNotFound indicates the upstream returned 404.
var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) var ErrUpstreamNotFound = fmt.Errorf("upstream: not found")
// serveArtifactError writes response for a failed fetch:
// 404 when upstream reports artifact missing, 502 otherwise.
func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg string) {
if errors.Is(err, ErrUpstreamNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
}
p.Logger.Error("failed to get artifact", "error", err)
http.Error(w, clientMsg, http.StatusBadGateway)
}
// errStale304 is returned when upstream sends 304 but the cached file is missing. // errStale304 is returned when upstream sends 304 but the cached file is missing.
var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing") var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing")
@ -552,20 +474,18 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
cached, readErr := p.Storage.Open(ctx, entry.StoragePath) cached, readErr := p.Storage.Open(ctx, entry.StoragePath)
if readErr == nil { if readErr == nil {
defer func() { _ = cached.Close() }() defer func() { _ = cached.Close() }()
data, readErr := p.ReadMetadata(cached) data, readErr := ReadMetadata(cached)
if readErr == nil { if readErr == nil {
ct := contentTypeJSON ct := contentTypeJSON
if entry.ContentType.Valid { if entry.ContentType.Valid {
ct = entry.ContentType.String ct = entry.ContentType.String
} }
metrics.RecordCacheHit(ecosystem)
return data, ct, nil return data, ct, nil
} }
} }
// Cache file missing/unreadable, fall through to upstream // Cache file missing/unreadable, fall through to upstream
} }
} }
p.recordMetadataCacheMiss(ecosystem)
accept := contentTypeJSON accept := contentTypeJSON
if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { if len(acceptHeaders) > 0 && acceptHeaders[0] != "" {
@ -599,7 +519,7 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
} }
defer func() { _ = cached.Close() }() defer func() { _ = cached.Close() }()
data, readErr := p.ReadMetadata(cached) data, readErr := ReadMetadata(cached)
if readErr != nil { if readErr != nil {
return nil, "", fmt.Errorf("upstream failed and cached read error: %w", err) return nil, "", fmt.Errorf("upstream failed and cached read error: %w", err)
} }
@ -613,12 +533,6 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
return data, ct, nil return data, ct, nil
} }
func (p *Proxy) recordMetadataCacheMiss(ecosystem string) {
if p.CacheMetadata {
metrics.RecordCacheMiss(ecosystem)
}
}
// fetchUpstreamMetadata fetches metadata from upstream, using ETag for conditional revalidation. // fetchUpstreamMetadata fetches metadata from upstream, using ETag for conditional revalidation.
// Returns the body, content type, ETag, upstream Last-Modified time, and any error. // Returns the body, content type, ETag, upstream Last-Modified time, and any error.
func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept string) ([]byte, string, string, time.Time, error) { func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept string) ([]byte, string, string, time.Time, error) {
@ -629,7 +543,6 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
return nil, "", "", zeroTime, fmt.Errorf("creating request: %w", err) return nil, "", "", zeroTime, fmt.Errorf("creating request: %w", err)
} }
req.Header.Set("Accept", accept) req.Header.Set("Accept", accept)
p.applyUpstreamAuth(req)
if entry != nil && entry.ETag.Valid { if entry != nil && entry.ETag.Valid {
req.Header.Set("If-None-Match", entry.ETag.String) req.Header.Set("If-None-Match", entry.ETag.String)
@ -648,7 +561,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
return nil, "", "", zeroTime, errStale304 return nil, "", "", zeroTime, errStale304
} }
defer func() { _ = cached.Close() }() defer func() { _ = cached.Close() }()
data, readErr := p.ReadMetadata(cached) data, readErr := ReadMetadata(cached)
if readErr != nil { if readErr != nil {
return nil, "", "", zeroTime, errStale304 return nil, "", "", zeroTime, errStale304
} }
@ -670,7 +583,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
return nil, "", "", zeroTime, fmt.Errorf("upstream returned %d", resp.StatusCode) return nil, "", "", zeroTime, fmt.Errorf("upstream returned %d", resp.StatusCode)
} }
body, err := p.ReadMetadata(resp.Body) body, err := ReadMetadata(resp.Body)
if err != nil { if err != nil {
return nil, "", "", zeroTime, fmt.Errorf("reading response: %w", err) return nil, "", "", zeroTime, fmt.Errorf("reading response: %w", err)
} }
@ -767,14 +680,9 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL,
return return
} }
p.writeMetadataCachedResponse(w, r, ecosystem, cacheKey, body, contentType)
}
// writeMetadataCachedResponse writes a cached metadata response and handles
// conditional request headers using metadata cache validators.
func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) {
cm := p.lookupCachedMeta(ecosystem, cacheKey) cm := p.lookupCachedMeta(ecosystem, cacheKey)
// Honor client conditional request headers
if cm.etag != "" { if cm.etag != "" {
if match := r.Header.Get("If-None-Match"); match != "" && match == cm.etag { if match := r.Header.Get("If-None-Match"); match != "" && match == cm.etag {
w.WriteHeader(http.StatusNotModified) w.WriteHeader(http.StatusNotModified)
@ -819,7 +727,6 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst
accept = acceptHeaders[0] accept = acceptHeaders[0]
} }
req.Header.Set("Accept", accept) req.Header.Set("Accept", accept)
p.applyUpstreamAuth(req)
for _, header := range []string{headerAcceptEncoding, "If-Modified-Since", "If-None-Match"} { for _, header := range []string{headerAcceptEncoding, "If-Modified-Since", "If-None-Match"} {
if v := r.Header.Get(header); v != "" { if v := r.Header.Get(header); v != "" {
@ -844,17 +751,6 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst
_, _ = io.Copy(w, resp.Body) _, _ = io.Copy(w, resp.Body)
} }
func (p *Proxy) applyUpstreamAuth(req *http.Request) {
if p.AuthForURL == nil {
return
}
headerName, headerValue := p.AuthForURL(req.URL.String())
if headerName != "" && headerValue != "" {
req.Header.Set(headerName, headerValue)
}
}
// GetOrFetchArtifactFromURL retrieves an artifact from cache or fetches from a specific URL. // GetOrFetchArtifactFromURL retrieves an artifact from cache or fetches from a specific URL.
// This is useful for registries where download URLs are determined from metadata. // This is useful for registries where download URLs are determined from metadata.
func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) { func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) {
@ -862,17 +758,18 @@ func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name,
} }
// GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL // GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL
// with additional request-specific HTTP headers. // with additional HTTP headers. This is needed for registries that require authentication
// (e.g. Docker Hub requires a Bearer token even for public images).
func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) { func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) {
if cached, err := p.GetCachedArtifact(ctx, ecosystem, name, version, filename); err != nil { pkgPURL := purl.MakePURLString(ecosystem, name, "")
versionPURL := purl.MakePURLString(ecosystem, name, version)
if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil {
return nil, err return nil, err
} else if cached != nil { } else if cached != nil {
return cached, nil return cached, nil
} }
metrics.RecordCacheMiss(ecosystem)
pkgPURL := purl.MakePURLString(ecosystem, name, "")
versionPURL := purl.MakePURLString(ecosystem, name, version)
return p.fetchAndCacheFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers) return p.fetchAndCacheFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers)
} }
@ -882,9 +779,6 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi
artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers)
if err != nil { if err != nil {
if errors.Is(err, fetch.ErrNotFound) {
return nil, ErrUpstreamNotFound
}
return nil, fmt.Errorf("fetching from upstream: %w", err) return nil, fmt.Errorf("fetching from upstream: %w", err)
} }

View file

@ -1,300 +0,0 @@
package handler
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
"github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/proxy/internal/storage"
"github.com/git-pkgs/purl"
"github.com/git-pkgs/registries/fetch"
)
const benchmarkArtifactSize = 64 << 10
const benchmarkMetadataSize = 1 << 20
type benchmarkResponseWriter struct {
header http.Header
}
func (w *benchmarkResponseWriter) Header() http.Header {
return w.header
}
func (w *benchmarkResponseWriter) Write(p []byte) (int, error) {
return len(p), nil
}
func (w *benchmarkResponseWriter) WriteHeader(_ int) {}
func benchmarkCachedProxy(b *testing.B) (*Proxy, *mockStorage) {
b.Helper()
proxy, db, store, _ := setupTestProxy(b)
content := strings.Repeat("x", benchmarkArtifactSize)
seedPackage(b, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", content)
artifact, err := db.GetArtifact("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz")
if err != nil {
b.Fatalf("get seeded artifact: %v", err)
}
sum := sha256.Sum256([]byte(content))
artifact.ContentHash.String = hex.EncodeToString(sum[:])
if err := db.UpsertArtifact(artifact); err != nil {
b.Fatalf("update seeded artifact hash: %v", err)
}
return proxy, store
}
func BenchmarkArtifactCacheHit(b *testing.B) {
ctx := context.Background()
b.Run("stream-64KiB", func(b *testing.B) {
proxy, _ := benchmarkCachedProxy(b)
w := &benchmarkResponseWriter{header: make(http.Header)}
b.SetBytes(benchmarkArtifactSize)
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz")
if err != nil {
b.Fatal(err)
}
ServeArtifact(w, result)
}
})
b.Run("direct-serve", func(b *testing.B) {
proxy, store := benchmarkCachedProxy(b)
proxy.DirectServe = true
store.signedURL = "https://storage.example/npm/lodash-4.17.21.tgz?signature=abc"
w := &benchmarkResponseWriter{header: make(http.Header)}
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz")
if err != nil {
b.Fatal(err)
}
ServeArtifact(w, result)
}
})
}
func BenchmarkArtifactCacheHitParallel(b *testing.B) {
proxy, _ := benchmarkCachedProxy(b)
ctx := context.Background()
b.SetBytes(benchmarkArtifactSize)
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
w := &benchmarkResponseWriter{header: make(http.Header)}
for pb.Next() {
result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz")
if err != nil {
b.Error(err)
return
}
ServeArtifact(w, result)
}
})
}
func BenchmarkReadMetadata(b *testing.B) {
payload := bytes.Repeat([]byte("x"), benchmarkMetadataSize)
proxy := &Proxy{MetadataMaxSize: benchmarkMetadataSize}
b.SetBytes(benchmarkMetadataSize)
b.ReportAllocs()
var data []byte
for b.Loop() {
var err error
data, err = proxy.ReadMetadata(bytes.NewReader(payload))
if err != nil {
b.Fatal(err)
}
}
if len(data) != len(payload) {
b.Fatalf("metadata size = %d, want %d", len(data), len(payload))
}
}
func BenchmarkArtifactPURLConstruction(b *testing.B) {
for _, tc := range []struct {
name string
ecosystem string
packageID string
}{
{"npm", "npm", "lodash"},
{"scoped-npm", "npm", "@scope/package"},
{"go", "golang", "github.com/git-pkgs/proxy"},
} {
b.Run(tc.name, func(b *testing.B) {
b.ReportAllocs()
var packagePURL, versionPURL string
for b.Loop() {
packagePURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "")
versionPURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "1.2.3")
}
if packagePURL == "" || versionPURL == "" {
b.Fatal("empty PURL")
}
})
}
}
type benchmarkNPMServer struct {
client *http.Client
requestURL string
db *database.DB
versionPURL string
filename string
}
func newBenchmarkNPMServer(b *testing.B) *benchmarkNPMServer {
b.Helper()
ctx := context.Background()
dir := b.TempDir()
db, err := database.Create(filepath.Join(dir, "benchmark.db"))
if err != nil {
b.Fatalf("create database: %v", err)
}
b.Cleanup(func() { _ = db.Close() })
store, err := storage.OpenBucket(ctx, "file://"+filepath.Join(dir, "cache"))
if err != nil {
b.Fatalf("open storage: %v", err)
}
b.Cleanup(func() { _ = store.Close() })
content := bytes.Repeat([]byte("x"), benchmarkArtifactSize)
storagePath := storage.ArtifactPath("npm", "", "lodash", "4.17.21", "lodash-4.17.21.tgz")
size, hash, err := store.Store(ctx, storagePath, bytes.NewReader(content))
if err != nil {
b.Fatalf("store artifact: %v", err)
}
pkg := &database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"}
if err := db.UpsertPackage(pkg); err != nil {
b.Fatalf("seed package: %v", err)
}
version := &database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: pkg.PURL}
if err := db.UpsertVersion(version); err != nil {
b.Fatalf("seed version: %v", err)
}
artifact := &database.Artifact{
VersionPURL: version.PURL,
Filename: "lodash-4.17.21.tgz",
UpstreamURL: "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
StoragePath: sql.NullString{String: storagePath, Valid: true},
ContentHash: sql.NullString{String: hash, Valid: true},
Size: sql.NullInt64{Int64: size, Valid: true},
ContentType: sql.NullString{String: "application/gzip", Valid: true},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
}
if err := db.UpsertArtifact(artifact); err != nil {
b.Fatalf("seed artifact: %v", err)
}
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
proxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger)
handler := NewNPMHandler(proxy, "http://proxy.example", "https://registry.npmjs.org")
server := httptest.NewServer(handler.Routes())
b.Cleanup(server.Close)
client := server.Client()
return &benchmarkNPMServer{
client: client,
requestURL: server.URL + "/lodash/-/lodash-4.17.21.tgz",
db: db,
versionPURL: version.PURL,
filename: artifact.Filename,
}
}
func (s *benchmarkNPMServer) request() error {
resp, err := s.client.Get(s.requestURL)
if err != nil {
return fmt.Errorf("GET cached artifact: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("GET cached artifact status = %d, want %d", resp.StatusCode, http.StatusOK)
}
n, err := io.Copy(io.Discard, resp.Body)
if err != nil {
return fmt.Errorf("read cached artifact: %w", err)
}
if n != benchmarkArtifactSize {
return fmt.Errorf("cached artifact size = %d, want %d", n, benchmarkArtifactSize)
}
return nil
}
func (s *benchmarkNPMServer) hitCount(b *testing.B) int64 {
b.Helper()
artifact, err := s.db.GetArtifact(s.versionPURL, s.filename)
if err != nil {
b.Fatalf("get artifact hit count: %v", err)
}
return artifact.HitCount
}
func benchmarkNPMArtifactCacheHitHTTP(b *testing.B, parallel bool) {
server := newBenchmarkNPMServer(b)
if err := server.request(); err != nil {
b.Fatal(err)
}
startHits := server.hitCount(b)
b.SetBytes(benchmarkArtifactSize)
b.ReportAllocs()
b.ResetTimer()
if parallel {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if err := server.request(); err != nil {
b.Error(err)
return
}
}
})
} else {
for b.Loop() {
if err := server.request(); err != nil {
b.Fatal(err)
}
}
}
b.StopTimer()
if hitCount := server.hitCount(b) - startHits; hitCount != int64(b.N) {
b.Fatalf("new artifact hits = %d, want %d", hitCount, b.N)
}
b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "requests/s")
}
func BenchmarkNPMArtifactCacheHitHTTP(b *testing.B) {
benchmarkNPMArtifactCacheHitHTTP(b, false)
}
func BenchmarkNPMArtifactCacheHitHTTPParallel(b *testing.B) {
benchmarkNPMArtifactCacheHitHTTP(b, true)
}

View file

@ -5,6 +5,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"errors" "errors"
"fmt"
"io" "io"
"log/slog" "log/slog"
"net/http" "net/http"
@ -13,13 +14,9 @@ import (
"testing" "testing"
"time" "time"
"github.com/git-pkgs/proxy/internal/config"
"github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/proxy/internal/metrics"
"github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/proxy/internal/storage"
"github.com/git-pkgs/purl"
"github.com/git-pkgs/registries/fetch" "github.com/git-pkgs/registries/fetch"
"github.com/prometheus/client_golang/prometheus/testutil"
) )
// mockStorage implements storage.Storage for testing. // mockStorage implements storage.Storage for testing.
@ -44,7 +41,7 @@ func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64,
return 0, "", err return 0, "", err
} }
s.files[path] = data s.files[path] = data
return int64(len(data)), sha256Hex(string(data)), nil return int64(len(data)), "fakehash123", nil
} }
func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error) { func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error) {
@ -100,11 +97,10 @@ func (s *mockStorage) Close() error { return nil }
// mockFetcher implements fetch.FetcherInterface for testing. // mockFetcher implements fetch.FetcherInterface for testing.
type mockFetcher struct { type mockFetcher struct {
artifact *fetch.Artifact artifact *fetch.Artifact
fetchErr error fetchErr error
fetchErrByURL map[string]error fetchCalled bool
fetchCalled bool fetchedURL string
fetchedURL string
} }
func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) {
@ -114,11 +110,6 @@ func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, e
func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) {
f.fetchCalled = true f.fetchCalled = true
f.fetchedURL = url f.fetchedURL = url
if f.fetchErrByURL != nil {
if err, ok := f.fetchErrByURL[url]; ok {
return nil, err
}
}
if f.fetchErr != nil { if f.fetchErr != nil {
return nil, f.fetchErr return nil, f.fetchErr
} }
@ -130,7 +121,7 @@ func (f *mockFetcher) Head(_ context.Context, _ string) (int64, string, error) {
} }
// setupTestProxy creates a Proxy with a real DB (SQLite in temp dir) and mock storage/fetcher. // setupTestProxy creates a Proxy with a real DB (SQLite in temp dir) and mock storage/fetcher.
func setupTestProxy(t testing.TB) (*Proxy, *database.DB, *mockStorage, *mockFetcher) { func setupTestProxy(t *testing.T) (*Proxy, *database.DB, *mockStorage, *mockFetcher) {
t.Helper() t.Helper()
dir := t.TempDir() dir := t.TempDir()
@ -150,11 +141,11 @@ func setupTestProxy(t testing.TB) (*Proxy, *database.DB, *mockStorage, *mockFetc
} }
// seedPackage creates a package, version, and cached artifact in the test DB and storage. // seedPackage creates a package, version, and cached artifact in the test DB and storage.
func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) { func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) {
t.Helper() t.Helper()
pkg := &database.Package{ pkg := &database.Package{
PURL: purl.MakePURLString(ecosystem, name, ""), PURL: fmt.Sprintf("pkg:%s/%s", ecosystem, name),
Ecosystem: ecosystem, Ecosystem: ecosystem,
Name: name, Name: name,
} }
@ -162,7 +153,7 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n
t.Fatalf("failed to upsert package: %v", err) t.Fatalf("failed to upsert package: %v", err)
} }
versionPURL := purl.MakePURLString(ecosystem, name, version) versionPURL := fmt.Sprintf("pkg:%s/%s@%s", ecosystem, name, version)
ver := &database.Version{ ver := &database.Version{
PURL: versionPURL, PURL: versionPURL,
PackagePURL: pkg.PURL, PackagePURL: pkg.PURL,
@ -179,7 +170,7 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n
Filename: filename, Filename: filename,
UpstreamURL: "https://example.com/" + filename, UpstreamURL: "https://example.com/" + filename,
StoragePath: sql.NullString{String: storagePath, Valid: true}, StoragePath: sql.NullString{String: storagePath, Valid: true},
ContentHash: sql.NullString{String: sha256Hex(content), Valid: true}, ContentHash: sql.NullString{String: "abc123", Valid: true},
Size: sql.NullInt64{Int64: int64(len(content)), Valid: true}, Size: sql.NullInt64{Int64: int64(len(content)), Valid: true},
ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
@ -268,80 +259,13 @@ func TestGetOrFetchArtifact_CacheHit(t *testing.T) {
if result.ContentType != "application/octet-stream" { if result.ContentType != "application/octet-stream" {
t.Errorf("got content type %q, want %q", result.ContentType, "application/octet-stream") t.Errorf("got content type %q, want %q", result.ContentType, "application/octet-stream")
} }
if result.Hash != sha256Hex("cached content") { if result.Hash != "abc123" {
t.Errorf("got hash %q, want %q", result.Hash, sha256Hex("cached content")) t.Errorf("got hash %q, want %q", result.Hash, "abc123")
}
}
func TestGetCachedArtifactRejectsMalformedIntegrityMetadata(t *testing.T) {
tests := []struct {
name string
malformedHash string
malformedIntegrity string
}{
{name: "content hash", malformedHash: "abc123"},
{name: "native integrity", malformedIntegrity: "sha512-abc123"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assertMalformedCacheRejected(t, test.malformedHash, test.malformedIntegrity)
})
}
}
func assertMalformedCacheRejected(t *testing.T, malformedHash, malformedIntegrity string) {
t.Helper()
proxy, db, store, _ := setupTestProxy(t)
const (
packageName = "broken"
version = "1.0.0"
filename = "broken-1.0.0.tgz"
)
seedPackage(t, db, store, "npm", packageName, version, filename, "cached content")
versionPURL := purl.MakePURLString("npm", packageName, version)
if malformedHash != "" {
artifact, err := db.GetArtifact(versionPURL, filename)
if err != nil {
t.Fatal(err)
}
artifact.ContentHash = sql.NullString{String: malformedHash, Valid: true}
if err := db.UpsertArtifact(artifact); err != nil {
t.Fatal(err)
}
}
if malformedIntegrity != "" {
versionRecord := &database.Version{
PURL: versionPURL,
PackagePURL: purl.MakePURLString("npm", packageName, ""),
Integrity: sql.NullString{String: malformedIntegrity, Valid: true},
}
if err := db.UpsertVersion(versionRecord); err != nil {
t.Fatal(err)
}
}
proxy.DirectServe = true
store.signedURL = "https://cache.example/broken"
result, err := proxy.GetCachedArtifact(context.Background(), "npm", packageName, version, filename)
if err != nil {
t.Fatalf("GetCachedArtifact: %v", err)
}
if result != nil {
t.Errorf("GetCachedArtifact = %+v, want nil", result)
}
artifact, err := db.GetArtifact(versionPURL, filename)
if err != nil {
t.Fatal(err)
}
if artifact.StoragePath.Valid {
t.Error("unusable cache record retained its storage path")
} }
} }
func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) { func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t) proxy, _, _, fetcher := setupTestProxy(t)
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm"))
// The resolver will fail because "nonexistent" isn't a real package, // The resolver will fail because "nonexistent" isn't a real package,
// but we're testing that it tries to fetch (doesn't return from cache). // but we're testing that it tries to fetch (doesn't return from cache).
@ -351,10 +275,6 @@ func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) {
if err == nil { if err == nil {
t.Fatal("expected error for uncached package") t.Fatal("expected error for uncached package")
} }
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm"))
if diff := missesAfter - missesBefore; diff != 1 {
t.Errorf("cache misses delta = %.0f, want 1", diff)
}
} }
func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) { func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) {
@ -370,7 +290,7 @@ func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) {
Filename: "missing-1.0.0.tgz", Filename: "missing-1.0.0.tgz",
UpstreamURL: "https://example.com/missing.tgz", UpstreamURL: "https://example.com/missing.tgz",
StoragePath: sql.NullString{String: "nonexistent/path.tgz", Valid: true}, StoragePath: sql.NullString{String: "nonexistent/path.tgz", Valid: true},
ContentHash: sql.NullString{String: sha256Hex("missing content"), Valid: true}, ContentHash: sql.NullString{String: "hash", Valid: true},
Size: sql.NullInt64{Int64: 100, Valid: true}, Size: sql.NullInt64{Int64: 100, Valid: true},
ContentType: sql.NullString{String: "application/octet-stream", Valid: true}, ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
@ -613,7 +533,6 @@ func TestServeArtifact_Stream(t *testing.T) {
func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) {
proxy, db, store, fetcher := setupTestProxy(t) proxy, db, store, fetcher := setupTestProxy(t)
seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content") seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content")
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi"))
result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "https://pypi.org/files/requests-2.28.0.tar.gz") result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "https://pypi.org/files/requests-2.28.0.tar.gz")
if err != nil { if err != nil {
@ -627,15 +546,10 @@ func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) {
if fetcher.fetchCalled { if fetcher.fetchCalled {
t.Error("fetcher should not be called on cache hit") t.Error("fetcher should not be called on cache hit")
} }
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi"))
if diff := missesAfter - missesBefore; diff != 0 {
t.Errorf("cache misses delta = %.0f, want 0", diff)
}
} }
func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) {
proxy, _, store, fetcher := setupTestProxy(t) proxy, _, store, fetcher := setupTestProxy(t)
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi"))
fetcher.artifact = &fetch.Artifact{ fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("fetched content")), Body: io.NopCloser(strings.NewReader("fetched content")),
@ -668,10 +582,6 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) {
if _, ok := store.files[storagePath]; !ok { if _, ok := store.files[storagePath]; !ok {
t.Error("artifact was not stored in storage") t.Error("artifact was not stored in storage")
} }
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi"))
if diff := missesAfter - missesBefore; diff != 1 {
t.Errorf("cache misses delta = %.0f, want 1", diff)
}
} }
func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) {
@ -961,8 +871,6 @@ func TestProxyCached_NoValidators_OmitsHeaders(t *testing.T) {
} }
func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) { func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) {
hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test"))
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test"))
upstreamHits := 0 upstreamHits := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstreamHits++ upstreamHits++
@ -989,12 +897,6 @@ func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) {
if upstreamHits != 1 { if upstreamHits != 1 {
t.Fatalf("expected 1 upstream hit, got %d", upstreamHits) t.Fatalf("expected 1 upstream hit, got %d", upstreamHits)
} }
if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) - missesBefore; diff != 1 {
t.Errorf("cache misses delta after first request = %.0f, want 1", diff)
}
if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test")) - hitsBefore; diff != 0 {
t.Errorf("cache hits delta after first request = %.0f, want 0", diff)
}
// Second request within TTL should serve from cache without hitting upstream // Second request within TTL should serve from cache without hitting upstream
body, _, err = proxy.FetchOrCacheMetadata(ctx, "test", "ttl-pkg", upstream.URL+"/pkg") body, _, err = proxy.FetchOrCacheMetadata(ctx, "test", "ttl-pkg", upstream.URL+"/pkg")
@ -1007,16 +909,9 @@ func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) {
if upstreamHits != 1 { if upstreamHits != 1 {
t.Errorf("expected upstream to still be hit only once, got %d", upstreamHits) t.Errorf("expected upstream to still be hit only once, got %d", upstreamHits)
} }
if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test")) - hitsBefore; diff != 1 {
t.Errorf("cache hits delta after second request = %.0f, want 1", diff)
}
if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) - missesBefore; diff != 1 {
t.Errorf("cache misses delta after second request = %.0f, want 1", diff)
}
} }
func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) { func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) {
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test"))
upstreamHits := 0 upstreamHits := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstreamHits++ upstreamHits++
@ -1045,40 +940,6 @@ func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) {
if upstreamHits != 2 { if upstreamHits != 2 {
t.Errorf("expected 2 upstream hits with TTL=0, got %d", upstreamHits) t.Errorf("expected 2 upstream hits with TTL=0, got %d", upstreamHits)
} }
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test"))
if diff := missesAfter - missesBefore; diff != 2 {
t.Errorf("cache misses delta = %.0f, want 2", diff)
}
}
func TestFetchOrCacheMetadata_CacheDisabledDoesNotRecordMetrics(t *testing.T) {
const ecosystem = "metadata-disabled"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"v":1}`))
}))
t.Cleanup(upstream.Close)
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem))
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem))
_, _, err := proxy.FetchOrCacheMetadata(context.Background(), ecosystem, "pkg", upstream.URL+"/pkg")
if err != nil {
t.Fatalf("fetch metadata: %v", err)
}
hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem))
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem))
if diff := hitsAfter - hitsBefore; diff != 0 {
t.Errorf("cache hits delta = %.0f, want 0", diff)
}
if diff := missesAfter - missesBefore; diff != 0 {
t.Errorf("cache misses delta = %.0f, want 0", diff)
}
} }
func TestProxyCached_StaleWarningHeader(t *testing.T) { func TestProxyCached_StaleWarningHeader(t *testing.T) {
@ -1143,33 +1004,3 @@ func TestProxyCached_FreshResponse_NoWarningHeader(t *testing.T) {
t.Errorf("Warning should be empty for fresh response, got %q", got) t.Errorf("Warning should be empty for fresh response, got %q", got)
} }
} }
// TestCanonicalPackagePURLMatchesConfig ensures the runtime cooldown lookup key
// agrees with config.CooldownConfig.NormalizedPackages for the same package,
// so a configured override is actually found regardless of how the user wrote it.
func TestCanonicalPackagePURLMatchesConfig(t *testing.T) {
tests := []struct {
ecosystem string
requestName string
configKey string
}{
{"npm", "@babel/core", "pkg:npm/@babel/core"},
{"npm", "@babel/core", "pkg:npm/%40babel/core"},
{"npm", "@typescript/typescript-darwin-arm64", "pkg:npm/@typescript/typescript-darwin-arm64"},
{"pypi", "Django", "pkg:pypi/Django"},
{"pypi", "django", "pkg:pypi/Django"},
{"composer", "symfony/console", "pkg:composer/Symfony/Console"},
{"cargo", "serde", "pkg:cargo/serde"},
}
for _, tt := range tests {
t.Run(tt.ecosystem+"/"+tt.requestName+"<="+tt.configKey, func(t *testing.T) {
cfg := config.CooldownConfig{Packages: map[string]string{tt.configKey: "1d"}}
normalized := cfg.NormalizedPackages()
lookup := canonicalPackagePURL(tt.ecosystem, tt.requestName)
if _, ok := normalized[lookup]; !ok {
t.Errorf("lookup key %q not found in normalized config %v", lookup, normalized)
}
})
}
}

View file

@ -1,364 +0,0 @@
package handler
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strings"
"time"
"gopkg.in/yaml.v3"
)
const (
helmMetadataEcosystem = "helm"
helmIndexFilename = "index.yaml"
sha256HexLength = 64
)
// HelmHandler serves read-only HTTP Helm chart repositories. Each configured
// repository is mounted at /helm/{repository}/.
type HelmHandler struct {
proxy *Proxy
proxyURL string
repositories map[string]string
}
// NewHelmHandler creates a Helm chart repository protocol handler.
func NewHelmHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *HelmHandler {
h := &HelmHandler{
proxyURL: strings.TrimSuffix(proxyURL, "/"),
repositories: make(map[string]string, len(repositories)),
proxy: proxy,
}
for name, repositoryURL := range repositories {
h.repositories[name] = strings.TrimSuffix(repositoryURL, "/")
}
return h
}
// Routes returns the HTTP handler for Helm chart repository requests.
func (h *HelmHandler) Routes() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /{repository}/index.yaml", h.handleIndex)
mux.HandleFunc("GET /{repository}/charts/{digest}/{filename}", h.handleChart)
return mux
}
func (h *HelmHandler) handleIndex(w http.ResponseWriter, r *http.Request) {
repository, upstreamURL, ok := h.repositoryForRequest(r)
if !ok {
http.NotFound(w, r)
return
}
body, contentType, err := h.fetchIndex(r, repository, upstreamURL)
if err != nil {
h.serveIndexError(w, err)
return
}
rewritten, err := h.rewriteIndex(repository, upstreamURL, body)
if err != nil {
h.proxy.Logger.Warn("failed to rewrite Helm index", "repository", repository, "error", err)
http.Error(w, "invalid Helm repository index", http.StatusBadGateway)
return
}
h.proxy.writeMetadataCachedResponse(w, r, helmMetadataEcosystem, h.indexCacheKey(repository, upstreamURL), rewritten, contentType)
}
func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) {
repository, upstreamURL, ok := h.repositoryForRequest(r)
if !ok {
http.NotFound(w, r)
return
}
digest, ok := normalizeHelmDigest(r.PathValue("digest"))
filename := r.PathValue("filename")
if !ok || filename == "" || strings.Contains(filename, "/") || containsPathTraversal(filename) {
http.Error(w, "invalid chart request", http.StatusBadRequest)
return
}
cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename)
if err != nil {
h.proxy.Logger.Error("failed to check Helm chart cache", "error", err)
http.Error(w, "failed to check chart cache", http.StatusInternalServerError)
return
}
if cached != nil {
h.serveChart(w, r, repository, digest, filename, cached)
return
}
body, _, err := h.fetchIndex(r, repository, upstreamURL)
if err != nil {
h.serveIndexError(w, err)
return
}
downloadURL, err := h.findChartDownload(upstreamURL, body, digest, filename)
if err != nil {
if errors.Is(err, errHelmChartNotFound) {
http.NotFound(w, r)
return
}
h.proxy.Logger.Warn("failed to read Helm index", "repository", repository, "error", err)
http.Error(w, "invalid Helm repository index", http.StatusBadGateway)
return
}
result, err := h.proxy.GetOrFetchArtifactFromURL(
r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL)
if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch chart")
return
}
h.serveChart(w, r, repository, digest, filename, result)
}
func (h *HelmHandler) serveChart(w http.ResponseWriter, r *http.Request, repository, digest, filename string, result *CacheResult) {
if !strings.EqualFold(result.Hash, digest) {
if result.Reader != nil {
_ = result.Reader.Close()
}
if clearErr := h.proxy.ClearCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename); clearErr != nil {
h.proxy.Logger.Warn("failed to clear Helm chart with invalid digest", "error", clearErr)
}
http.Error(w, "chart digest verification failed", http.StatusBadGateway)
return
}
if result.ContentType == "" {
w.Header().Set("Content-Type", "application/gzip")
}
ServeArtifact(w, result)
}
func (h *HelmHandler) repositoryForRequest(r *http.Request) (name, upstreamURL string, ok bool) {
name = r.PathValue("repository")
upstreamURL, ok = h.repositories[name]
return name, upstreamURL, ok
}
func (h *HelmHandler) fetchIndex(r *http.Request, repository, upstreamURL string) ([]byte, string, error) {
return h.proxy.FetchOrCacheMetadata(
r.Context(),
helmMetadataEcosystem,
h.indexCacheKey(repository, upstreamURL),
upstreamURL+"/"+helmIndexFilename,
"application/x-yaml, text/yaml;q=0.9, */*;q=0.1",
)
}
func (h *HelmHandler) indexCacheKey(repository, upstreamURL string) string {
identity := repository + "\x00" + upstreamURL
digest := sha256.Sum256([]byte(identity))
return hex.EncodeToString(digest[:])
}
func (h *HelmHandler) serveIndexError(w http.ResponseWriter, err error) {
if errors.Is(err, ErrUpstreamNotFound) {
http.Error(w, "Helm repository not found", http.StatusNotFound)
return
}
h.proxy.Logger.Error("failed to fetch Helm index", "error", err)
http.Error(w, "failed to fetch Helm repository index", http.StatusBadGateway)
}
func (h *HelmHandler) rewriteIndex(repository, upstreamURL string, body []byte) ([]byte, error) {
document, entries, err := parseHelmIndex(body)
if err != nil {
return nil, err
}
for i := 0; i < len(entries.Content); i += 2 {
chartName := entries.Content[i].Value
releases := entries.Content[i+1]
if releases.Kind != yaml.SequenceNode {
return nil, fmt.Errorf("chart %q releases must be a sequence", chartName)
}
filtered := make([]*yaml.Node, 0, len(releases.Content))
for _, release := range releases.Content {
chart, err := h.parseChartRelease(chartName, upstreamURL, release)
if err != nil {
return nil, err
}
if h.chartOnCooldown(chartName, chart.created) {
continue
}
for _, download := range chart.downloads {
download.node.Value = h.chartProxyURL(repository, chart.digest, download.filename)
}
filtered = append(filtered, release)
}
releases.Content = filtered
}
return yaml.Marshal(document)
}
func (h *HelmHandler) findChartDownload(upstreamURL string, body []byte, digest, filename string) (string, error) {
_, entries, err := parseHelmIndex(body)
if err != nil {
return "", err
}
for i := 0; i < len(entries.Content); i += 2 {
chartName := entries.Content[i].Value
releases := entries.Content[i+1]
if releases.Kind != yaml.SequenceNode {
return "", fmt.Errorf("chart %q releases must be a sequence", chartName)
}
for _, release := range releases.Content {
chart, err := h.parseChartRelease(chartName, upstreamURL, release)
if err != nil {
return "", err
}
if chart.digest != digest || h.chartOnCooldown(chartName, chart.created) {
continue
}
for _, download := range chart.downloads {
if download.filename == filename {
return download.url, nil
}
}
}
}
return "", errHelmChartNotFound
}
func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool {
return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() &&
!h.proxy.Cooldown.IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created)
}
type helmChartDownload struct {
node *yaml.Node
url string
filename string
}
type helmChartRelease struct {
created time.Time
digest string
downloads []helmChartDownload
}
var errHelmChartNotFound = errors.New("chart not found in Helm index")
func (h *HelmHandler) parseChartRelease(chartName, upstreamURL string, release *yaml.Node) (helmChartRelease, error) {
digestNode := helmMappingValue(release, "digest")
urlsNode := helmMappingValue(release, "urls")
if digestNode == nil || urlsNode == nil || urlsNode.Kind != yaml.SequenceNode || len(urlsNode.Content) == 0 {
return helmChartRelease{}, fmt.Errorf("chart %q has no digest or URLs", chartName)
}
digest, ok := normalizeHelmDigest(digestNode.Value)
if !ok {
return helmChartRelease{}, fmt.Errorf("chart %q has invalid digest", chartName)
}
baseURL, err := url.Parse(upstreamURL + "/" + helmIndexFilename)
if err != nil {
return helmChartRelease{}, fmt.Errorf("parsing Helm repository URL: %w", err)
}
chart := helmChartRelease{digest: digest}
if createdNode := helmMappingValue(release, "created"); createdNode != nil && createdNode.Value != "" {
chart.created, err = time.Parse(time.RFC3339Nano, createdNode.Value)
if err != nil {
return helmChartRelease{}, fmt.Errorf("chart %q has invalid creation time: %w", chartName, err)
}
}
for _, urlNode := range urlsNode.Content {
if urlNode.Kind != yaml.ScalarNode {
return helmChartRelease{}, fmt.Errorf("chart %q has invalid URL", chartName)
}
reference, err := url.Parse(urlNode.Value)
if err != nil {
return helmChartRelease{}, fmt.Errorf("parsing chart %q URL: %w", chartName, err)
}
downloadURL := baseURL.ResolveReference(reference)
if (downloadURL.Scheme != "http" && downloadURL.Scheme != "https") || downloadURL.Host == "" {
return helmChartRelease{}, fmt.Errorf("chart %q URL must be HTTP(S)", chartName)
}
filename := path.Base(downloadURL.Path)
if filename == "." || filename == "/" || filename == "" || !strings.HasSuffix(filename, ".tgz") {
return helmChartRelease{}, fmt.Errorf("chart %q URL must point to a .tgz file", chartName)
}
chart.downloads = append(chart.downloads, helmChartDownload{
node: urlNode,
url: downloadURL.String(),
filename: filename,
})
}
return chart, nil
}
func (h *HelmHandler) chartProxyURL(repository, digest, filename string) string {
return fmt.Sprintf("%s/helm/%s/charts/%s/%s", h.proxyURL,
url.PathEscape(repository), digest, url.PathEscape(filename))
}
func parseHelmIndex(body []byte) (*yaml.Node, *yaml.Node, error) {
var document yaml.Node
if err := yaml.Unmarshal(body, &document); err != nil {
return nil, nil, fmt.Errorf("parsing Helm index: %w", err)
}
entries, err := helmIndexEntries(&document)
if err != nil {
return nil, nil, err
}
return &document, entries, nil
}
func helmIndexEntries(document *yaml.Node) (*yaml.Node, error) {
if document == nil {
return nil, errors.New("helm index is empty")
}
if len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode {
return nil, errors.New("helm index must be a mapping")
}
entries := helmMappingValue(document.Content[0], "entries")
if entries == nil || entries.Kind != yaml.MappingNode {
return nil, errors.New("helm index has no entries mapping")
}
if len(entries.Content)%2 != 0 {
return nil, errors.New("helm index entries mapping has an incomplete key-value pair")
}
return entries, nil
}
func helmMappingValue(mapping *yaml.Node, key string) *yaml.Node {
if mapping == nil || mapping.Kind != yaml.MappingNode {
return nil
}
for i := 0; i+1 < len(mapping.Content); i += 2 {
if mapping.Content[i].Value == key {
return mapping.Content[i+1]
}
}
return nil
}
func normalizeHelmDigest(value string) (string, bool) {
digest := strings.TrimPrefix(strings.ToLower(value), "sha256:")
if len(digest) != sha256HexLength {
return "", false
}
for _, char := range digest {
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
return "", false
}
}
return digest, true
}

View file

@ -1,337 +0,0 @@
package handler
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/git-pkgs/cooldown"
upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient"
"github.com/git-pkgs/proxy/internal/storage"
"github.com/git-pkgs/registries/fetch"
"gopkg.in/yaml.v3"
)
func TestHelmHandler_RewritesIndexAndCachesChart(t *testing.T) {
chart := []byte("a Helm chart")
digest := helmSHA256Hex(chart)
var available atomic.Bool
available.Store(true)
var indexRequests atomic.Int32
var chartRequests atomic.Int32
var upstream *httptest.Server
upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !available.Load() {
http.Error(w, "unavailable", http.StatusServiceUnavailable)
return
}
switch r.URL.Path {
case "/charts/index.yaml":
indexRequests.Add(1)
w.Header().Set("Content-Type", "application/x-yaml")
_, _ = fmt.Fprintf(w, `apiVersion: v1
entries:
demo:
- annotations:
example.com/retained: "true"
created: 2020-01-02T03:04:05Z
digest: %s
name: demo
urls:
- demo-1.0.0.tgz
- %s/charts/mirror/demo-1.0.0.tgz
version: 1.0.0
generated: 2020-01-02T03:04:05Z
`, digest, upstream.URL)
case "/charts/demo-1.0.0.tgz", "/charts/mirror/demo-1.0.0.tgz":
chartRequests.Add(1)
w.Header().Set("Content-Type", "application/gzip")
_, _ = w.Write(chart)
default:
http.NotFound(w, r)
}
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.CacheMetadata = true
proxy.MetadataTTL = time.Hour
proxy.HTTPClient = upstream.Client()
fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
proxy.Fetcher = fetcher
t.Cleanup(func() { _ = fetcher.Close() })
h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL + "/charts"})
indexResponse := serveHelmRequest(h, "/stable/index.yaml")
if indexResponse.Code != http.StatusOK {
t.Fatalf("index status = %d, want 200: %s", indexResponse.Code, indexResponse.Body.String())
}
if got := indexResponse.Header().Get("Content-Type"); got != "application/x-yaml" {
t.Errorf("index Content-Type = %q, want application/x-yaml", got)
}
if strings.Contains(indexResponse.Body.String(), upstream.URL) {
t.Errorf("rewritten index contains upstream URL: %s", indexResponse.Body.String())
}
if !strings.Contains(indexResponse.Body.String(), "example.com/retained") {
t.Errorf("rewritten index lost an unrelated field: %s", indexResponse.Body.String())
}
var index map[string]any
if err := yaml.Unmarshal(indexResponse.Body.Bytes(), &index); err != nil {
t.Fatalf("parse rewritten index: %v", err)
}
entries := index["entries"].(map[string]any)
release := entries["demo"].([]any)[0].(map[string]any)
urls := release["urls"].([]any)
wantURL := "http://proxy.example/helm/stable/charts/" + digest + "/demo-1.0.0.tgz"
for _, rawURL := range urls {
if rawURL != wantURL {
t.Errorf("rewritten URL = %q, want %q", rawURL, wantURL)
}
}
firstChart := serveHelmRequest(h, "/stable/charts/"+digest+"/demo-1.0.0.tgz")
if firstChart.Code != http.StatusOK {
t.Fatalf("chart status = %d, want 200: %s", firstChart.Code, firstChart.Body.String())
}
if got := firstChart.Body.String(); got != string(chart) {
t.Errorf("chart body = %q, want %q", got, chart)
}
if got := firstChart.Header().Get("Content-Type"); got != "application/gzip" {
t.Errorf("chart Content-Type = %q, want application/gzip", got)
}
// Artifact cache availability must not depend on metadata caching or a
// reachable index upstream.
proxy.CacheMetadata = false
available.Store(false)
cachedChart := serveHelmRequest(h, "/stable/charts/"+digest+"/demo-1.0.0.tgz")
if cachedChart.Code != http.StatusOK {
t.Fatalf("cached chart status = %d, want 200: %s", cachedChart.Code, cachedChart.Body.String())
}
if got := cachedChart.Body.String(); got != string(chart) {
t.Errorf("cached chart body = %q, want %q", got, chart)
}
if got := indexRequests.Load(); got != 1 {
t.Errorf("index requests = %d, want 1", got)
}
if got := chartRequests.Load(); got != 1 {
t.Errorf("chart requests = %d, want 1", got)
}
}
func TestHelmHandler_RejectsChartDigestMismatch(t *testing.T) {
chart := []byte("tampered chart")
digest := helmSHA256Hex([]byte("expected chart"))
requests := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/index.yaml":
_, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", digest)
case "/demo.tgz":
requests++
_, _ = w.Write(chart)
default:
http.NotFound(w, r)
}
}))
defer upstream.Close()
proxy, _, store, _ := setupTestProxy(t)
proxy.CacheMetadata = true
proxy.MetadataTTL = time.Hour
proxy.HTTPClient = upstream.Client()
fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
proxy.Fetcher = fetcher
t.Cleanup(func() { _ = fetcher.Close() })
h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"test": upstream.URL})
for range 2 {
response := serveHelmRequest(h, "/test/charts/"+digest+"/demo.tgz")
if response.Code != http.StatusBadGateway {
t.Errorf("status = %d, want 502: %s", response.Code, response.Body.String())
}
}
if requests != 2 {
t.Errorf("chart requests = %d, want 2 after invalid cache entry is cleared", requests)
}
storagePath := storage.ArtifactPath(helmMetadataEcosystem, "", "test", digest, "demo.tgz")
if exists, err := store.Exists(t.Context(), storagePath); err != nil {
t.Fatalf("checking rejected chart storage: %v", err)
} else if exists {
t.Errorf("rejected chart remains in storage at %q", storagePath)
}
}
func TestHelmHandler_IndexCacheChangesWithUpstreamURL(t *testing.T) {
firstDigest := strings.Repeat("a", sha256HexLength)
secondDigest := strings.Repeat("b", sha256HexLength)
firstRequests := 0
secondRequests := 0
first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
firstRequests++
_, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", firstDigest)
}))
defer first.Close()
second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
secondRequests++
_, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", secondDigest)
}))
defer second.Close()
proxy, db, store, _ := setupTestProxy(t)
proxy.CacheMetadata = true
proxy.MetadataTTL = time.Hour
proxy.HTTPClient = first.Client()
firstHandler := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": first.URL})
if response := serveHelmRequest(firstHandler, "/stable/index.yaml"); response.Code != http.StatusOK {
t.Fatalf("first index status = %d, want 200: %s", response.Code, response.Body.String())
}
// Model a restarted server with the same database and storage but a changed
// repository URL. Its cache key must not reuse the previous index or ETag.
restartedProxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), nil)
restartedProxy.CacheMetadata = true
restartedProxy.MetadataTTL = time.Hour
restartedProxy.HTTPClient = second.Client()
secondHandler := NewHelmHandler(restartedProxy, "http://proxy.example", map[string]string{"stable": second.URL})
response := serveHelmRequest(secondHandler, "/stable/index.yaml")
if response.Code != http.StatusOK {
t.Fatalf("second index status = %d, want 200: %s", response.Code, response.Body.String())
}
if !strings.Contains(response.Body.String(), secondDigest) {
t.Errorf("second index did not use the new upstream: %s", response.Body.String())
}
if firstRequests != 1 {
t.Errorf("first upstream requests = %d, want 1", firstRequests)
}
if secondRequests != 1 {
t.Errorf("second upstream requests = %d, want 1", secondRequests)
}
}
func TestHelmHandler_UsesConfiguredUpstreamAuthentication(t *testing.T) {
chart := []byte("private Helm chart")
digest := helmSHA256Hex(chart)
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer private-token" {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
switch r.URL.Path {
case "/index.yaml":
_, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [demo.tgz]\n", digest)
case "/demo.tgz":
_, _ = w.Write(chart)
default:
http.NotFound(w, r)
}
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.CacheMetadata = true
proxy.MetadataTTL = time.Hour
authClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport,
upstreamhttp.AuthFunc(func(string) (string, string) {
return "Authorization", "Bearer private-token"
}))}
proxy.HTTPClient = authClient
fetcher := fetch.NewFetcher(fetch.WithHTTPClient(authClient), fetch.WithMaxRetries(0))
proxy.Fetcher = fetcher
t.Cleanup(func() { _ = fetcher.Close() })
h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"private": upstream.URL})
response := serveHelmRequest(h, "/private/charts/"+digest+"/demo.tgz")
if response.Code != http.StatusOK {
t.Fatalf("status = %d, want 200: %s", response.Code, response.Body.String())
}
if got := response.Body.String(); got != string(chart) {
t.Errorf("body = %q, want %q", got, chart)
}
}
func TestHelmHandler_FiltersNewChartsFromIndex(t *testing.T) {
oldDigest := strings.Repeat("a", 64)
newDigest := strings.Repeat("b", 64)
proxy := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}}
h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"test": "https://charts.example"})
body := fmt.Sprintf(`apiVersion: v1
entries:
demo:
- created: %s
digest: %s
urls: [demo-old.tgz]
- created: %s
digest: %s
urls: [demo-new.tgz]
`, time.Now().Add(-10*24*time.Hour).Format(time.RFC3339), oldDigest,
time.Now().Add(-time.Hour).Format(time.RFC3339), newDigest)
rewritten, err := h.rewriteIndex("test", "https://charts.example", []byte(body))
if err != nil {
t.Fatalf("rewriteIndex() error = %v", err)
}
if strings.Contains(string(rewritten), newDigest) {
t.Errorf("rewritten index includes a chart still in cooldown: %s", rewritten)
}
if !strings.Contains(string(rewritten), oldDigest) {
t.Errorf("rewritten index omitted an old chart: %s", rewritten)
}
}
func TestNormalizeHelmDigest(t *testing.T) {
digest := strings.Repeat("a", 64)
for _, input := range []string{digest, "sha256:" + digest, "SHA256:" + strings.ToUpper(digest)} {
if got, ok := normalizeHelmDigest(input); !ok || got != digest {
t.Errorf("normalizeHelmDigest(%q) = %q, %t; want %q, true", input, got, ok, digest)
}
}
if _, ok := normalizeHelmDigest("bad"); ok {
t.Error("normalizeHelmDigest accepted an invalid digest")
}
}
func TestHelmIndexEntriesRejectsIncompleteMapping(t *testing.T) {
entries := &yaml.Node{
Kind: yaml.MappingNode,
Content: []*yaml.Node{
{Kind: yaml.ScalarNode, Value: "demo"},
},
}
document := &yaml.Node{
Kind: yaml.DocumentNode,
Content: []*yaml.Node{{
Kind: yaml.MappingNode,
Content: []*yaml.Node{
{Kind: yaml.ScalarNode, Value: "entries"},
entries,
},
}},
}
if _, err := helmIndexEntries(document); err == nil {
t.Fatal("helmIndexEntries() error = nil, want incomplete mapping error")
}
}
func serveHelmRequest(h *HelmHandler, target string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil))
return w
}
func helmSHA256Hex(data []byte) string {
digest := sha256.Sum256(data)
return hex.EncodeToString(digest[:])
}

View file

@ -10,6 +10,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
"google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/encoding/protowire"
) )
@ -53,13 +54,30 @@ func (h *HexHandler) Routes() http.Handler {
// handleDownload serves a package tarball, fetching and caching from upstream if needed. // handleDownload serves a package tarball, fetching and caching from upstream if needed.
func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
h.proxy.handleFilenameDownload(w, r, filenameDownload{ filename := r.PathValue("filename")
ecosystem: "hex", if filename == "" || !strings.HasSuffix(filename, ".tar") {
suffix: ".tar", http.Error(w, "invalid filename", http.StatusBadRequest)
parseErr: "could not parse tarball filename", return
fetchErr: "failed to fetch package", }
parse: h.parseTarballFilename,
}) // Extract name and version from filename (e.g., "phoenix-1.7.10.tar")
name, version := h.parseTarballFilename(filename)
if name == "" || version == "" {
http.Error(w, "could not parse tarball filename", http.StatusBadRequest)
return
}
h.proxy.Logger.Info("hex download request",
"name", name, "version", version, "filename", filename)
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "hex", name, version, filename)
if err != nil {
h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return
}
ServeArtifact(w, result)
} }
// parseTarballFilename extracts name and version from a hex tarball filename. // parseTarballFilename extracts name and version from a hex tarball filename.
@ -219,7 +237,7 @@ func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[st
return nil, err return nil, err
} }
packagePURL := canonicalPackagePURL("hex", name) packagePURL := purl.MakePURLString("hex", name, "")
filtered := make(map[string]bool) filtered := make(map[string]bool)
for _, release := range pkg.Releases { for _, release := range pkg.Releases {

View file

@ -11,7 +11,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
"google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/encoding/protowire"
) )

View file

@ -1,100 +1,140 @@
package handler package handler
import ( import (
"crypto/sha256"
"crypto/sha512"
"crypto/subtle"
"encoding/base64"
"encoding/hex"
"fmt" "fmt"
"hash"
"io" "io"
"strings"
"github.com/git-pkgs/integrity"
) )
type integrityChecks struct { // parseSRI parses a Subresource Integrity string (e.g. "sha512-abc==") into
contentHash integrity.SRI // an algorithm name and raw digest bytes. Returns ok=false for empty,
native integrity.SRI // malformed, or unsupported entries. Only the first hash in a multi-hash
algorithms []integrity.Algorithm // SRI string is considered.
} func parseSRI(s string) (algo string, digest []byte, ok bool) {
s = strings.TrimSpace(s)
func newIntegrityChecks(contentHash, native string) (integrityChecks, error) { if s == "" {
checks := integrityChecks{} return "", nil, false
if contentHash != "" {
digest, err := integrity.ParseHex(integrity.SHA256, contentHash)
if err != nil {
return integrityChecks{}, fmt.Errorf("parse content_hash: %w", err)
}
checks.contentHash = integrity.SRI{digest}
checks.algorithms = append(checks.algorithms, integrity.SHA256)
} }
if i := strings.IndexByte(s, ' '); i >= 0 {
if native != "" { s = s[:i]
digests, err := integrity.ParseSRI(native)
if err != nil {
return integrityChecks{}, fmt.Errorf("parse integrity: %w", err)
}
checks.native = digests
for _, digest := range digests {
checks.algorithms = append(checks.algorithms, digest.Algorithm())
}
} }
algo, b64, found := strings.Cut(s, "-")
return checks, nil if !found {
} return "", nil, false
func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) {
if len(c.algorithms) == 0 {
return source, nil
} }
reader, err := integrity.NewReader(source, c.algorithms...) d, err := base64.StdEncoding.DecodeString(b64)
if err != nil { if err != nil {
return nil, fmt.Errorf("create integrity reader: %w", err) return "", nil, false
}
switch algo {
case "sha256", "sha384", "sha512":
return algo, d, true
default:
return "", nil, false
} }
return &verifyingReader{
source: source,
reader: reader,
checks: c,
onMismatch: onMismatch,
}, nil
} }
// verifyingReader forwards Close to its source and reports completed digest func newSRIHash(algo string) hash.Hash {
// mismatches after its shared integrity reader observes EOF. switch algo {
case "sha256":
return sha256.New()
case "sha384":
return sha512.New384()
case "sha512":
return sha512.New()
}
return nil
}
// verifyingReader wraps an io.ReadCloser and computes SHA256 (and optionally
// a second SRI hash) as bytes are read. When the underlying reader reaches
// EOF it compares the digests against the expected values and calls
// onMismatch for each failure. Verification is skipped if the stream was
// not fully consumed (e.g. client disconnect) to avoid false positives.
type verifyingReader struct { type verifyingReader struct {
source io.ReadCloser r io.ReadCloser
reader *integrity.Reader sha256 hash.Hash
checks integrityChecks wantSHA256 string
sri hash.Hash
sriAlgo string
wantSRI []byte
onMismatch func(reason string) onMismatch func(reason string)
eof bool
verified bool verified bool
} }
func (r *verifyingReader) Read(p []byte) (int, error) { func newVerifyingReader(r io.ReadCloser, contentHash, sri string, onMismatch func(string)) io.ReadCloser {
n, err := r.reader.Read(p) if contentHash == "" && sri == "" {
return r
}
v := &verifyingReader{
r: r,
onMismatch: onMismatch,
}
if contentHash != "" {
v.sha256 = sha256.New()
v.wantSHA256 = contentHash
}
if algo, digest, ok := parseSRI(sri); ok {
v.sri = newSRIHash(algo)
v.sriAlgo = algo
v.wantSRI = digest
}
if v.sha256 == nil && v.sri == nil {
return r
}
return v
}
func (v *verifyingReader) Read(p []byte) (int, error) {
n, err := v.r.Read(p)
if n > 0 {
if v.sha256 != nil {
v.sha256.Write(p[:n])
}
if v.sri != nil {
v.sri.Write(p[:n])
}
}
if err == io.EOF { if err == io.EOF {
r.verify() v.eof = true
v.verify()
} }
return n, err return n, err
} }
func (r *verifyingReader) Close() error { func (v *verifyingReader) Close() error {
return r.source.Close() if v.eof {
v.verify()
}
return v.r.Close()
} }
func (r *verifyingReader) verify() { func (v *verifyingReader) verify() {
if r.verified { if v.verified {
return
}
r.verified = true
result := r.reader.Result()
if !result.Complete {
return return
} }
v.verified = true
if len(r.checks.contentHash) > 0 { if v.sha256 != nil {
if err := result.Verify(r.checks.contentHash); err != nil { got := hex.EncodeToString(v.sha256.Sum(nil))
r.onMismatch("content_hash: " + err.Error()) if subtle.ConstantTimeCompare([]byte(got), []byte(v.wantSHA256)) != 1 {
v.onMismatch(fmt.Sprintf("content_hash mismatch: stored=%s computed=%s", v.wantSHA256, got))
} }
} }
if len(r.checks.native) > 0 { if v.sri != nil {
if err := result.Verify(r.checks.native); err != nil { got := v.sri.Sum(nil)
r.onMismatch("integrity: " + err.Error()) if subtle.ConstantTimeCompare(got, v.wantSRI) != 1 {
v.onMismatch(fmt.Sprintf("integrity mismatch: %s expected=%s computed=%s",
v.sriAlgo,
base64.StdEncoding.EncodeToString(v.wantSRI),
base64.StdEncoding.EncodeToString(got)))
} }
} }
} }

View file

@ -5,7 +5,6 @@ import (
"crypto/sha512" "crypto/sha512"
"encoding/base64" "encoding/base64"
"encoding/hex" "encoding/hex"
"errors"
"io" "io"
"strings" "strings"
"testing" "testing"
@ -16,68 +15,42 @@ func sha256Hex(data string) string {
return hex.EncodeToString(sum[:]) return hex.EncodeToString(sum[:])
} }
func sha256SRI(data string) string {
sum := sha256.Sum256([]byte(data))
return "sha256-" + base64.StdEncoding.EncodeToString(sum[:])
}
func sha384SRI(data string) string {
sum := sha512.Sum384([]byte(data))
return "sha384-" + base64.StdEncoding.EncodeToString(sum[:])
}
func sha512SRI(data string) string { func sha512SRI(data string) string {
sum := sha512.Sum512([]byte(data)) sum := sha512.Sum512([]byte(data))
return "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) return "sha512-" + base64.StdEncoding.EncodeToString(sum[:])
} }
func wrapIntegrityReader(t *testing.T, source io.ReadCloser, contentHash, native string, onMismatch func(string)) io.ReadCloser { func TestParseSRI(t *testing.T) {
t.Helper()
checks, err := newIntegrityChecks(contentHash, native)
if err != nil {
t.Fatalf("newIntegrityChecks: %v", err)
}
reader, err := checks.wrap(source, onMismatch)
if err != nil {
t.Fatalf("wrap: %v", err)
}
return reader
}
func TestNewIntegrityChecksCollectsAlgorithms(t *testing.T) {
checks, err := newIntegrityChecks(
sha256Hex("hello"),
strings.Join([]string{sha256SRI("first"), sha512SRI("second"), sha384SRI("third"), sha512SRI("alternative")}, " "),
)
if err != nil {
t.Fatal(err)
}
if len(checks.algorithms) != 5 {
t.Fatalf("algorithms = %v, want 5 entries", checks.algorithms)
}
if len(checks.native) != 4 {
t.Errorf("native digests = %d, want 4", len(checks.native))
}
}
func TestNewIntegrityChecksRejectsMalformedMetadata(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
contentHash string input string
native string algo string
ok bool
}{ }{
{name: "short content hash", contentHash: "abc123"}, {"sha512", sha512SRI("hello"), "sha512", true},
{name: "non-hex content hash", contentHash: strings.Repeat("z", sha256.Size*2)}, {"sha256", "sha256-" + base64.StdEncoding.EncodeToString([]byte("0123456789012345678901234567890123456789")), "sha256", true},
{name: "missing SRI separator", native: "sha512"}, {"empty", "", "", false},
{name: "malformed SRI base64", native: "sha512-not!base64"}, {"no dash", "sha512abc", "", false},
{name: "wrong SRI length", native: "sha512-" + base64.StdEncoding.EncodeToString([]byte("short"))}, {"bad base64", "sha512-not!base64", "", false},
{name: "unsupported SRI algorithm", native: "md5-1B2M2Y8AsgTpgAmY7PhCfg=="}, {"unsupported algo", "md5-" + base64.StdEncoding.EncodeToString([]byte("x")), "", false},
{name: "invalid SRI alternative", native: sha512SRI("valid") + " sha384-nope"}, {"multi hash takes first", sha512SRI("a") + " " + sha512SRI("b"), "sha512", true},
{"whitespace", " " + sha512SRI("x") + " ", "sha512", true},
} }
for _, test := range tests {
t.Run(test.name, func(t *testing.T) { for _, tt := range tests {
if _, err := newIntegrityChecks(test.contentHash, test.native); err == nil { t.Run(tt.name, func(t *testing.T) {
t.Fatal("newIntegrityChecks returned nil error") algo, digest, ok := parseSRI(tt.input)
if ok != tt.ok {
t.Fatalf("ok = %v, want %v", ok, tt.ok)
}
if !tt.ok {
return
}
if algo != tt.algo {
t.Errorf("algo = %q, want %q", algo, tt.algo)
}
if len(digest) == 0 {
t.Error("digest is empty")
} }
}) })
} }
@ -94,156 +67,69 @@ func TestVerifyingReader(t *testing.T) {
sri string sri string
wantCalls int wantCalls int
}{ }{
{name: "both match", hash: goodSHA, sri: goodSRI}, {"both match", goodSHA, goodSRI, 0},
{name: "SHA-256 only match", hash: goodSHA}, {"sha256 only match", goodSHA, "", 0},
{name: "SRI only match", sri: goodSRI}, {"sri only match", "", goodSRI, 0},
{name: "SHA-256 mismatch", hash: sha256Hex("other"), wantCalls: 1}, {"sha256 mismatch", sha256Hex("other"), "", 1},
{name: "SRI mismatch", sri: sha512SRI("other"), wantCalls: 1}, {"sri mismatch", "", sha512SRI("other"), 1},
{name: "both mismatch", hash: sha256Hex("other"), sri: sha512SRI("other"), wantCalls: 2}, {"both mismatch", sha256Hex("other"), sha512SRI("other"), 2},
{name: "no checks"}, {"no checks", "", "", 0},
{"unparseable sri ignored", goodSHA, "garbage", 0},
} }
for _, test := range tests { for _, tt := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
var calls []string var calls []string
reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), test.hash, test.sri, r := newVerifyingReader(io.NopCloser(strings.NewReader(data)), tt.hash, tt.sri,
func(reason string) { calls = append(calls, reason) }) func(reason string) { calls = append(calls, reason) })
got, err := io.ReadAll(reader) got, err := io.ReadAll(r)
if err != nil { if err != nil {
t.Fatalf("ReadAll: %v", err) t.Fatalf("ReadAll: %v", err)
} }
if string(got) != data { if string(got) != data {
t.Errorf("data corrupted: got %q", got) t.Errorf("data corrupted: got %q", got)
} }
if err := reader.Close(); err != nil { if err := r.Close(); err != nil {
t.Fatalf("Close: %v", err) t.Fatalf("Close: %v", err)
} }
if len(calls) != test.wantCalls {
t.Errorf("onMismatch called %d times, want %d: %v", len(calls), test.wantCalls, calls) if len(calls) != tt.wantCalls {
t.Errorf("onMismatch called %d times, want %d: %v", len(calls), tt.wantCalls, calls)
} }
}) })
} }
} }
func TestVerifyingReaderUsesStrongestNativeAlgorithm(t *testing.T) {
const data = "artifact"
tests := []struct {
name string
native string
wantCalls int
}{
{
name: "weaker match does not override stronger mismatch",
native: sha256SRI(data) + " " + sha512SRI("other"),
wantCalls: 1,
},
{
name: "stronger match ignores weaker mismatch",
native: sha256SRI("other") + " " + sha512SRI(data),
},
{
name: "same algorithm alternative matches",
native: sha512SRI("other") + " " + sha512SRI(data),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var calls int
reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), "", test.native, func(string) { calls++ })
if _, err := io.Copy(io.Discard, reader); err != nil {
t.Fatal(err)
}
if calls != test.wantCalls {
t.Errorf("onMismatch called %d times, want %d", calls, test.wantCalls)
}
})
}
}
func TestVerifyingReaderMismatchMessages(t *testing.T) {
const data = "actual"
wantHash := sha256Hex("expected")
wantSRI := sha512SRI("expected")
var reasons []string
reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader(data)), wantHash, wantSRI,
func(reason string) { reasons = append(reasons, reason) })
if _, err := io.Copy(io.Discard, reader); err != nil {
t.Fatal(err)
}
if len(reasons) != 2 {
t.Fatalf("reasons = %v, want two", reasons)
}
wantContentReason := "content_hash: integrity mismatch: expected " + sha256SRI("expected") + ", calculated " + sha256SRI(data)
if reasons[0] != wantContentReason {
t.Errorf("content reason = %q, want %q", reasons[0], wantContentReason)
}
wantNativeReason := "integrity: integrity mismatch: expected " + wantSRI + ", calculated " + sha512SRI(data)
if reasons[1] != wantNativeReason {
t.Errorf("native reason = %q, want %q", reasons[1], wantNativeReason)
}
}
func TestVerifyingReaderPassthrough(t *testing.T) { func TestVerifyingReaderPassthrough(t *testing.T) {
source := io.NopCloser(strings.NewReader("x")) src := io.NopCloser(strings.NewReader("x"))
reader := wrapIntegrityReader(t, source, "", "", func(string) { t.Fatal("should not be called") }) r := newVerifyingReader(src, "", "", func(string) { t.Fatal("should not be called") })
if reader != source { if r != src {
t.Error("expected passthrough when no hashes were provided") t.Error("expected passthrough when no hashes provided")
} }
} }
type closeTrackingReader struct {
io.Reader
closed bool
}
func (r *closeTrackingReader) Close() error {
r.closed = true
return nil
}
func TestVerifyingReaderPartialRead(t *testing.T) { func TestVerifyingReaderPartialRead(t *testing.T) {
source := &closeTrackingReader{Reader: strings.NewReader("hello world")}
var calls int var calls int
reader := wrapIntegrityReader(t, source, sha256Hex("other"), "", func(string) { calls++ }) r := newVerifyingReader(io.NopCloser(strings.NewReader("hello world")),
sha256Hex("hello world"), "", func(string) { calls++ })
buffer := make([]byte, 5) buf := make([]byte, 5)
_, _ = reader.Read(buffer) _, _ = r.Read(buf)
_ = reader.Close() _ = r.Close()
if calls != 0 { if calls != 0 {
t.Errorf("onMismatch called %d times for partial read, want 0", calls) t.Errorf("onMismatch called %d times for partial read, want 0", calls)
} }
if !source.closed {
t.Error("Close was not forwarded to the source")
}
}
func TestVerifyingReaderNonEOFError(t *testing.T) {
var calls int
reader := wrapIntegrityReader(t, io.NopCloser(errorFixtureReader{}), sha256Hex("data"), "", func(string) { calls++ })
if _, err := io.ReadAll(reader); !errors.Is(err, errIntegrityReadFixture) {
t.Fatalf("ReadAll error = %v", err)
}
if calls != 0 {
t.Errorf("onMismatch called %d times after non-EOF error", calls)
}
}
var errIntegrityReadFixture = errors.New("integrity read fixture")
type errorFixtureReader struct{}
func (errorFixtureReader) Read(p []byte) (int, error) {
return copy(p, "data"), errIntegrityReadFixture
} }
func TestVerifyingReaderVerifyOnce(t *testing.T) { func TestVerifyingReaderVerifyOnce(t *testing.T) {
var calls int var calls int
reader := wrapIntegrityReader(t, io.NopCloser(strings.NewReader("x")), sha256Hex("y"), "", func(string) { calls++ }) r := newVerifyingReader(io.NopCloser(strings.NewReader("x")), sha256Hex("y"), "",
_, _ = io.ReadAll(reader) func(string) { calls++ })
_ = reader.Close() _, _ = io.ReadAll(r)
_ = reader.Close() _ = r.Close()
_ = r.Close()
if calls != 1 { if calls != 1 {
t.Errorf("onMismatch called %d times, want 1", calls) t.Errorf("onMismatch called %d times, want 1", calls)
} }

View file

@ -1,344 +0,0 @@
package handler
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"sync"
"github.com/BurntSushi/toml"
)
const (
juliaUpstream = "https://pkg.julialang.org"
juliaGeneralRegistryUUID = "23338594-aafe-5451-b93e-139f81909106"
juliaArtifactName = "_artifact"
juliaRegistryName = "_registry"
)
var (
juliaHexPattern = regexp.MustCompile(`^[0-9a-f]{40,64}$`)
juliaUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
)
// JuliaHandler handles Julia Pkg server protocol requests.
//
// See https://pkgdocs.julialang.org/v1/registries/ and the PkgServer.jl
// reference implementation. The protocol is content-addressed: registry,
// package and artifact resources are all identified by git tree hashes
// and are immutable once published.
type JuliaHandler struct {
proxy *Proxy
upstreamURL string
mu sync.RWMutex
names map[string]string
namesHash string
loadMu sync.Mutex
}
// NewJuliaHandler creates a new Julia Pkg server handler.
func NewJuliaHandler(proxy *Proxy, _ string) *JuliaHandler {
return &JuliaHandler{
proxy: proxy,
upstreamURL: juliaUpstream,
names: make(map[string]string),
}
}
// Routes returns the HTTP handler for Julia requests.
func (h *JuliaHandler) Routes() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /registries", h.handleRegistries)
mux.HandleFunc("GET /registries.eager", h.handleRegistries)
mux.HandleFunc("GET /registries.conservative", h.handleRegistries)
mux.HandleFunc("GET /registry/{uuid}/{hash}", h.handleRegistry)
mux.HandleFunc("GET /package/{uuid}/{hash}", h.handlePackage)
mux.HandleFunc("GET /artifact/{hash}", h.handleArtifact)
mux.HandleFunc("GET /meta", h.proxyUpstream)
return mux
}
// handleRegistries serves the list of available registries. This is the only
// mutable endpoint in the protocol so it goes through the metadata cache.
func (h *JuliaHandler) handleRegistries(w http.ResponseWriter, r *http.Request) {
cacheKey := strings.TrimPrefix(r.URL.Path, "/")
h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "julia", cacheKey, "*/*")
}
// handleRegistry serves an immutable registry tarball and refreshes the
// UUID→name map from its Registry.toml.
func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) {
uuid := r.PathValue("uuid")
hash := r.PathValue("hash")
if !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
http.Error(w, "invalid registry reference", http.StatusBadRequest)
return
}
h.proxy.Logger.Info("julia registry request", "uuid", uuid, "hash", hash)
upstreamURL := h.upstreamURL + r.URL.Path
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL)
if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch registry")
return
}
go h.refreshNamesFromRegistry(uuid, hash)
ServeArtifact(w, result)
}
// handlePackage serves an immutable package source tarball.
func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) {
uuid := r.PathValue("uuid")
hash := r.PathValue("hash")
if !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
http.Error(w, "invalid package reference", http.StatusBadRequest)
return
}
if err := h.ensureNames(r.Context()); err != nil {
h.proxy.Logger.Warn("julia name map unavailable, using uuid", "error", err)
}
name := h.resolveName(uuid)
h.proxy.Logger.Info("julia package request", "name", name, "uuid", uuid, "hash", hash)
upstreamURL := h.upstreamURL + r.URL.Path
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL)
if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package")
return
}
ServeArtifact(w, result)
}
// handleArtifact serves an immutable binary artifact tarball. Artifacts are
// anonymous content-addressed blobs with no associated package name.
func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) {
hash := r.PathValue("hash")
if !juliaHexPattern.MatchString(hash) {
http.Error(w, "invalid artifact hash", http.StatusBadRequest)
return
}
h.proxy.Logger.Info("julia artifact request", "hash", hash)
upstreamURL := h.upstreamURL + r.URL.Path
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL)
if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch artifact")
return
}
ServeArtifact(w, result)
}
// proxyUpstream forwards a request to the upstream Pkg server without caching.
func (h *JuliaHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil)
}
// resolveName returns the human-readable package name for a UUID, falling
// back to the UUID itself if it is not present in the loaded registry.
func (h *JuliaHandler) resolveName(uuid string) string {
h.mu.RLock()
defer h.mu.RUnlock()
if name, ok := h.names[uuid]; ok {
return name
}
return uuid
}
// ensureNames lazily populates the UUID→name map from the General registry.
// Returns immediately if the map is already populated; otherwise blocks until
// a single in-flight load completes. Failed loads are retried on the next call.
func (h *JuliaHandler) ensureNames(ctx context.Context) error {
if h.namesLoaded() {
return nil
}
h.loadMu.Lock()
defer h.loadMu.Unlock()
if h.namesLoaded() {
return nil
}
return h.loadNamesFromUpstream(ctx)
}
func (h *JuliaHandler) namesLoaded() bool {
h.mu.RLock()
defer h.mu.RUnlock()
return len(h.names) > 0
}
// loadNamesFromUpstream fetches the current /registries listing, downloads the
// General registry tarball at its current hash, and parses Registry.toml.
func (h *JuliaHandler) loadNamesFromUpstream(ctx context.Context) error {
hash, err := h.fetchGeneralRegistryHash(ctx)
if err != nil {
return err
}
return h.loadRegistryTarball(ctx, juliaGeneralRegistryUUID, hash)
}
// fetchGeneralRegistryHash reads /registries and returns the current tree hash
// for the General registry.
func (h *JuliaHandler) fetchGeneralRegistryHash(ctx context.Context) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.upstreamURL+"/registries", nil)
if err != nil {
return "", err
}
resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
return "", err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("upstream /registries returned %d", resp.StatusCode)
}
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
uuid, hash, ok := parseRegistryLine(scanner.Text())
if ok && uuid == juliaGeneralRegistryUUID {
return hash, nil
}
}
if err := scanner.Err(); err != nil {
return "", err
}
return "", fmt.Errorf("general registry not listed in /registries")
}
// refreshNamesFromRegistry reloads the UUID→name map from a registry tarball
// that has just been cached. Errors are logged but do not affect the response.
func (h *JuliaHandler) refreshNamesFromRegistry(uuid, hash string) {
if uuid != juliaGeneralRegistryUUID {
return
}
h.mu.RLock()
current := h.namesHash
h.mu.RUnlock()
if current == hash {
return
}
if err := h.loadRegistryTarball(context.Background(), uuid, hash); err != nil {
h.proxy.Logger.Warn("failed to refresh julia name map", "error", err)
}
}
// loadRegistryTarball downloads a registry tarball and replaces the name map
// with the contents of its Registry.toml.
func (h *JuliaHandler) loadRegistryTarball(ctx context.Context, uuid, hash string) error {
url := fmt.Sprintf("%s/registry/%s/%s", h.upstreamURL, uuid, hash)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("upstream registry returned %d", resp.StatusCode)
}
names, err := extractRegistryNames(resp.Body)
if err != nil {
return err
}
h.mu.Lock()
h.names = names
h.namesHash = hash
h.mu.Unlock()
h.proxy.Logger.Info("loaded julia registry name map", "packages", len(names), "hash", hash)
return nil
}
// extractRegistryNames reads a gzipped registry tarball, finds Registry.toml
// at the root, and returns its [packages] table as a UUID→name map.
func extractRegistryNames(r io.Reader) (map[string]string, error) {
gz, err := gzip.NewReader(r)
if err != nil {
return nil, fmt.Errorf("opening gzip stream: %w", err)
}
defer func() { _ = gz.Close() }()
tr := tar.NewReader(gz)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil, fmt.Errorf("no Registry.toml in tarball")
}
if err != nil {
return nil, err
}
if strings.TrimPrefix(hdr.Name, "./") != "Registry.toml" {
continue
}
data, err := io.ReadAll(tr)
if err != nil {
return nil, err
}
return parseRegistryToml(data)
}
}
type juliaRegistryFile struct {
Packages map[string]struct {
Name string `toml:"name"`
} `toml:"packages"`
}
// parseRegistryToml decodes the [packages] table of a Registry.toml file.
func parseRegistryToml(data []byte) (map[string]string, error) {
var reg juliaRegistryFile
if _, err := toml.NewDecoder(bytes.NewReader(data)).Decode(&reg); err != nil {
return nil, fmt.Errorf("parsing Registry.toml: %w", err)
}
names := make(map[string]string, len(reg.Packages))
for uuid, pkg := range reg.Packages {
if pkg.Name != "" {
names[uuid] = pkg.Name
}
}
return names, nil
}
// parseRegistryLine parses a single line from /registries of the form
// "/registry/{uuid}/{hash}" and returns the uuid and hash.
func parseRegistryLine(line string) (uuid, hash string, ok bool) {
line = strings.TrimSpace(line)
line = strings.TrimPrefix(line, "/registry/")
uuid, hash, found := strings.Cut(line, "/")
if !found || !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
return "", "", false
}
return uuid, hash, true
}
// validJuliaUUID reports whether s looks like a lowercase RFC 4122 UUID.
func validJuliaUUID(s string) bool {
return juliaUUIDPattern.MatchString(s)
}

View file

@ -1,167 +0,0 @@
package handler
import (
"archive/tar"
"bytes"
"compress/gzip"
"log/slog"
"net/http"
"net/http/httptest"
"testing"
)
func TestJuliaParseRegistryLine(t *testing.T) {
tests := []struct {
line string
wantUUID string
wantHash string
wantOK bool
}{
{
"/registry/23338594-aafe-5451-b93e-139f81909106/342327538ed6c1ec54c69fa145e7b6bf5934201e",
"23338594-aafe-5451-b93e-139f81909106",
"342327538ed6c1ec54c69fa145e7b6bf5934201e",
true,
},
{
" /registry/23338594-aafe-5451-b93e-139f81909106/342327538ed6c1ec54c69fa145e7b6bf5934201e\n",
"23338594-aafe-5451-b93e-139f81909106",
"342327538ed6c1ec54c69fa145e7b6bf5934201e",
true,
},
{"/registry/not-a-uuid/0000", "", "", false},
{"junk", "", "", false},
{"", "", "", false},
}
for _, tt := range tests {
uuid, hash, ok := parseRegistryLine(tt.line)
if uuid != tt.wantUUID || hash != tt.wantHash || ok != tt.wantOK {
t.Errorf("parseRegistryLine(%q) = (%q, %q, %v), want (%q, %q, %v)",
tt.line, uuid, hash, ok, tt.wantUUID, tt.wantHash, tt.wantOK)
}
}
}
func TestJuliaValidUUID(t *testing.T) {
tests := []struct {
s string
want bool
}{
{"23338594-aafe-5451-b93e-139f81909106", true},
{"295af30f-e4ad-537b-8983-00126c2a3abe", true},
{"23338594-AAFE-5451-b93e-139f81909106", false},
{"23338594aafe5451b93e139f81909106", false},
{"23338594-aafe-5451-b93e-139f8190910", false},
{"23338594-aafe-5451-b93e-139f81909106-", false},
{"23338594-gafe-5451-b93e-139f81909106", false},
{"", false},
}
for _, tt := range tests {
if got := validJuliaUUID(tt.s); got != tt.want {
t.Errorf("validJuliaUUID(%q) = %v, want %v", tt.s, got, tt.want)
}
}
}
func TestJuliaParseRegistryToml(t *testing.T) {
data := []byte(`name = "General"
uuid = "23338594-aafe-5451-b93e-139f81909106"
[packages]
295af30f-e4ad-537b-8983-00126c2a3abe = { name = "Revise", path = "R/Revise" }
91a5bcdd-55d7-5caf-9e0b-520d859cae80 = { name = "Plots", path = "P/Plots" }
`)
names, err := parseRegistryToml(data)
if err != nil {
t.Fatalf("parseRegistryToml: %v", err)
}
if got := names["295af30f-e4ad-537b-8983-00126c2a3abe"]; got != "Revise" {
t.Errorf("names[Revise uuid] = %q, want Revise", got)
}
if got := names["91a5bcdd-55d7-5caf-9e0b-520d859cae80"]; got != "Plots" {
t.Errorf("names[Plots uuid] = %q, want Plots", got)
}
if len(names) != 2 {
t.Errorf("len(names) = %d, want 2", len(names))
}
}
func TestJuliaExtractRegistryNames(t *testing.T) {
registryToml := `name = "General"
[packages]
295af30f-e4ad-537b-8983-00126c2a3abe = { name = "Revise", path = "R/Revise" }
`
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
tw := tar.NewWriter(gw)
for _, f := range []struct{ name, body string }{
{"R/Revise/Package.toml", "name = \"Revise\"\n"},
{"Registry.toml", registryToml},
} {
if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o644, Size: int64(len(f.body))}); err != nil {
t.Fatalf("WriteHeader: %v", err)
}
if _, err := tw.Write([]byte(f.body)); err != nil {
t.Fatalf("Write: %v", err)
}
}
if err := tw.Close(); err != nil {
t.Fatalf("tar Close: %v", err)
}
if err := gw.Close(); err != nil {
t.Fatalf("gzip Close: %v", err)
}
names, err := extractRegistryNames(bytes.NewReader(buf.Bytes()))
if err != nil {
t.Fatalf("extractRegistryNames: %v", err)
}
if got := names["295af30f-e4ad-537b-8983-00126c2a3abe"]; got != "Revise" {
t.Errorf("names[Revise uuid] = %q, want Revise", got)
}
}
func TestJuliaResolveName(t *testing.T) {
h := &JuliaHandler{
proxy: &Proxy{Logger: slog.Default()},
names: map[string]string{
"295af30f-e4ad-537b-8983-00126c2a3abe": "Revise",
},
}
if got := h.resolveName("295af30f-e4ad-537b-8983-00126c2a3abe"); got != "Revise" {
t.Errorf("resolveName(known) = %q, want Revise", got)
}
if got := h.resolveName("00000000-0000-0000-0000-000000000000"); got != "00000000-0000-0000-0000-000000000000" {
t.Errorf("resolveName(unknown) = %q, want uuid fallback", got)
}
}
func TestJuliaRoutesValidation(t *testing.T) {
h := NewJuliaHandler(&Proxy{Logger: slog.Default()}, "")
routes := h.Routes()
tests := []struct {
path string
want int
}{
{"/package/not-a-uuid/342327538ed6c1ec54c69fa145e7b6bf5934201e", http.StatusBadRequest},
{"/package/295af30f-e4ad-537b-8983-00126c2a3abe/short", http.StatusBadRequest},
{"/registry/295af30f-e4ad-537b-8983-00126c2a3abe/zzzz", http.StatusBadRequest},
{"/artifact/nothex", http.StatusBadRequest},
{"/nope", http.StatusNotFound},
}
for _, tt := range tests {
req := httptest.NewRequest(http.MethodGet, tt.path, nil)
rr := httptest.NewRecorder()
routes.ServeHTTP(rr, req)
if rr.Code != tt.want {
t.Errorf("GET %s = %d, want %d", tt.path, rr.Code, tt.want)
}
}
}

View file

@ -1,7 +1,6 @@
package handler package handler
import ( import (
"errors"
"fmt" "fmt"
"net/http" "net/http"
"path" "path"
@ -9,33 +8,23 @@ import (
) )
const ( const (
mavenCentralUpstream = "https://repo1.maven.org/maven2" mavenUpstream = "https://repo1.maven.org/maven2"
gradlePluginPortalUpstream = "https://plugins.gradle.org/m2" minMavenParts = 4 // group path segments + artifact + version + filename
minMavenParts = 4 // group path segments + artifact + version + filename
) )
// MavenHandler handles Maven repository protocol requests. // MavenHandler handles Maven repository protocol requests.
type MavenHandler struct { type MavenHandler struct {
proxy *Proxy proxy *Proxy
upstreamURL string upstreamURL string
pluginPortalUpstreamURL string proxyURL string
proxyURL string
} }
// NewMavenHandler creates a new Maven repository handler. // NewMavenHandler creates a new Maven repository handler.
func NewMavenHandler(proxy *Proxy, proxyURL, upstreamURL, pluginPortalUpstreamURL string) *MavenHandler { func NewMavenHandler(proxy *Proxy, proxyURL string) *MavenHandler {
if strings.TrimSpace(upstreamURL) == "" {
upstreamURL = mavenCentralUpstream
}
if strings.TrimSpace(pluginPortalUpstreamURL) == "" {
pluginPortalUpstreamURL = gradlePluginPortalUpstream
}
return &MavenHandler{ return &MavenHandler{
proxy: proxy, proxy: proxy,
upstreamURL: strings.TrimSuffix(upstreamURL, "/"), upstreamURL: mavenUpstream,
pluginPortalUpstreamURL: strings.TrimSuffix(pluginPortalUpstreamURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"),
proxyURL: strings.TrimSuffix(proxyURL, "/"),
} }
} }
@ -62,7 +51,8 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) {
filename := path.Base(urlPath) filename := path.Base(urlPath)
if h.isMetadataFile(filename) { if h.isMetadataFile(filename) {
h.handleMetadata(w, r, urlPath) cacheKey := strings.ReplaceAll(urlPath, "/", "_")
h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "maven", cacheKey, "*/*")
return return
} }
@ -76,32 +66,6 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) {
h.proxyUpstream(w, r) h.proxyUpstream(w, r)
} }
func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, urlPath string) {
cacheKey := strings.ReplaceAll(urlPath, "/", "_")
upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, urlPath)
body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*")
if err != nil {
if errors.Is(err, ErrUpstreamNotFound) {
pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath)
h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal",
"path", urlPath)
body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*")
}
}
if err != nil {
if errors.Is(err, ErrUpstreamNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
}
h.proxy.Logger.Error("metadata fetch failed", "error", err)
http.Error(w, "failed to fetch from upstream", http.StatusBadGateway)
return
}
h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType)
}
// handleDownload serves an artifact file, fetching and caching from upstream if needed. // handleDownload serves an artifact file, fetching and caching from upstream if needed.
func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, urlPath string) { func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, urlPath string) {
// Parse Maven path: group/artifact/version/filename // Parse Maven path: group/artifact/version/filename
@ -122,15 +86,8 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, ur
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, upstreamURL) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, upstreamURL)
if err != nil { if err != nil {
if errors.Is(err, ErrUpstreamNotFound) { h.proxy.Logger.Error("failed to get artifact", "error", err)
pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath) http.Error(w, "failed to fetch artifact", http.StatusBadGateway)
h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal",
"group", group, "artifact", artifact, "version", version, "filename", filename)
result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, pluginPortalURL)
}
}
if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch artifact")
return return
} }
@ -158,7 +115,7 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file
// isArtifactFile returns true if the filename looks like a Maven artifact. // isArtifactFile returns true if the filename looks like a Maven artifact.
func (h *MavenHandler) isArtifactFile(filename string) bool { func (h *MavenHandler) isArtifactFile(filename string) bool {
// Common artifact extensions // Common artifact extensions
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"} extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib"}
for _, ext := range extensions { for _, ext := range extensions {
if strings.HasSuffix(filename, ext) { if strings.HasSuffix(filename, ext) {
return true return true

View file

@ -52,7 +52,6 @@ func TestMavenIsArtifactFile(t *testing.T) {
}{ }{
{"guava-32.1.3-jre.jar", true}, {"guava-32.1.3-jre.jar", true},
{"guava-32.1.3-jre.pom", true}, {"guava-32.1.3-jre.pom", true},
{"guava-32.1.3-jre.module", true},
{"app-1.0.war", true}, {"app-1.0.war", true},
{"lib-1.0.aar", true}, {"lib-1.0.aar", true},
{"maven-metadata.xml", false}, {"maven-metadata.xml", false},
@ -66,63 +65,3 @@ func TestMavenIsArtifactFile(t *testing.T) {
} }
} }
} }
func TestMavenIsMetadataFile(t *testing.T) {
h := &MavenHandler{}
tests := []struct {
name string
filename string
want bool
}{
{
name: "pom is artifact, not metadata",
filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom",
want: false,
},
{
name: "pom checksum is metadata",
filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1",
want: true,
},
{
name: "metadata file",
filename: "maven-metadata.xml",
want: true,
},
{
name: "metadata checksum",
filename: "maven-metadata.xml.sha256",
want: true,
},
{
name: "jar checksum is metadata",
filename: "guava-32.1.3-jre.jar.sha1",
want: true,
},
{
name: "asc signature is metadata",
filename: "guava-32.1.3-jre.jar.asc",
want: true,
},
{
name: "regular jar is not metadata",
filename: "guava-32.1.3-jre.jar",
want: false,
},
{
name: "pom checksum is metadata",
filename: "guava-32.1.3-jre.pom.sha1",
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := h.isMetadataFile(tt.filename)
if got != tt.want {
t.Errorf("isMetadataFile(%q) = %v, want %v", tt.filename, got, tt.want)
}
})
}
}

View file

@ -1,139 +0,0 @@
package handler
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/git-pkgs/registries/fetch"
)
func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) {
tests := []struct {
name string
path string
handler func(p *Proxy) http.Handler
}{
{"debian", "/pool/main/n/nginx/nginx_1.18.0-6_amd64.deb",
func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost", "").Routes() }},
{"rpm", "/releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm",
func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }},
{"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg",
func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }},
{"pypi", "/packages/packages/ab/cd/ef0123456789/requests-2.31.0-py3-none-any.whl",
func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://localhost").Routes() }},
{"cran", "/src/contrib/ggplot2_3.4.4.tar.gz",
func(p *Proxy) http.Handler { return NewCRANHandler(p, "http://localhost").Routes() }},
{"conda", "/conda-forge/linux-64/numpy-1.26.0-py311_0.tar.bz2",
func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }},
{"conan", "/v1/files/zlib/1.3.1/_/_/0/recipe/conan_sources.tgz",
func(p *Proxy) http.Handler { return NewConanHandler(p, "http://localhost").Routes() }},
{"gem", "/gems/rails-7.1.0.gem",
func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }},
{"hex", "/tarballs/phoenix-1.7.10.tar",
func(p *Proxy) http.Handler { return NewHexHandler(p, "http://localhost").Routes() }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
fetcher.fetchErr = fetch.ErrNotFound
srv := httptest.NewServer(tt.handler(proxy))
defer srv.Close()
resp, err := http.Get(srv.URL + tt.path)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode)
}
})
}
}
func TestJuliaPackageUpstreamNotFoundReturns404(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
fetcher.fetchErr = fetch.ErrNotFound
dead := httptest.NewServer(http.NotFoundHandler())
defer dead.Close()
h := NewJuliaHandler(proxy, "http://localhost")
h.upstreamURL = dead.URL
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL +
"/package/7876af07-990d-54b4-ab0e-23690620f79a/0123456789abcdef0123456789abcdef01234567")
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("want 404 for missing upstream package, got %d", resp.StatusCode)
}
}
func TestComposerDownloadUpstreamNotFoundReturns404(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
fetcher.fetchErr = fetch.ErrNotFound
meta := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/p2/monolog/monolog.json" {
_, _ = w.Write([]byte(`{
"packages": {
"monolog/monolog": [
{"version": "2.9.1", "dist": {"url": "https://example.com/monolog-2.9.1.zip", "type": "zip"}}
]
}
}`))
return
}
http.NotFound(w, r)
}))
defer meta.Close()
h := &ComposerHandler{proxy: proxy, repoURL: meta.URL, proxyURL: "http://localhost"}
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL + "/files/monolog/monolog/2.9.1/monolog-2.9.1.zip")
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("want 404 for missing upstream dist, got %d", resp.StatusCode)
}
}
func TestContainerBlobUpstreamNotFoundReturns404(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
fetcher.fetchErr = fetch.ErrNotFound
h := &ContainerHandler{
proxy: proxy,
registryURL: "https://registry-1.docker.io",
proxyURL: "http://localhost:8080",
}
req := httptest.NewRequest(http.MethodGet,
"/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusNotFound {
t.Errorf("want 404 for missing upstream blob, got %d; body: %s", w.Code, w.Body.String())
}
if !strings.Contains(w.Body.String(), "BLOB_UNKNOWN") {
t.Errorf("want BLOB_UNKNOWN error code in body, got: %s", w.Body.String())
}
}

View file

@ -1,83 +0,0 @@
package handler
import (
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/git-pkgs/registries/fetch"
)
func TestErrUpstreamNotFoundWrapsFetchErrNotFound(t *testing.T) {
if !errors.Is(ErrUpstreamNotFound, fetch.ErrNotFound) {
t.Fatal("ErrUpstreamNotFound does not wrap fetch.ErrNotFound")
}
}
func TestGetOrFetchArtifactFromURL_NotFound(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
fetcher.fetchErr = fetch.ErrNotFound
_, err := proxy.GetOrFetchArtifactFromURL(context.Background(),
"maven", "org.example:missing", "1.0", "missing-1.0.jar",
"http://upstream.test/org/example/missing/1.0/missing-1.0.jar")
if !errors.Is(err, ErrUpstreamNotFound) {
t.Fatalf("want ErrUpstreamNotFound, got %v", err)
}
}
func TestMavenHandler_UpstreamNotFoundReturns404(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
fetcher.fetchErr = fetch.ErrNotFound
h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test")
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL + "/org/example/missing/1.0/missing-1.0.jar")
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode)
}
}
func TestMavenHandler_PluginPortalFallback(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
fetcher.fetchErrByURL = map[string]error{
"http://upstream.test/org/example/plugin/1.0/plugin-1.0.jar": fetch.ErrNotFound,
}
fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("portal artifact")),
ContentType: "application/java-archive",
}
h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test")
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL + "/org/example/plugin/1.0/plugin-1.0.jar")
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
t.Fatalf("want 200 via plugin portal fallback, got %d", resp.StatusCode)
}
body, _ := io.ReadAll(resp.Body)
if string(body) != "portal artifact" {
t.Errorf("want portal artifact body, got %q", body)
}
if fetcher.fetchedURL != "http://portal.test/org/example/plugin/1.0/plugin-1.0.jar" {
t.Errorf("fallback did not hit plugin portal, last URL: %s", fetcher.fetchedURL)
}
}

View file

@ -9,11 +9,13 @@ import (
"sort" "sort"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
) )
const ( const (
npmUpstream = "https://registry.npmjs.org" npmUpstream = "https://registry.npmjs.org"
npmAcceptDefault = "application/vnd.npm.install-v1+json;q=1.0, application/json;q=0.8" npmAbbreviatedCT = "application/vnd.npm.install-v1+json"
scopedParts = 2 // scope + name in scoped packages scopedParts = 2 // scope + name in scoped packages
) )
@ -25,14 +27,10 @@ type NPMHandler struct {
} }
// NewNPMHandler creates a new npm protocol handler. // NewNPMHandler creates a new npm protocol handler.
func NewNPMHandler(proxy *Proxy, proxyURL, upstreamURL string) *NPMHandler { func NewNPMHandler(proxy *Proxy, proxyURL string) *NPMHandler {
if strings.TrimSpace(upstreamURL) == "" {
upstreamURL = npmUpstream
}
return &NPMHandler{ return &NPMHandler{
proxy: proxy, proxy: proxy,
upstreamURL: strings.TrimSuffix(upstreamURL, "/"), upstreamURL: npmUpstream,
proxyURL: strings.TrimSuffix(proxyURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"),
} }
} }
@ -71,12 +69,9 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques
upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName))
// Prefer the smaller abbreviated packument format but include application/json // Use abbreviated metadata when cooldown is disabled — it's much smaller
// as a fallback so upstreams that reject the abbreviated type (e.g. JFrog // (e.g. drizzle-orm: 4MB vs 92MB) but lacks the time map needed for cooldown.
// Artifactory, which returns 406) can still respond with full metadata. accept := npmAbbreviatedCT
// When cooldown is enabled we must use full metadata exclusively because the
// abbreviated format omits the "time" map required for version age filtering.
accept := npmAcceptDefault
if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() {
accept = contentTypeJSON accept = contentTypeJSON
} }
@ -139,7 +134,7 @@ func (h *NPMHandler) applyCooldownFiltering(metadata map[string]any, versions ma
return return
} }
packagePURL := canonicalPackagePURL("npm", packageName) packagePURL := purl.MakePURLString("npm", packageName, "")
for version := range versions { for version := range versions {
publishedStr, ok := timeMap[version].(string) publishedStr, ok := timeMap[version].(string)
@ -268,27 +263,8 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
h.proxy.Logger.Info("npm download request", h.proxy.Logger.Info("npm download request",
"package", packageName, "version", version, "filename", filename) "package", packageName, "version", version, "filename", filename)
if h.versionInCooldown(r, packageName, version) { result, err := h.proxy.GetOrFetchArtifact(r.Context(), "npm", packageName, version, filename)
h.proxy.Logger.Info("cooldown: withholding npm tarball",
"package", packageName, "version", version)
JSONError(w, http.StatusNotFound, "version not found")
return
}
downloadURL := fmt.Sprintf(
"%s/%s/-/%s",
h.upstreamURL,
escapeNPMDownloadPackage(packageName),
url.PathEscape(filename),
)
result, err := h.proxy.GetOrFetchArtifactFromURL(
r.Context(), "npm", packageName, version, filename, downloadURL,
)
if err != nil { if err != nil {
if errors.Is(err, ErrUpstreamNotFound) {
JSONError(w, http.StatusNotFound, "package not found")
return
}
h.proxy.Logger.Error("failed to get artifact", "error", err) h.proxy.Logger.Error("failed to get artifact", "error", err)
JSONError(w, http.StatusBadGateway, "failed to fetch package") JSONError(w, http.StatusBadGateway, "failed to fetch package")
return return
@ -297,58 +273,6 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
ServeArtifact(w, result) ServeArtifact(w, result)
} }
// versionInCooldown reports whether a version is still inside the cooldown
// window. Filtering the packument is not enough on its own: tarball URLs are
// predictable and lockfiles record them directly, so `npm ci` reaches the
// download path without ever requesting metadata.
//
// The packument is served from the metadata cache, so this normally costs no
// extra upstream request. A version with no usable publish time is allowed
// through, matching how applyCooldownFiltering treats it.
func (h *NPMHandler) versionInCooldown(r *http.Request, packageName, version string) bool {
if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() {
return false
}
upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName))
body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON)
if err != nil {
h.proxy.Logger.Warn("cooldown: could not fetch npm metadata for download check",
"package", packageName, "version", version, "error", err)
return false
}
var metadata struct {
Time map[string]string `json:"time"`
}
if err := json.Unmarshal(body, &metadata); err != nil {
h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check",
"package", packageName, "version", version, "error", err)
return false
}
published, ok := metadata.Time[version]
if !ok {
return false
}
publishedAt, err := time.Parse(time.RFC3339, published)
if err != nil {
return false
}
return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt)
}
func escapeNPMDownloadPackage(packageName string) string {
scope, name, scoped := strings.Cut(packageName, "/")
if scoped && strings.HasPrefix(scope, "@") && len(scope) > 1 && name != "" && !strings.Contains(name, "/") {
return url.PathEscape(scope) + "/" + url.PathEscape(name)
}
return url.PathEscape(packageName)
}
// extractPackageName extracts the package name from the request path. // extractPackageName extracts the package name from the request path.
// Handles both scoped (@scope/name) and unscoped (name) packages. // Handles both scoped (@scope/name) and unscoped (name) packages.
func (h *NPMHandler) extractPackageName(r *http.Request) string { func (h *NPMHandler) extractPackageName(r *http.Request) string {

View file

@ -2,16 +2,13 @@ package handler
import ( import (
"encoding/json" "encoding/json"
"io"
"log/slog" "log/slog"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings"
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
"github.com/git-pkgs/registries/fetch"
) )
const testVersion100 = "1.0.0" const testVersion100 = "1.0.0"
@ -49,86 +46,6 @@ func TestNPMExtractVersionFromFilename(t *testing.T) {
} }
} }
func TestNPMHandlerUsesConfiguredUpstream(t *testing.T) {
t.Run("metadata", func(t *testing.T) {
var requestPath, authHeader string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestPath = r.URL.Path
authHeader = r.Header.Get("Authorization")
if authHeader != "Bearer npm-token" {
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"versions":{}}`)
}))
defer upstream.Close()
proxy, _, _, _ := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
proxy.AuthForURL = func(string) (string, string) {
return "Authorization", "Bearer npm-token"
}
h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/root/")
req := httptest.NewRequest(http.MethodGet, "/testpkg", nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
}
if requestPath != "/root/testpkg" {
t.Errorf("upstream path = %q, want %q", requestPath, "/root/testpkg")
}
if authHeader != "Bearer npm-token" {
t.Errorf("Authorization = %q, want %q", authHeader, "Bearer npm-token")
}
})
t.Run("download", func(t *testing.T) {
proxy, _, _, artifactFetcher := setupTestProxy(t)
artifactFetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("package")),
ContentType: "application/gzip",
}
h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/")
req := httptest.NewRequest(http.MethodGet, "/testpkg/-/testpkg-1.0.0.tgz", nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
}
want := "https://npm.example.test/root/testpkg/-/testpkg-1.0.0.tgz"
if artifactFetcher.fetchedURL != want {
t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want)
}
})
t.Run("scoped download", func(t *testing.T) {
proxy, _, _, artifactFetcher := setupTestProxy(t)
artifactFetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("package")),
ContentType: "application/gzip",
}
h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/")
req := httptest.NewRequest(http.MethodGet, "/@scope/name/-/name-1.0.0.tgz", nil)
w := httptest.NewRecorder()
h.Routes().ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String())
}
want := "https://npm.example.test/root/@scope/name/-/name-1.0.0.tgz"
if artifactFetcher.fetchedURL != want {
t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want)
}
})
}
func TestNPMRewriteMetadata(t *testing.T) { func TestNPMRewriteMetadata(t *testing.T) {
h := &NPMHandler{ h := &NPMHandler{
proxy: testProxy(), proxy: testProxy(),
@ -396,7 +313,7 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) {
})) }))
defer upstream.Close() defer upstream.Close()
t.Run("no cooldown uses combined accept header", func(t *testing.T) { t.Run("no cooldown uses abbreviated metadata", func(t *testing.T) {
h := &NPMHandler{ h := &NPMHandler{
proxy: testProxy(), proxy: testProxy(),
upstreamURL: upstream.URL, upstreamURL: upstream.URL,
@ -407,12 +324,12 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) {
w := httptest.NewRecorder() w := httptest.NewRecorder()
h.handlePackageMetadata(w, req) h.handlePackageMetadata(w, req)
if gotAccept != npmAcceptDefault { if gotAccept != npmAbbreviatedCT {
t.Errorf("Accept = %q, want %q", gotAccept, npmAcceptDefault) t.Errorf("Accept = %q, want abbreviated metadata header", gotAccept)
} }
}) })
t.Run("cooldown enabled uses full metadata only", func(t *testing.T) { t.Run("cooldown enabled uses full metadata", func(t *testing.T) {
proxy := testProxy() proxy := testProxy()
proxy.Cooldown = &cooldown.Config{Default: "3d"} proxy.Cooldown = &cooldown.Config{Default: "3d"}
@ -426,8 +343,8 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) {
w := httptest.NewRecorder() w := httptest.NewRecorder()
h.handlePackageMetadata(w, req) h.handlePackageMetadata(w, req)
if gotAccept != contentTypeJSON { if gotAccept == npmAbbreviatedCT {
t.Errorf("Accept = %q, want %q (cooldown requires full metadata)", gotAccept, contentTypeJSON) t.Error("cooldown enabled should use full metadata, not abbreviated")
} }
}) })
} }
@ -454,81 +371,3 @@ func TestNPMHandlerMetadataNotFound(t *testing.T) {
t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound) t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound)
} }
} }
func TestNPMDownloadCooldown(t *testing.T) {
now := time.Now()
packument := `{
"name": "leftpad",
"dist-tags": {"latest": "2.0.0"},
"time": {
"1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `",
"2.0.0": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"
},
"versions": {"1.0.0": {}, "2.0.0": {}}
}`
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", contentTypeJSON)
_, _ = io.WriteString(w, packument)
}))
defer upstream.Close()
tests := []struct {
name string
version string
wantStatus int
}{
{"published before the window serves the tarball", testVersion100, http.StatusOK},
{"published inside the window is withheld", "2.0.0", http.StatusNotFound},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
proxy.Cooldown = &cooldown.Config{Default: "7d"}
fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("tarball data")),
ContentType: "application/octet-stream",
}
h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL)
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz")
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != tt.wantStatus {
t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus)
}
if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled {
t.Error("fetched a version that is still inside the cooldown window")
}
})
}
}
func TestNPMDownloadCooldownDisabled(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
t.Error("metadata must not be fetched when cooldown is disabled")
w.WriteHeader(http.StatusInternalServerError)
}))
defer upstream.Close()
proxy, _, _, fetcher := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("tarball data")),
ContentType: "application/octet-stream",
}
h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL)
if h.versionInCooldown(httptest.NewRequest(http.MethodGet, "/", nil), "leftpad", testVersion100) {
t.Error("versionInCooldown = true, want false when cooldown is not configured")
}
}

View file

@ -8,6 +8,8 @@ import (
"net/http" "net/http"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
) )
const ( const (
@ -191,7 +193,7 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request
return return
} }
body, err := h.proxy.ReadMetadata(resp.Body) body, err := ReadMetadata(resp.Body)
if err != nil { if err != nil {
http.Error(w, "failed to read response", http.StatusInternalServerError) http.Error(w, "failed to read response", http.StatusInternalServerError)
return return
@ -269,7 +271,7 @@ func (h *NuGetHandler) applyCooldownFiltering(body []byte) ([]byte, error) {
} }
} }
packagePURL := canonicalPackagePURL("nuget", strings.ToLower(id)) packagePURL := purl.MakePURLString("nuget", strings.ToLower(id), "")
if !h.proxy.Cooldown.IsAllowed("nuget", packagePURL, publishedAt) { if !h.proxy.Cooldown.IsAllowed("nuget", packagePURL, publishedAt) {
h.proxy.Logger.Info("cooldown: filtering nuget version", h.proxy.Logger.Info("cooldown: filtering nuget version",
@ -314,7 +316,8 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "nuget", name, version, filename, upstreamURL) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "nuget", name, version, filename, upstreamURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return return
} }

View file

@ -10,7 +10,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
) )
func nugetTestProxy() *Proxy { func nugetTestProxy() *Proxy {

View file

@ -7,6 +7,8 @@ import (
"net/http" "net/http"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
) )
const ( const (
@ -67,7 +69,8 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
result, err := h.proxy.GetOrFetchArtifact(r.Context(), "pub", name, version, filename) result, err := h.proxy.GetOrFetchArtifact(r.Context(), "pub", name, version, filename)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return return
} }
@ -124,7 +127,7 @@ func (h *PubHandler) rewriteMetadata(name string, body []byte) ([]byte, error) {
return body, nil return body, nil
} }
packagePURL := canonicalPackagePURL("pub", name) packagePURL := purl.MakePURLString("pub", name, "")
filtered := h.filterAndRewriteVersions(name, packagePURL, versions) filtered := h.filterAndRewriteVersions(name, packagePURL, versions)
metadata["versions"] = filtered metadata["versions"] = filtered

View file

@ -6,7 +6,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
) )
func TestPubRewriteMetadata(t *testing.T) { func TestPubRewriteMetadata(t *testing.T) {

View file

@ -12,6 +12,8 @@ import (
"regexp" "regexp"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/purl"
) )
const ( const (
@ -19,14 +21,7 @@ const (
minWheelParts = 5 // name + version + python + abi + platform minWheelParts = 5 // name + version + python + abi + platform
minSubmatchParts = 2 // full match + first capture group minSubmatchParts = 2 // full match + first capture group
minPyPIPathParts = 3 // hash_prefix + hash + filename minPyPIPathParts = 3 // hash_prefix + hash + filename
minEggParts = 3 // name + version + python tag minPythonTagLen = 2 // minimum length for a python tag (e.g., "py")
// PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip
// appends to a distribution URL when the index advertises core metadata.
// A sidecar resolves to the same name and version as the distribution it
// describes, so it is cached alongside it; consumers that expect an openable
// archive must skip these.
PyPIMetadataSuffix = ".metadata"
) )
// PyPIHandler handles PyPI registry protocol requests. // PyPIHandler handles PyPI registry protocol requests.
@ -110,14 +105,24 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request
// that should be filtered out due to cooldown. // that should be filtered out due to cooldown.
func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[string]bool { func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[string]bool {
jsonURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name) jsonURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name)
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, jsonURL, nil)
body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", name+"/json", jsonURL)
if err != nil { if err != nil {
return nil return nil
} }
req.Header.Set("Accept", "application/json")
resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
return nil
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil
}
var metadata map[string]any var metadata map[string]any
if err := json.Unmarshal(body, &metadata); err != nil { if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
return nil return nil
} }
@ -126,7 +131,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[st
return nil return nil
} }
packagePURL := canonicalPackagePURL("pypi", name) packagePURL := purl.MakePURLString("pypi", name, "")
filtered := make(map[string]bool) filtered := make(map[string]bool)
for version, files := range releases { for version, files := range releases {
@ -257,7 +262,7 @@ func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, error) {
packageName, _ := extractPyPIName(metadata) packageName, _ := extractPyPIName(metadata)
packagePURL := "" packagePURL := ""
if packageName != "" { if packageName != "" {
packagePURL = canonicalPackagePURL("pypi", packageName) packagePURL = purl.MakePURLString("pypi", packageName, "")
} }
h.filterAndRewriteReleases(metadata, packageName, packagePURL) h.filterAndRewriteReleases(metadata, packageName, packagePURL)
@ -301,21 +306,6 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool {
return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt)
} }
// versionInCooldown reports whether a version is still inside the cooldown
// window. Filtering the simple index is not enough on its own: file URLs are
// recorded in lockfiles and requirements pins, so pip can reach the download
// path without ever reading the index.
//
// A release whose upload time cannot be determined is allowed through, matching
// how fetchFilteredVersions treats it.
func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) bool {
if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() {
return false
}
return h.fetchFilteredVersions(r, name)[version]
}
// rewriteFileEntries rewrites URLs in a list of file entries. // rewriteFileEntries rewrites URLs in a list of file entries.
func (h *PyPIHandler) rewriteFileEntries(files any) { func (h *PyPIHandler) rewriteFileEntries(files any) {
filesArr, ok := files.([]any) filesArr, ok := files.([]any)
@ -422,13 +412,6 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
filename := parts[len(parts)-1] filename := parts[len(parts)-1]
name, version := h.parseFilename(filename) name, version := h.parseFilename(filename)
if name != "" && h.versionInCooldown(r, name, version) {
h.proxy.Logger.Info("cooldown: withholding pypi file",
"name", name, "version", version, "filename", filename)
http.Error(w, "not found", http.StatusNotFound)
return
}
if name == "" { if name == "" {
// Can't determine name/version, use hash as identifier // Can't determine name/version, use hash as identifier
name = fmt.Sprintf("_hash_%s", hashPath(path)) name = fmt.Sprintf("_hash_%s", hashPath(path))
@ -445,170 +428,64 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "pypi", name, version, filename, upstreamURL) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "pypi", name, version, filename, upstreamURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") h.proxy.Logger.Error("failed to get artifact", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return return
} }
ServeArtifact(w, result) ServeArtifact(w, result)
} }
// archiveExtensions are sdist formats of the form {name}-{version}{ext}. They
// carry no trailing tags, but legacy sdist names may contain hyphens.
var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".tgz", ".tar", ".zip"}
// windowsInstallerExtensions are the legacy distutils bdist_wininst and
// bdist_msi formats, which share a filename layout.
var windowsInstallerExtensions = []string{".exe", ".msi"}
// parseFilename extracts package name and version from a PyPI filename. // parseFilename extracts package name and version from a PyPI filename.
// Handles wheels, sdists and legacy bdist formats: // Handles both wheels and sdists:
// - requests-2.31.0-py3-none-any.whl // - requests-2.31.0-py3-none-any.whl
// - requests-2.31.0.tar.gz // - requests-2.31.0.tar.gz
// - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg
// - numpy-1.8.0.win32-py2.7.exe
func (h *PyPIHandler) parseFilename(filename string) (name, version string) { func (h *PyPIHandler) parseFilename(filename string) (name, version string) {
// PEP 658/714 core-metadata sidecars are the distribution filename plus // Try wheel format first: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl
// ".metadata"; they describe the same name and version. Without this, pip's if strings.HasSuffix(filename, ".whl") {
// metadata-only fetches fall back to a hash-derived package identifier. base := strings.TrimSuffix(filename, ".whl")
filename = strings.TrimSuffix(filename, PyPIMetadataSuffix) parts := strings.Split(base, "-")
if len(parts) >= minWheelParts {
switch { // Find where version ends (version followed by python tag)
case strings.HasSuffix(filename, ".whl"): for i := 1; i < len(parts)-2; i++ {
return parseWheelFilename(strings.TrimSuffix(filename, ".whl")) // Check if this looks like a python tag (py2, py3, cp39, etc)
case strings.HasSuffix(filename, ".egg"): if isPythonTag(parts[i]) {
return parseEggFilename(strings.TrimSuffix(filename, ".egg")) name = strings.Join(parts[:i-1], "-")
} version = parts[i-1]
return
for _, ext := range windowsInstallerExtensions { }
if strings.HasSuffix(filename, ext) { }
return parseWindowsInstallerFilename(strings.TrimSuffix(filename, ext))
} }
} }
for _, ext := range archiveExtensions { // Try sdist formats: {name}-{version}.tar.gz, {name}-{version}.zip
for _, ext := range []string{".tar.gz", ".tar.bz2", ".zip", ".tar"} {
if strings.HasSuffix(filename, ext) { if strings.HasSuffix(filename, ext) {
return splitNameVersion(strings.TrimSuffix(filename, ext)) base := strings.TrimSuffix(filename, ext)
// Find last hyphen followed by version
for i := len(base) - 1; i >= 0; i-- {
if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) {
return base[:i], base[i+1:]
}
}
} }
} }
return "", "" return "", ""
} }
// parseWheelFilename parses the PEP 427 layout func isPythonTag(s string) bool {
// {name}-{version}(-{build})?-{python}-{abi}-{platform}, base being the if len(s) < minPythonTagLen {
// filename without its ".whl" suffix. The spec escapes every hyphen in the name
// and version to '_', so the first two fields are authoritative even when the
// optional build tag is present.
func parseWheelFilename(base string) (name, version string) {
parts := strings.Split(base, "-")
if len(parts) < minWheelParts {
return "", ""
}
return parts[0], parts[1]
}
// parseEggFilename parses the setuptools bdist_egg layout
// {name}-{version}-py{X.Y}(-{platform})?, base being the filename without its
// ".egg" suffix. setuptools escapes hyphens in the name and version to '_', but
// eggs built by other tooling do not always, so the version is located relative
// to the interpreter field rather than assumed to be the second field.
func parseEggFilename(base string) (name, version string) {
parts := strings.Split(base, "-")
// Scan from the end: the trailing platform fields never look like an
// interpreter tag, so the last match is the real one even when the package
// name itself carries a "py{N}" component. Stop before index 1, since a tag
// any earlier would leave no room for both a name and a version.
for i := len(parts) - 1; i >= minEggParts-1; i-- {
if !isEggPythonTag(parts[i]) || !isVersionField(parts[i-1]) {
continue
}
return strings.Join(parts[:i-1], "-"), parts[i-1]
}
// No interpreter field: {name}-{version}.
return splitNameVersion(base)
}
// parseWindowsInstallerFilename parses the distutils bdist_wininst and
// bdist_msi layout {name}-{version}.{platform}(-py{X.Y})?, base being the
// filename without its ".exe" or ".msi" suffix. The platform is joined to the
// version with a '.' rather than a '-' and may itself contain a hyphen
// ("win-amd64"), so both trailing fields are stripped before the name and
// version are split apart.
func parseWindowsInstallerFilename(base string) (name, version string) {
if i := strings.LastIndex(base, "-py"); i >= 0 && isDottedNumber(base[i+len("-py"):]) {
base = base[:i]
}
// The platform is the final '.'-separated field. Requiring it to start with
// a non-digit keeps a dotted version from being truncated when a filename
// carries no platform tag.
i := strings.LastIndex(base, ".")
if i < 0 || i+1 >= len(base) || isVersionStart(base[i+1]) {
return "", ""
}
return splitFullname(base[:i])
}
// splitFullname splits the distutils fullname {name}-{version} that precedes a
// Windows installer's platform field. Unlike an sdist, a wininst fullname may
// carry a trailing build variant ("cx_Oracle-5.1.2-11g"), which belongs to
// neither the name nor the version, so the first purely numeric field wins and
// anything after it is discarded.
func splitFullname(fullname string) (name, version string) {
parts := strings.Split(fullname, "-")
for i := 1; i < len(parts); i++ {
if isDottedNumber(parts[i]) {
return strings.Join(parts[:i], "-"), parts[i]
}
}
// No purely numeric field, e.g. a prerelease version like "1.0b1".
return splitNameVersion(fullname)
}
// splitNameVersion splits a {name}-{version} pair at the last hyphen that
// starts a version, leaving hyphens inside the name intact.
func splitNameVersion(base string) (name, version string) {
for i := len(base) - 1; i >= 0; i-- {
if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) {
return base[:i], base[i+1:]
}
}
return "", ""
}
// isEggPythonTag reports whether field is the py{X.Y} interpreter field that
// setuptools places directly after the version in an egg filename.
func isEggPythonTag(field string) bool {
const prefix = "py"
return len(field) > len(prefix) && strings.HasPrefix(field, prefix) && isVersionStart(field[len(prefix)])
}
// isVersionField reports whether field can be a version, i.e. it is non-empty
// and starts with a digit as every PEP 440 release segment does.
func isVersionField(field string) bool {
return field != "" && isVersionStart(field[0])
}
// isDottedNumber reports whether s is a dotted numeric version such as "2.7".
func isDottedNumber(s string) bool {
if s == "" || !isVersionStart(s[0]) {
return false return false
} }
// Python tags start with py, cp, pp, ip, jy
for i := range len(s) { prefixes := []string{"py", "cp", "pp", "ip", "jy"}
if !isVersionStart(s[i]) && s[i] != '.' { for _, p := range prefixes {
return false if strings.HasPrefix(s, p) {
return true
} }
} }
return false
return true
} }
func isVersionStart(c byte) bool { func isVersionStart(c byte) bool {

View file

@ -7,11 +7,10 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings" "strings"
"sync/atomic"
"testing" "testing"
"time" "time"
"github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/cooldown"
"github.com/git-pkgs/registries/fetch" "github.com/git-pkgs/registries/fetch"
) )
@ -29,58 +28,13 @@ func TestPyPIParseFilename(t *testing.T) {
{"aws-sdk-1.0.0.tar.gz", "aws-sdk", "1.0.0"}, {"aws-sdk-1.0.0.tar.gz", "aws-sdk", "1.0.0"},
{"zipp-3.17.0.zip", "zipp", "3.17.0"}, {"zipp-3.17.0.zip", "zipp", "3.17.0"},
// Additional sdist archive formats
{"lxml-4.9.3.tar.xz", "lxml", "4.9.3"},
{"docutils-0.20.1.tgz", "docutils", "0.20.1"},
{"psycopg2-2.9.9.tar.bz2", "psycopg2", "2.9.9"},
// Wheel formats // Wheel formats
{"requests-2.31.0-py3-none-any.whl", "requests", "2.31.0"}, {"requests-2.31.0-py3-none-any.whl", "requests", "2.31.0"},
{"numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.whl", "numpy", "1.26.2"}, {"numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.whl", "numpy", "1.26.2"},
{"cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", "cryptography", "41.0.5"}, {"cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", "cryptography", "41.0.5"},
// Wheels with a build tag must not fold the tag into the version
{"foo-1.0-1-py3-none-any.whl", "foo", "1.0"},
{"tensorflow-2.15.0-2-cp311-cp311-manylinux_2_17_x86_64.whl", "tensorflow", "2.15.0"},
// PEP 658 core-metadata sidecars resolve to the distribution they describe
{"backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", "backports_asyncio_runner", "1.2.0"},
{"requests-2.31.0-py3-none-any.whl.metadata", "requests", "2.31.0"},
{"requests-2.31.0.tar.gz.metadata", "requests", "2.31.0"},
// Eggs: {name}-{version}-py{X.Y}(-{platform})?.egg. Unescaped hyphens in
// the name must not be mistaken for the field separator before the version.
{"numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg", "numpy", "1.8.0"},
{"aws-sdk-1.0.0-py3.11.egg", "aws-sdk", "1.0.0"},
{"aws-sdk-1.0.0-py2.7-macosx-10.9-x86_64.egg", "aws-sdk", "1.0.0"},
{"aws-sdk-1.0.0.egg", "aws-sdk", "1.0.0"},
// A "py{N}" component inside the name is not the interpreter field, so
// the interpreter must be located from the end of the filename.
{"django-rest-py3-1.0-py3.6.egg", "django-rest-py3", "1.0"},
// Windows installers: {name}-{version}.{platform}(-py{X.Y})?.{exe,msi}.
// The platform is not part of the version, and may contain a hyphen.
{"foo-1.0.win32-py2.0.exe", "foo", "1.0"},
{"pywin32-223.win32-py2.7.exe", "pywin32", "223"},
{"numpy-1.8.0.win-amd64-py2.7.exe", "numpy", "1.8.0"},
{"aws-sdk-1.0.0.win32-py2.7.exe", "aws-sdk", "1.0.0"},
{"pywin32-223.win32.exe", "pywin32", "223"},
{"cx_Oracle-5.1.2.win32-py2.7.msi", "cx_Oracle", "5.1.2"},
{"numpy-1.8.0.win-amd64.msi", "numpy", "1.8.0"},
// A trailing build variant belongs to neither the name nor the version.
{"cx_Oracle-5.1.2-11g.win32-py2.7.exe", "cx_Oracle", "5.1.2"},
// A prerelease version has no purely numeric field to anchor on.
{"foo-1.0b1.win32-py2.7.exe", "foo", "1.0b1"},
// Invalid // Invalid
{"invalid", "", ""}, {"invalid", "", ""},
{"invalid.metadata", "", ""},
{"backports.ssl_match_hostname-3.4.0.2-py2.7.whl", "", ""},
{"invalid.exe", "", ""},
{"foo-1.0.exe", "", ""},
// An egg with an interpreter field but no version must not promote the
// trailing component of a hyphenated name to the version.
{"aws-sdk-py2.7.egg", "", ""},
} }
for _, tt := range tests { for _, tt := range tests {
@ -140,24 +94,25 @@ func TestPyPIRewriteJSONMetadataCooldown(t *testing.T) {
} }
} }
// TestPyPIParseFilenameNoHashFallback guards the identifier used for caching: func TestIsPythonTag(t *testing.T) {
// a filename that parses to an empty name makes handleDownload fall back to a tests := []struct {
// "_hash_<digest>" package name, which surfaces as a bogus PURL in the package tag string
// overview. want bool
func TestPyPIParseFilenameNoHashFallback(t *testing.T) { }{
h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}} {"py3", true},
{"py2", true},
filenames := []string{ {"cp311", true},
"backports_asyncio_runner-1.2.0-py3-none-any.whl", {"cp37", true},
"backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", {"pp39", true},
"backports_asyncio_runner-1.2.0.tar.gz", {"none", false},
{"any", false},
{"manylinux", false},
} }
for _, filename := range filenames { for _, tt := range tests {
name, version := h.parseFilename(filename) got := isPythonTag(tt.tag)
if name != "backports_asyncio_runner" || version != "1.2.0" { if got != tt.want {
t.Errorf("parseFilename(%q) = (%q, %q), want (%q, %q)", t.Errorf("isPythonTag(%q) = %v, want %v", tt.tag, got, tt.want)
filename, name, version, "backports_asyncio_runner", "1.2.0")
} }
} }
} }
@ -237,116 +192,3 @@ func TestPyPIHandler_DownloadCacheMiss(t *testing.T) {
t.Error("expected fetcher to be called on cache miss") t.Error("expected fetcher to be called on cache miss")
} }
} }
func TestPyPIDownloadCooldown(t *testing.T) {
now := time.Now()
releases := `{"releases": {
"1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}],
"2.0.0": [{"upload_time_iso_8601": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"}]
}}`
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", contentTypeJSON)
_, _ = io.WriteString(w, releases)
}))
defer upstream.Close()
tests := []struct {
name string
filename string
wantStatus int
}{
{"published before the window serves the file", "newpkg-1.0.0.tar.gz", http.StatusOK},
{"published inside the window is withheld", "newpkg-2.0.0.tar.gz", http.StatusNotFound},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
proxy, _, _, fetcher := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
proxy.Cooldown = &cooldown.Config{Default: "7d"}
fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("sdist data")),
ContentType: "application/octet-stream",
}
h := &PyPIHandler{
proxy: proxy,
upstreamURL: upstream.URL,
proxyURL: "http://localhost",
}
srv := httptest.NewServer(h.Routes())
defer srv.Close()
resp, err := http.Get(srv.URL + "/packages/packages/ab/cd/ef0123456789/" + tt.filename)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != tt.wantStatus {
t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus)
}
if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled {
t.Error("fetched a version that is still inside the cooldown window")
}
})
}
}
// TestPyPIDownloadCooldownMetadataCache ensures that repeated downloads that
// trigger cooldown filtering reuse the cached PyPI JSON metadata instead of
// fetching it from upstream once per download.
func TestPyPIDownloadCooldownMetadataCache(t *testing.T) {
now := time.Now()
releases := `{"releases": {
"1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}],
"2.0.0": [{"upload_time_iso_8601": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"}]
}}`
var metadataRequests atomic.Int64
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/pypi/newpkg/json" {
metadataRequests.Add(1)
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, releases)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = io.WriteString(w, "package data")
}))
defer upstream.Close()
proxy, _, _, fetcher := setupTestProxy(t)
proxy.HTTPClient = upstream.Client()
proxy.CacheMetadata = true
proxy.MetadataTTL = time.Hour
proxy.Cooldown = &cooldown.Config{Default: "7d"}
fetcher.artifact = &fetch.Artifact{
Body: io.NopCloser(strings.NewReader("package data")),
ContentType: "application/octet-stream",
}
h := &PyPIHandler{
proxy: proxy,
upstreamURL: upstream.URL,
proxyURL: "http://localhost",
}
srv := httptest.NewServer(h.Routes())
defer srv.Close()
// Two downloads of the same package: one outside the cooldown window
// (served) and one inside (withheld). Both go through the download path
// that resolves filtered versions.
for _, filename := range []string{"newpkg-1.0.0.tar.gz", "newpkg-2.0.0.tar.gz"} {
resp, err := http.Get(srv.URL + "/packages/packages/ab/cd/ef0123456789/" + filename)
if err != nil {
t.Fatalf("request failed: %v", err)
}
_ = resp.Body.Close()
}
if got := metadataRequests.Load(); got != 1 {
t.Errorf("upstream metadata JSON requests = %d, want 1 (repeated downloads should reuse the cached metadata)", got)
}
}

View file

@ -7,12 +7,9 @@ import (
) )
func TestReadMetadata(t *testing.T) { func TestReadMetadata(t *testing.T) {
const limit = 1024
p := &Proxy{MetadataMaxSize: limit}
t.Run("small body", func(t *testing.T) { t.Run("small body", func(t *testing.T) {
data := []byte("hello world") data := []byte("hello world")
got, err := p.ReadMetadata(bytes.NewReader(data)) got, err := ReadMetadata(bytes.NewReader(data))
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@ -22,39 +19,27 @@ func TestReadMetadata(t *testing.T) {
}) })
t.Run("exactly at limit", func(t *testing.T) { t.Run("exactly at limit", func(t *testing.T) {
data := make([]byte, limit) data := make([]byte, maxMetadataSize)
for i := range data { for i := range data {
data[i] = 'x' data[i] = 'x'
} }
got, err := p.ReadMetadata(bytes.NewReader(data)) got, err := ReadMetadata(bytes.NewReader(data))
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
if len(got) != limit { if len(got) != int(maxMetadataSize) {
t.Errorf("got length %d, want %d", len(got), limit) t.Errorf("got length %d, want %d", len(got), maxMetadataSize)
} }
}) })
t.Run("over limit returns error", func(t *testing.T) { t.Run("over limit returns error", func(t *testing.T) {
data := make([]byte, limit+100) data := make([]byte, maxMetadataSize+100)
for i := range data { for i := range data {
data[i] = 'x' data[i] = 'x'
} }
_, err := p.ReadMetadata(bytes.NewReader(data)) _, err := ReadMetadata(bytes.NewReader(data))
if !errors.Is(err, ErrMetadataTooLarge) { if !errors.Is(err, ErrMetadataTooLarge) {
t.Errorf("got error %v, want ErrMetadataTooLarge", err) t.Errorf("got error %v, want ErrMetadataTooLarge", err)
} }
}) })
t.Run("zero limit uses default", func(t *testing.T) {
p := &Proxy{}
data := make([]byte, 1<<20)
got, err := p.ReadMetadata(bytes.NewReader(data))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(got) != len(data) {
t.Errorf("got length %d, want %d", len(got), len(data))
}
})
} }

View file

@ -83,7 +83,8 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques
result, err := h.proxy.GetOrFetchArtifactFromURL( result, err := h.proxy.GetOrFetchArtifactFromURL(
r.Context(), "rpm", name, version, filename, downloadURL) r.Context(), "rpm", name, version, filename, downloadURL)
if err != nil { if err != nil {
h.proxy.serveArtifactError(w, err, "failed to fetch package") h.proxy.Logger.Error("failed to get rpm package", "error", err)
http.Error(w, "failed to fetch package", http.StatusBadGateway)
return return
} }

View file

@ -1,74 +0,0 @@
package httpclient
import (
"log/slog"
"net/http"
"net/url"
"strings"
"time"
"github.com/git-pkgs/proxy/internal/accesslog"
)
type accessLogTransport struct {
base http.RoundTripper
accessLog *accesslog.Logger
logger *slog.Logger
}
// NewAccessLogTransport records each upstream HTTP exchange around base.
func NewAccessLogTransport(base http.RoundTripper, log *accesslog.Logger, logger *slog.Logger) http.RoundTripper {
if base == nil {
base = http.DefaultTransport
}
if logger == nil {
logger = slog.Default()
}
if log == nil {
return base
}
return &accessLogTransport{
base: base,
accessLog: log,
logger: logger,
}
}
func (t *accessLogTransport) RoundTrip(req *http.Request) (*http.Response, error) {
start := time.Now()
resp, err := t.base.RoundTrip(req)
entry := accesslog.Entry{
Event: accesslog.EventUpstream,
RequestID: accesslog.RequestID(req.Context()),
Method: req.Method,
URL: accesslog.URLWithoutSecrets(req.URL),
DurationMS: time.Since(start).Milliseconds(),
}
if resp != nil {
entry.StatusCode = resp.StatusCode
}
if err != nil {
entry.Error = errorWithoutSecrets(err, req.URL)
}
if writeErr := t.accessLog.Write(entry); writeErr != nil {
t.logger.Error("failed to write access log", "error", writeErr)
}
return resp, err
}
func errorWithoutSecrets(err error, requestURL *url.URL) string {
message := err.Error()
if requestURL == nil {
return message
}
cleanURL := accesslog.URLWithoutSecrets(requestURL)
for _, value := range []string{requestURL.String(), requestURL.Redacted()} {
if value != "" {
message = strings.ReplaceAll(message, value, cleanURL)
}
}
return message
}

View file

@ -1,121 +0,0 @@
package httpclient
import (
"bufio"
"encoding/json"
"errors"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"github.com/git-pkgs/proxy/internal/accesslog"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
func TestAccessLogTransportRecordsUpstreamStatus(t *testing.T) {
path := filepath.Join(t.TempDir(), "access.jsonl")
accessLogger, err := accesslog.Open(path)
if err != nil {
t.Fatal(err)
}
base := roundTripFunc(func(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusTooManyRequests,
Body: io.NopCloser(strings.NewReader("rate limited")),
Request: req,
}, nil
})
client := &http.Client{Transport: NewAccessLogTransport(base, accessLogger, slog.Default())}
req, err := http.NewRequest(http.MethodGet, "https://user:password@registry.example/package.tgz?token=secret", nil)
if err != nil {
t.Fatal(err)
}
req = req.WithContext(accesslog.WithRequestID(req.Context(), "request-123"))
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
if err := accessLogger.Close(); err != nil {
t.Fatal(err)
}
entry := readAccessLogEntry(t, path)
if entry.Event != accesslog.EventUpstream {
t.Errorf("event = %q, want %q", entry.Event, accesslog.EventUpstream)
}
if entry.RequestID != "request-123" {
t.Errorf("request_id = %q, want %q", entry.RequestID, "request-123")
}
if entry.StatusCode != http.StatusTooManyRequests {
t.Errorf("status_code = %d, want %d", entry.StatusCode, http.StatusTooManyRequests)
}
if entry.URL != "https://registry.example/package.tgz" {
t.Errorf("url = %q, want URL without credentials or query", entry.URL)
}
}
func TestAccessLogTransportRecordsUpstreamError(t *testing.T) {
path := filepath.Join(t.TempDir(), "access.jsonl")
accessLogger, err := accesslog.Open(path)
if err != nil {
t.Fatal(err)
}
wantErr := errors.New("GET https://user:password@registry.example/package.tgz?token=secret: connection refused")
base := roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, wantErr
})
client := &http.Client{Transport: NewAccessLogTransport(base, accessLogger, slog.Default())}
_, err = client.Get("https://user:password@registry.example/package.tgz?token=secret")
if !errors.Is(err, wantErr) {
t.Fatalf("GET error = %v, want %v", err, wantErr)
}
if err := accessLogger.Close(); err != nil {
t.Fatal(err)
}
entry := readAccessLogEntry(t, path)
if entry.StatusCode != 0 {
t.Errorf("status_code = %d, want 0", entry.StatusCode)
}
if strings.Contains(entry.Error, "password") || strings.Contains(entry.Error, "secret") {
t.Errorf("error contains URL credentials or query: %q", entry.Error)
}
if !strings.Contains(entry.Error, "connection refused") {
t.Errorf("error = %q, want connection failure", entry.Error)
}
}
func readAccessLogEntry(t *testing.T, path string) accesslog.Entry {
t.Helper()
file, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = file.Close() }()
scanner := bufio.NewScanner(file)
if !scanner.Scan() {
t.Fatalf("access log is empty: %v", scanner.Err())
}
var entry accesslog.Entry
if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
t.Fatalf("decoding access log: %v", err)
}
return entry
}

View file

@ -1,433 +0,0 @@
// Package httpclient provides authentication-aware HTTP transports for upstream requests.
package httpclient
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
)
const (
defaultTokenLifetime = 60 * time.Second
tokenExpirySkew = 5 * time.Second
maxTokenResponseSize = 1 << 20
shortTokenSkewDivisor = 10
)
// AuthFunc returns a configured authentication header for a URL.
type AuthFunc func(url string) (headerName, headerValue string)
// Transport adds configured authentication and follows OCI Bearer challenges.
type Transport struct {
base http.RoundTripper
authForURL AuthFunc
mu sync.Mutex
tokens map[string]cachedToken
challenges map[string]bearerChallenge
}
type cachedToken struct {
value string
expiresAt time.Time
}
type bearerChallenge struct {
realm string
service string
scopes []string
}
type tokenResponse struct {
Token string `json:"token"`
AccessToken string `json:"access_token"`
ExpiresIn int64 `json:"expires_in"`
IssuedAt string `json:"issued_at"`
}
// NewTransport creates an authentication-aware transport around base.
func NewTransport(base http.RoundTripper, authForURL AuthFunc) *Transport {
if base == nil {
base = http.DefaultTransport
}
return &Transport{
base: base,
authForURL: authForURL,
tokens: make(map[string]cachedToken),
challenges: make(map[string]bearerChallenge),
}
}
// RoundTrip implements http.RoundTripper.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
hasExplicitAuthorization := req.Header.Get("Authorization") != ""
outbound := cloneRequest(req)
t.applyAuthentication(outbound, hasExplicitAuthorization)
resp, err := t.base.RoundTrip(outbound)
if err != nil || resp.StatusCode != http.StatusUnauthorized {
return resp, err
}
if hasExplicitAuthorization {
return resp, nil
}
if registryProtectionSpace(req.URL) == "" {
return resp, nil
}
challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate"))
if !ok || !canReplay(req) {
return resp, nil
}
drainAndClose(resp.Body)
token, err := t.token(req.Context(), challenge)
if err != nil {
return nil, fmt.Errorf("registry authentication: %w", err)
}
t.rememberChallenge(req.URL, challenge)
retry, err := cloneRequestForRetry(req)
if err != nil {
return nil, err
}
t.applyConfiguredAuthentication(retry)
retry.Header.Set("Authorization", "Bearer "+token)
return t.base.RoundTrip(retry)
}
func (t *Transport) applyAuthentication(req *http.Request, hasExplicitAuthorization bool) {
t.applyConfiguredAuthentication(req)
if hasExplicitAuthorization {
return
}
if token := t.cachedTokenForRequest(req.URL); token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
}
func (t *Transport) applyConfiguredAuthentication(req *http.Request) {
if t.authForURL == nil {
return
}
name, value := t.authForURL(req.URL.String())
if name != "" && value != "" && req.Header.Get(name) == "" {
req.Header.Set(name, value)
}
}
func (t *Transport) token(ctx context.Context, challenge bearerChallenge) (string, error) {
key := challenge.key()
if token := t.cachedToken(key); token != "" {
return token, nil
}
token, expiresAt, err := t.fetchToken(ctx, challenge)
if err != nil {
return "", err
}
t.cacheToken(key, cachedToken{value: token, expiresAt: expiresAt})
return token, nil
}
func (t *Transport) cacheToken(key string, token cachedToken) {
now := time.Now()
t.mu.Lock()
defer t.mu.Unlock()
for cachedKey, cached := range t.tokens {
if !now.Before(cached.expiresAt) {
delete(t.tokens, cachedKey)
}
}
t.tokens[key] = token
}
func (t *Transport) fetchToken(ctx context.Context, challenge bearerChallenge) (string, time.Time, error) {
tokenURL, err := url.Parse(challenge.realm)
if err != nil || !tokenURL.IsAbs() || (tokenURL.Scheme != "https" && tokenURL.Scheme != "http") {
return "", time.Time{}, fmt.Errorf("invalid token realm %q", challenge.realm)
}
query := tokenURL.Query()
if challenge.service != "" {
query.Set("service", challenge.service)
}
for _, scope := range challenge.scopes {
query.Add("scope", scope)
}
query.Set("client_id", "git-pkgs-proxy")
tokenURL.RawQuery = query.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL.String(), nil)
if err != nil {
return "", time.Time{}, err
}
client := &http.Client{Transport: configuredTransport{parent: t}}
resp, err := client.Do(req)
if err != nil {
return "", time.Time{}, fmt.Errorf("requesting token: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTokenResponseSize))
return "", time.Time{}, fmt.Errorf("token service returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
var payload tokenResponse
if err := json.NewDecoder(io.LimitReader(resp.Body, maxTokenResponseSize)).Decode(&payload); err != nil {
return "", time.Time{}, fmt.Errorf("decoding token response: %w", err)
}
token := payload.Token
if token == "" {
token = payload.AccessToken
}
if token == "" {
return "", time.Time{}, fmt.Errorf("token response did not contain a token")
}
issuedAt := time.Now()
if payload.IssuedAt != "" {
if parsed, parseErr := time.Parse(time.RFC3339, payload.IssuedAt); parseErr == nil {
issuedAt = parsed
}
}
lifetime := time.Duration(payload.ExpiresIn) * time.Second
if lifetime <= 0 {
lifetime = defaultTokenLifetime
}
expiresAt := issuedAt.Add(lifetime).Add(-expirySkew(lifetime))
return token, expiresAt, nil
}
type configuredTransport struct {
parent *Transport
}
func (t configuredTransport) RoundTrip(req *http.Request) (*http.Response, error) {
outbound := cloneRequest(req)
t.parent.applyConfiguredAuthentication(outbound)
return t.parent.base.RoundTrip(outbound)
}
func (t *Transport) cachedTokenForRequest(requestURL *url.URL) string {
space := registryProtectionSpace(requestURL)
if space == "" {
return ""
}
t.mu.Lock()
challenge, ok := t.challenges[space]
t.mu.Unlock()
if !ok {
return ""
}
return t.cachedToken(challenge.key())
}
func (t *Transport) cachedToken(key string) string {
now := time.Now()
t.mu.Lock()
defer t.mu.Unlock()
token, ok := t.tokens[key]
if !ok {
return ""
}
if !now.Before(token.expiresAt) {
delete(t.tokens, key)
return ""
}
return token.value
}
func (t *Transport) rememberChallenge(requestURL *url.URL, challenge bearerChallenge) {
space := registryProtectionSpace(requestURL)
if space == "" {
return
}
t.mu.Lock()
t.challenges[space] = challenge
t.mu.Unlock()
}
func (c bearerChallenge) key() string {
return c.realm + "\x00" + c.service + "\x00" + strings.Join(c.scopes, "\x00")
}
func registryProtectionSpace(u *url.URL) string {
const registryPrefix = "/v2/"
if u == nil || !strings.HasPrefix(u.Path, registryPrefix) {
return ""
}
rest := strings.TrimPrefix(u.Path, registryPrefix)
end := len(rest)
for _, marker := range []string{"/blobs/", "/manifests/", "/tags/", "/referrers/"} {
if index := strings.Index(rest, marker); index >= 0 && index < end {
end = index
}
}
if end == len(rest) || end == 0 {
return ""
}
return u.Scheme + "://" + u.Host + registryPrefix + rest[:end]
}
func parseBearerChallenge(values []string) (bearerChallenge, bool) {
for _, value := range values {
params, ok := bearerParameters(value)
if !ok || params["realm"] == "" {
continue
}
challenge := bearerChallenge{
realm: params["realm"],
service: params["service"],
}
if scope := params["scope"]; scope != "" {
challenge.scopes = append(challenge.scopes, scope)
}
return challenge, true
}
return bearerChallenge{}, false
}
func bearerParameters(value string) (map[string]string, bool) {
start := findAuthScheme(value, "Bearer")
if start < 0 {
return nil, false
}
rest := value[start+len("Bearer"):]
params := make(map[string]string)
for {
rest = strings.TrimLeft(rest, " \t,")
if rest == "" {
break
}
keyEnd := strings.IndexAny(rest, "= \t,")
if keyEnd <= 0 {
break
}
key := strings.ToLower(rest[:keyEnd])
rest = strings.TrimLeft(rest[keyEnd:], " \t")
if rest == "" || rest[0] != '=' {
break
}
rest = strings.TrimLeft(rest[1:], " \t")
parsed, remaining, ok := parseAuthValue(rest)
if !ok {
return nil, false
}
params[key] = parsed
rest = remaining
}
return params, true
}
func findAuthScheme(value, scheme string) int {
inQuote := false
escaped := false
for index := 0; index+len(scheme) <= len(value); index++ {
char := value[index]
if escaped {
escaped = false
continue
}
if char == '\\' && inQuote {
escaped = true
continue
}
if char == '"' {
inQuote = !inQuote
continue
}
if inQuote || !strings.EqualFold(value[index:index+len(scheme)], scheme) {
continue
}
beforeOK := index == 0 || value[index-1] == ',' || value[index-1] == ' ' || value[index-1] == '\t'
after := index + len(scheme)
afterOK := after < len(value) && (value[after] == ' ' || value[after] == '\t')
if beforeOK && afterOK {
return index
}
}
return -1
}
func parseAuthValue(value string) (parsed, remaining string, ok bool) {
if value == "" {
return "", "", false
}
if value[0] != '"' {
end := strings.IndexAny(value, " \t,")
if end < 0 {
return value, "", true
}
return value[:end], value[end:], end > 0
}
var builder strings.Builder
escaped := false
for index := 1; index < len(value); index++ {
char := value[index]
if escaped {
builder.WriteByte(char)
escaped = false
continue
}
if char == '\\' {
escaped = true
continue
}
if char == '"' {
return builder.String(), value[index+1:], true
}
builder.WriteByte(char)
}
return "", "", false
}
func cloneRequest(req *http.Request) *http.Request {
clone := req.Clone(req.Context())
clone.Header = req.Header.Clone()
return clone
}
func canReplay(req *http.Request) bool {
return req.Body == nil || req.GetBody != nil
}
func cloneRequestForRetry(req *http.Request) (*http.Request, error) {
clone := cloneRequest(req)
if req.Body == nil {
return clone, nil
}
body, err := req.GetBody()
if err != nil {
return nil, fmt.Errorf("replaying authenticated request: %w", err)
}
clone.Body = body
return clone, nil
}
func expirySkew(lifetime time.Duration) time.Duration {
if lifetime < tokenExpirySkew*2 {
return lifetime / shortTokenSkewDivisor
}
return tokenExpirySkew
}
func drainAndClose(body io.ReadCloser) {
_, _ = io.Copy(io.Discard, io.LimitReader(body, maxTokenResponseSize))
_ = body.Close()
}

View file

@ -1,251 +0,0 @@
package httpclient
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestTransportFollowsBearerChallengeAndCachesToken(t *testing.T) {
var registryRequests int
var tokenRequests int
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/token":
tokenRequests++
if got := r.URL.Query().Get("service"); got != "registry.test" {
t.Errorf("service = %q, want %q", got, "registry.test")
}
if got := r.URL.Query().Get("scope"); got != "repository:library/test:pull" {
t.Errorf("scope = %q, want %q", got, "repository:library/test:pull")
}
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"token":"registry-token","expires_in":3600}`)
case "/v2/library/test/blobs/sha256:first", "/v2/library/test/blobs/sha256:second":
registryRequests++
if r.Header.Get("Authorization") != "Bearer registry-token" {
w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token",service="registry.test",scope="repository:library/test:pull"`)
http.Error(w, "authentication required", http.StatusUnauthorized)
return
}
_, _ = io.WriteString(w, "blob")
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
for _, digest := range []string{"sha256:first", "sha256:second"} {
resp, err := client.Get(server.URL + "/v2/library/test/blobs/" + digest)
if err != nil {
t.Fatalf("GET %s: %v", digest, err)
}
body, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if readErr != nil {
t.Fatalf("read %s response: %v", digest, readErr)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("GET %s status = %d, want %d", digest, resp.StatusCode, http.StatusOK)
}
if string(body) != "blob" {
t.Errorf("GET %s body = %q, want %q", digest, body, "blob")
}
}
if tokenRequests != 1 {
t.Errorf("token requests = %d, want 1", tokenRequests)
}
if registryRequests != 3 {
t.Errorf("registry requests = %d, want 3", registryRequests)
}
}
func TestTransportAddsConfiguredAuthentication(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("X-Registry-Token"); got != "configured-token" {
t.Errorf("X-Registry-Token = %q, want %q", got, "configured-token")
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
authForURL := func(url string) (string, string) {
if strings.HasPrefix(url, server.URL) {
return "X-Registry-Token", "configured-token"
}
return "", ""
}
client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)}
resp, err := client.Get(server.URL + "/metadata")
if err != nil {
t.Fatalf("GET metadata: %v", err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent)
}
}
func TestTransportPreservesExplicitAuthentication(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" {
t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token")
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
authForURL := func(string) (string, string) {
return "Authorization", "Bearer configured-token"
}
client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)}
req, err := http.NewRequest(http.MethodGet, server.URL+"/artifact", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", "Bearer explicit-token")
resp, err := client.Do(req)
if err != nil {
t.Fatalf("GET artifact: %v", err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent)
}
}
func TestTransportDoesNotReplaceExplicitAuthenticationAfterBearerChallenge(t *testing.T) {
var registryRequests int
var tokenRequests int
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/token":
tokenRequests++
_, _ = io.WriteString(w, `{"token":"registry-token"}`)
case "/v2/library/test/blobs/sha256:test":
registryRequests++
if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" {
t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token")
}
w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`)
http.Error(w, "authentication required", http.StatusUnauthorized)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
req, err := http.NewRequest(http.MethodGet, server.URL+"/v2/library/test/blobs/sha256:test", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", "Bearer explicit-token")
resp, err := client.Do(req)
if err != nil {
t.Fatalf("GET blob: %v", err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusUnauthorized {
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized)
}
if registryRequests != 1 {
t.Errorf("registry requests = %d, want 1", registryRequests)
}
if tokenRequests != 0 {
t.Errorf("token requests = %d, want 0", tokenRequests)
}
}
func TestTransportDoesNotForwardConfiguredAuthenticationOnTokenRedirect(t *testing.T) {
destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("X-Registry-Token"); got != "" {
t.Errorf("redirected X-Registry-Token = %q, want empty", got)
}
_, _ = io.WriteString(w, `{"token":"registry-token"}`)
}))
defer destination.Close()
source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("X-Registry-Token"); got != "configured-token" {
t.Errorf("source X-Registry-Token = %q, want %q", got, "configured-token")
}
http.Redirect(w, r, destination.URL+"/token", http.StatusFound)
}))
defer source.Close()
authForURL := func(rawURL string) (string, string) {
if strings.HasPrefix(rawURL, source.URL) {
return "X-Registry-Token", "configured-token"
}
return "", ""
}
transport := NewTransport(http.DefaultTransport, authForURL)
token, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: source.URL + "/token"})
if err != nil {
t.Fatalf("fetchToken: %v", err)
}
if token != "registry-token" {
t.Errorf("token = %q, want %q", token, "registry-token")
}
}
func TestTransportPrunesExpiredTokens(t *testing.T) {
transport := NewTransport(http.DefaultTransport, nil)
transport.tokens["expired-unused"] = cachedToken{
value: "expired-token",
expiresAt: time.Now().Add(-time.Minute),
}
transport.cacheToken("current", cachedToken{
value: "current-token",
expiresAt: time.Now().Add(time.Minute),
})
if got := transport.cachedToken("current"); got != "current-token" {
t.Errorf("cachedToken(current) = %q, want %q", got, "current-token")
}
if _, ok := transport.tokens["expired-unused"]; ok {
t.Error("expired unused token was not pruned")
}
}
func TestTransportDoesNotFollowBearerChallengeOutsideOCIRegistry(t *testing.T) {
tokenRequests := 0
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/token" {
tokenRequests++
_, _ = io.WriteString(w, `{"token":"unexpected"}`)
return
}
w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`)
http.Error(w, "authentication required", http.StatusUnauthorized)
}))
defer server.Close()
client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
resp, err := client.Get(server.URL + "/api/packages")
if err != nil {
t.Fatalf("GET API: %v", err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusUnauthorized {
t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized)
}
if tokenRequests != 0 {
t.Errorf("token requests = %d, want 0", tokenRequests)
}
}

View file

@ -6,7 +6,6 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/git-pkgs/purl"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
) )
@ -129,14 +128,6 @@ var (
}, },
[]string{"ecosystem"}, []string{"ecosystem"},
) )
HealthProbeFailures = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "proxy_health_probe_failures_total",
Help: "Total number of storage health probe failures, by step (write|size|read|verify|delete).",
},
[]string{"step"},
)
) )
func init() { func init() {
@ -156,7 +147,6 @@ func init() {
StorageErrors, StorageErrors,
ActiveRequests, ActiveRequests,
IntegrityFailures, IntegrityFailures,
HealthProbeFailures,
) )
} }
@ -174,12 +164,12 @@ func RecordRequest(ecosystem string, status int, duration time.Duration) {
// RecordCacheHit increments cache hit counter. // RecordCacheHit increments cache hit counter.
func RecordCacheHit(ecosystem string) { func RecordCacheHit(ecosystem string) {
CacheHits.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() CacheHits.WithLabelValues(ecosystem).Inc()
} }
// RecordCacheMiss increments cache miss counter. // RecordCacheMiss increments cache miss counter.
func RecordCacheMiss(ecosystem string) { func RecordCacheMiss(ecosystem string) {
CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() CacheMisses.WithLabelValues(ecosystem).Inc()
} }
// RecordUpstreamFetch tracks upstream fetch duration. // RecordUpstreamFetch tracks upstream fetch duration.
@ -202,12 +192,6 @@ func RecordIntegrityFailure(ecosystem string) {
IntegrityFailures.WithLabelValues(ecosystem).Inc() IntegrityFailures.WithLabelValues(ecosystem).Inc()
} }
// RecordHealthProbeFailure increments the health probe failure counter.
// step is one of: "write", "size", "read", "verify", "delete".
func RecordHealthProbeFailure(step string) {
HealthProbeFailures.WithLabelValues(step).Inc()
}
// RecordStorageError increments storage error counter. // RecordStorageError increments storage error counter.
func RecordStorageError(operation string) { func RecordStorageError(operation string) {
StorageErrors.WithLabelValues(operation).Inc() StorageErrors.WithLabelValues(operation).Inc()

View file

@ -6,7 +6,6 @@ import (
"time" "time"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
) )
@ -192,45 +191,22 @@ func TestMetricsEndpointOutput(t *testing.T) {
func TestMetricsLabeling(t *testing.T) { func TestMetricsLabeling(t *testing.T) {
// Test that different ecosystems are properly labeled // Test that different ecosystems are properly labeled
ecosystems := []struct { ecosystems := []string{"npm", "pypi", "cargo", "gem"}
input string
label string
}{
{input: "npm", label: "npm"},
{input: "pypi", label: "pypi"},
{input: "cargo", label: "cargo"},
{input: "gem", label: "rubygems"},
}
for _, eco := range ecosystems { for _, eco := range ecosystems {
RecordRequest(eco.input, 200, 10*time.Millisecond) RecordRequest(eco, 200, 10*time.Millisecond)
RecordCacheHit(eco.input) RecordCacheHit(eco)
} }
// Verify each ecosystem has metrics // Verify each ecosystem has metrics
for _, eco := range ecosystems { for _, eco := range ecosystems {
val := getMetricValue(t, CacheHits, eco.label) val := getMetricValue(t, CacheHits, eco)
if val == 0 { if val == 0 {
t.Errorf("no cache hits recorded for %s", eco.label) t.Errorf("no cache hits recorded for %s", eco)
} }
} }
} }
func TestCacheMetricLabelsAreNormalized(t *testing.T) {
rubyHitsBefore := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems"))
composerMissesBefore := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist"))
RecordCacheHit("gem")
RecordCacheMiss("composer")
if diff := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems")) - rubyHitsBefore; diff != 1 {
t.Errorf("rubygems cache hits delta = %.0f, want 1", diff)
}
if diff := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist")) - composerMissesBefore; diff != 1 {
t.Errorf("packagist cache misses delta = %.0f, want 1", diff)
}
}
func TestMetricNames(t *testing.T) { func TestMetricNames(t *testing.T) {
// Verify metric names follow Prometheus naming conventions // Verify metric names follow Prometheus naming conventions
expectedMetrics := []string{ expectedMetrics := []string{

View file

@ -0,0 +1,16 @@
package mirror
import (
"context"
"fmt"
)
// RegistrySource enumerates all packages in a registry for full mirroring.
// Registry enumeration is not yet implemented for any ecosystem.
type RegistrySource struct {
Ecosystem string
}
func (s *RegistrySource) Enumerate(_ context.Context, _ func(PackageVersion) error) error {
return fmt.Errorf("registry enumeration is not yet implemented for ecosystem %q", s.Ecosystem)
}

View file

@ -0,0 +1,46 @@
package mirror
import (
"context"
"testing"
)
func TestRegistrySourceUnsupported(t *testing.T) {
source := &RegistrySource{Ecosystem: "golang"}
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
return nil
})
if err == nil {
t.Fatal("expected error for unsupported ecosystem")
}
}
func TestRegistrySourceNPMNotImplemented(t *testing.T) {
source := &RegistrySource{Ecosystem: "npm"}
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
return nil
})
if err == nil {
t.Fatal("expected not-implemented error")
}
}
func TestRegistrySourcePyPINotImplemented(t *testing.T) {
source := &RegistrySource{Ecosystem: "pypi"}
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
return nil
})
if err == nil {
t.Fatal("expected not-implemented error")
}
}
func TestRegistrySourceCargoNotImplemented(t *testing.T) {
source := &RegistrySource{Ecosystem: "cargo"}
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
return nil
})
if err == nil {
t.Fatal("expected not-implemented error")
}
}

View file

@ -139,11 +139,12 @@ type BulkResponse struct {
// Resolves namespaced package names (Composer vendor/name, npm @scope/name) from the path. // Resolves namespaced package names (Composer vendor/name, npm @scope/name) from the path.
func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem") ecosystem := chi.URLParam(r, "ecosystem")
segments, err := packagePathSegments(r) wildcard := chi.URLParam(r, "*")
if err != nil { if err := validatePackagePath(wildcard); err != nil {
badRequest(w, err.Error()) badRequest(w, err.Error())
return return
} }
segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) == 0 { if ecosystem == "" || len(segments) == 0 {
badRequest(w, "ecosystem and name are required") badRequest(w, "ecosystem and name are required")
@ -276,11 +277,12 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste
// Supports both {name} and {name}/{version} paths with namespaced package names. // Supports both {name} and {name}/{version} paths with namespaced package names.
func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem") ecosystem := chi.URLParam(r, "ecosystem")
segments, err := packagePathSegments(r) wildcard := chi.URLParam(r, "*")
if err != nil { if err := validatePackagePath(wildcard); err != nil {
badRequest(w, err.Error()) badRequest(w, err.Error())
return return
} }
segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) == 0 { if ecosystem == "" || len(segments) == 0 {
badRequest(w, "ecosystem and name are required") badRequest(w, "ecosystem and name are required")

View file

@ -1,7 +1,6 @@
package server package server
import ( import (
"bufio"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
@ -11,46 +10,27 @@ import (
"github.com/git-pkgs/archives" "github.com/git-pkgs/archives"
"github.com/git-pkgs/archives/diff" "github.com/git-pkgs/archives/diff"
"github.com/git-pkgs/magic"
"github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/proxy/internal/handler"
"github.com/git-pkgs/purl" "github.com/git-pkgs/purl"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
) )
const ( const contentTypePlainText = "text/plain; charset=utf-8"
contentTypePlainText = "text/plain; charset=utf-8"
browseSniffSize = 512
)
// maxBrowseArchiveSize caps how much data openArchive will buffer for // maxBrowseArchiveSize caps how much data openArchive will buffer for
// prefix detection. Artifacts larger than this are rejected to prevent // prefix detection. Artifacts larger than this are rejected to prevent
// memory exhaustion from a single request. // memory exhaustion from a single request.
const maxBrowseArchiveSize = 512 << 20 // 512 MB const maxBrowseArchiveSize = 512 << 20 // 512 MB
// firstBrowsableArtifact returns the first cached artifact that can be opened as // archiveFilename returns a filename suitable for archive format detection.
// an archive, or nil if the version has none. // Some ecosystems (e.g. composer) store artifacts with bare hash filenames
// // that have no extension. This adds .zip when the original has no extension
// A version's artifact list is not all archives: a PEP 658 core-metadata sidecar // and the content is likely a zip archive.
// resolves to the same name and version as the distribution it describes, so it func archiveFilename(filename string) string {
// is cached under that version too. Sidecars are plain text, and because '-' if path.Ext(filename) == "" {
// sorts before '.' one can even precede the real distribution in the return filename + ".zip"
// filename-ordered list, so selecting blindly would hand openArchive a file it
// cannot parse.
func firstBrowsableArtifact(artifacts []database.Artifact) *database.Artifact {
for i := range artifacts {
if artifacts[i].StoragePath.Valid && !isMetadataSidecar(artifacts[i].Filename) {
return &artifacts[i]
}
} }
return filename
return nil
}
// isMetadataSidecar reports whether filename is a core-metadata sidecar rather
// than a distribution archive.
func isMetadataSidecar(filename string) bool {
return strings.HasSuffix(filename, handler.PyPIMetadataSuffix)
} }
// detectSingleRootDir returns the single top-level directory name if all files // detectSingleRootDir returns the single top-level directory name if all files
@ -86,6 +66,8 @@ func detectSingleRootDir(reader archives.Reader) string {
// and stripping a single top-level directory prefix (like GitHub zipballs). // and stripping a single top-level directory prefix (like GitHub zipballs).
// For npm, the hardcoded "package/" prefix takes precedence. // For npm, the hardcoded "package/" prefix takes precedence.
func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations
fname := archiveFilename(filename)
limited := io.LimitReader(content, maxBrowseArchiveSize+1) limited := io.LimitReader(content, maxBrowseArchiveSize+1)
data, err := io.ReadAll(limited) data, err := io.ReadAll(limited)
if err != nil { if err != nil {
@ -96,17 +78,17 @@ func openArchive(filename string, content io.Reader, ecosystem string) (archives
} }
if ecosystem == "npm" { if ecosystem == "npm" {
return archives.OpenBytesWithPrefix(filename, data, "package/") return archives.OpenBytesWithPrefix(fname, data, "package/")
} }
probe, err := archives.OpenBytes(filename, data) probe, err := archives.OpenBytes(fname, data)
if err != nil { if err != nil {
return nil, err return nil, err
} }
prefix := detectSingleRootDir(probe) prefix := detectSingleRootDir(probe)
_ = probe.Close() _ = probe.Close()
return archives.OpenBytesWithPrefix(filename, data, prefix) return archives.OpenBytesWithPrefix(fname, data, prefix)
} }
// BrowseListResponse contains the file listing for a directory in an archives. // BrowseListResponse contains the file listing for a directory in an archives.
@ -137,7 +119,7 @@ type BrowseFileInfo struct {
// @Success 200 {object} BrowseListResponse // @Success 200 {object} BrowseListResponse
// @Failure 404 {object} ErrorResponse // @Failure 404 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse // @Failure 500 {object} ErrorResponse
// @Router /ui/api/browse/{ecosystem}/{name}/{version} [get] // @Router /api/browse/{ecosystem}/{name}/{version} [get]
// handleBrowsePath dispatches /api/browse/{ecosystem}/* to the appropriate browse handler. // handleBrowsePath dispatches /api/browse/{ecosystem}/* to the appropriate browse handler.
// It resolves namespaced package names by consulting the database. // It resolves namespaced package names by consulting the database.
// //
@ -147,11 +129,12 @@ type BrowseFileInfo struct {
// {name}/{version}/file/{path} -> browse file // {name}/{version}/file/{path} -> browse file
func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) { func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem") ecosystem := chi.URLParam(r, "ecosystem")
segments, err := packagePathSegments(r) wildcard := chi.URLParam(r, "*")
if err != nil { if err := validatePackagePath(wildcard); err != nil {
badRequest(w, err.Error()) badRequest(w, err.Error())
return return
} }
segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) < 2 { if ecosystem == "" || len(segments) < 2 {
badRequest(w, "ecosystem, name, and version required") badRequest(w, "ecosystem, name, and version required")
@ -202,11 +185,12 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
// Supported paths: {name}/{fromVersion}/{toVersion} // Supported paths: {name}/{fromVersion}/{toVersion}
func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) { func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem") ecosystem := chi.URLParam(r, "ecosystem")
segments, err := packagePathSegments(r) wildcard := chi.URLParam(r, "*")
if err != nil { if err := validatePackagePath(wildcard); err != nil {
badRequest(w, err.Error()) badRequest(w, err.Error())
return return
} }
segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) < 3 { if ecosystem == "" || len(segments) < 3 {
badRequest(w, "ecosystem, name, fromVersion, and toVersion required") badRequest(w, "ecosystem, name, fromVersion, and toVersion required")
@ -238,7 +222,14 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
return return
} }
cachedArtifact := firstBrowsableArtifact(artifacts) // Find the first cached artifact
var cachedArtifact *database.Artifact
for i := range artifacts {
if artifacts[i].StoragePath.Valid {
cachedArtifact = &artifacts[i]
break
}
}
if cachedArtifact == nil { if cachedArtifact == nil {
notFound(w, "artifact not cached") notFound(w, "artifact not cached")
@ -305,7 +296,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
// @Failure 400 {object} ErrorResponse // @Failure 400 {object} ErrorResponse
// @Failure 404 {object} ErrorResponse // @Failure 404 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse // @Failure 500 {object} ErrorResponse
// @Router /ui/api/browse/{ecosystem}/{name}/{version}/file/{filepath} [get] // @Router /api/browse/{ecosystem}/{name}/{version}/file/{filepath} [get]
func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, name, version, filePath string) { func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, name, version, filePath string) {
if filePath == "" { if filePath == "" {
badRequest(w, "file path required") badRequest(w, "file path required")
@ -325,7 +316,14 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
return return
} }
cachedArtifact := firstBrowsableArtifact(artifacts) // Find the first cached artifact
var cachedArtifact *database.Artifact
for i := range artifacts {
if artifacts[i].StoragePath.Valid {
cachedArtifact = &artifacts[i]
break
}
}
if cachedArtifact == nil { if cachedArtifact == nil {
notFound(w, "artifact not cached") notFound(w, "artifact not cached")
@ -363,14 +361,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
} }
defer func() { _ = fileReader.Close() }() defer func() { _ = fileReader.Close() }()
contentType, knownPath := detectContentTypeFromPath(filePath) contentType := detectContentType(filePath)
var content io.Reader = fileReader
if !knownPath {
bufferedFile := bufio.NewReaderSize(fileReader, browseSniffSize)
prefix, _ := bufferedFile.Peek(browseSniffSize)
contentType = detectContentTypeFromPrefix(prefix)
content = bufferedFile
}
w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Security-Policy", "sandbox") w.Header().Set("Content-Security-Policy", "sandbox")
w.Header().Set("X-Content-Type-Options", "nosniff") w.Header().Set("X-Content-Type-Options", "nosniff")
@ -379,103 +370,85 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename)) w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename))
// Stream the file // Stream the file
_, _ = io.Copy(w, content) _, _ = io.Copy(w, fileReader)
} }
func detectContentTypeFromPath(filename string) (string, bool) { // detectContentType returns an appropriate content type based on file extension.
func detectContentType(filename string) string {
ext := strings.ToLower(path.Ext(filename)) ext := strings.ToLower(path.Ext(filename))
switch ext { switch ext {
// Text formats // Text formats
case ".txt", ".md", ".markdown": case ".txt", ".md", ".markdown":
return contentTypePlainText, true return contentTypePlainText
case ".html", ".htm", ".xhtml": case ".html", ".htm", ".xhtml":
return contentTypePlainText, true return contentTypePlainText
case ".css": case ".css":
return "text/css; charset=utf-8", true return "text/css; charset=utf-8"
case ".js", ".mjs": case ".js", ".mjs":
return "application/javascript; charset=utf-8", true return "application/javascript; charset=utf-8"
case ".json": case ".json":
return "application/json; charset=utf-8", true return "application/json; charset=utf-8"
case ".xml": case ".xml":
return "application/xml; charset=utf-8", true return "application/xml; charset=utf-8"
case ".yaml", ".yml": case ".yaml", ".yml":
return "text/yaml; charset=utf-8", true return "text/yaml; charset=utf-8"
case ".toml": case ".toml":
return "text/toml; charset=utf-8", true return "text/toml; charset=utf-8"
// Programming languages // Programming languages
case ".go": case ".go":
return "text/x-go; charset=utf-8", true return "text/x-go; charset=utf-8"
case ".rs": case ".rs":
return "text/x-rust; charset=utf-8", true return "text/x-rust; charset=utf-8"
case ".py": case ".py":
return "text/x-python; charset=utf-8", true return "text/x-python; charset=utf-8"
case ".rb": case ".rb":
return "text/x-ruby; charset=utf-8", true return "text/x-ruby; charset=utf-8"
case ".java": case ".java":
return "text/x-java; charset=utf-8", true return "text/x-java; charset=utf-8"
case ".c", ".h": case ".c", ".h":
return "text/x-c; charset=utf-8", true return "text/x-c; charset=utf-8"
case ".cpp", ".cc", ".cxx", ".hpp": case ".cpp", ".cc", ".cxx", ".hpp":
return "text/x-c++; charset=utf-8", true return "text/x-c++; charset=utf-8"
case ".ts": case ".ts":
return "text/typescript; charset=utf-8", true return "text/typescript; charset=utf-8"
case ".tsx": case ".tsx":
return "text/tsx; charset=utf-8", true return "text/tsx; charset=utf-8"
case ".jsx": case ".jsx":
return "text/jsx; charset=utf-8", true return "text/jsx; charset=utf-8"
case ".php": case ".php":
return "text/x-php; charset=utf-8", true return "text/x-php; charset=utf-8"
// Config files // Config files
case ".conf", ".config", ".ini": case ".conf", ".config", ".ini":
return contentTypePlainText, true return contentTypePlainText
case ".sh", ".bash": case ".sh", ".bash":
return "text/x-shellscript; charset=utf-8", true return "text/x-shellscript; charset=utf-8"
case ".dockerfile": case ".dockerfile":
return "text/x-dockerfile; charset=utf-8", true return "text/x-dockerfile; charset=utf-8"
// Images // Images
case ".png": case ".png":
return "image/png", true return "image/png"
case ".jpg", ".jpeg": case ".jpg", ".jpeg":
return "image/jpeg", true return "image/jpeg"
case ".gif": case ".gif":
return "image/gif", true return "image/gif"
case ".svg": case ".svg":
return contentTypePlainText, true return contentTypePlainText
case ".ico": case ".ico":
return "image/x-icon", true return "image/x-icon"
// Archives // Archives
case ".zip", ".tar", ".gz", ".bz2", ".xz": case ".zip", ".tar", ".gz", ".bz2", ".xz":
return "application/octet-stream", true return "application/octet-stream"
default: default:
// Try to detect if it looks like text
if isLikelyText(filename) { if isLikelyText(filename) {
return contentTypePlainText, true return contentTypePlainText
} }
return "", false
}
}
func detectContentTypeFromPrefix(prefix []byte) string {
result := magic.DetectPrefix(prefix)
if result.Kind == magic.KindText {
return contentTypePlainText
}
switch result.Format {
case "png":
return "image/png"
case "jpeg":
return "image/jpeg"
case "gif":
return "image/gif"
case "pdf":
return "application/pdf"
default:
return "application/octet-stream" return "application/octet-stream"
} }
} }
@ -504,16 +477,10 @@ func isLikelyText(filename string) bool {
} }
// BrowseSourceData contains data for the browse source page. // BrowseSourceData contains data for the browse source page.
//
// Version is the decoded version, for display. EscapedVersion is the same value
// escaped as a single URL path segment and is what the links and the browse API
// calls must use; see database.Version.EscapedVersion.
type BrowseSourceData struct { type BrowseSourceData struct {
Layout Ecosystem string
Ecosystem string PackageName string
PackageName string Version string
Version string
EscapedVersion string
} }
// handleBrowseSource is now showBrowseSource in server.go, dispatched via handlePackagePath. // handleBrowseSource is now showBrowseSource in server.go, dispatched via handlePackagePath.
@ -531,7 +498,7 @@ type BrowseSourceData struct {
// @Success 200 {object} map[string]any // @Success 200 {object} map[string]any
// @Failure 404 {object} ErrorResponse // @Failure 404 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse // @Failure 500 {object} ErrorResponse
// @Router /ui/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get] // @Router /api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get]
func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, name, fromVersion, toVersion string) { func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, name, fromVersion, toVersion string) {
// Get artifacts for both versions // Get artifacts for both versions
fromPURL := purl.MakePURLString(ecosystem, name, fromVersion) fromPURL := purl.MakePURLString(ecosystem, name, fromVersion)
@ -550,8 +517,19 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
} }
// Find cached artifacts // Find cached artifacts
fromArtifact := firstBrowsableArtifact(fromArtifacts) var fromArtifact, toArtifact *database.Artifact
toArtifact := firstBrowsableArtifact(toArtifacts) for i := range fromArtifacts {
if fromArtifacts[i].StoragePath.Valid {
fromArtifact = &fromArtifacts[i]
break
}
}
for i := range toArtifacts {
if toArtifacts[i].StoragePath.Valid {
toArtifact = &toArtifacts[i]
break
}
}
if fromArtifact == nil || toArtifact == nil { if fromArtifact == nil || toArtifact == nil {
notFound(w, "one or both versions not cached") notFound(w, "one or both versions not cached")
@ -604,17 +582,11 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
} }
// ComparePageData contains data for the version comparison page. // ComparePageData contains data for the version comparison page.
//
// FromVersion and ToVersion are decoded, for display; the Escaped variants are
// the path-segment form used to build the compare API URL.
type ComparePageData struct { type ComparePageData struct {
Layout Ecosystem string
Ecosystem string PackageName string
PackageName string FromVersion string
FromVersion string ToVersion string
ToVersion string
EscapedFromVersion string
EscapedToVersion string
} }
// handleComparePage is now showComparePage in server.go, dispatched via handlePackagePath. // handleComparePage is now showComparePage in server.go, dispatched via handlePackagePath.

View file

@ -55,35 +55,3 @@ func BenchmarkOpenArchive(b *testing.B) {
}) })
} }
} }
func BenchmarkDetectContentType(b *testing.B) {
cases := []struct {
name string
filename string
prefix []byte
knownPath bool
}{
{"known-path", "README.md", nil, true},
{"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize), false},
{"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...), false},
}
for _, tc := range cases {
b.Run(tc.name, func(b *testing.B) {
b.ReportAllocs()
var contentType string
if tc.knownPath {
for b.Loop() {
contentType, _ = detectContentTypeFromPath(tc.filename)
}
} else {
for b.Loop() {
contentType = detectContentTypeFromPrefix(tc.prefix)
}
}
if contentType == "" {
b.Fatal("empty content type")
}
})
}
}

View file

@ -65,7 +65,7 @@ func TestHandleBrowseList(t *testing.T) {
} }
// Test listing root directory // Test listing root directory
req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0", nil) req := httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -83,7 +83,7 @@ func TestHandleBrowseList(t *testing.T) {
} }
// Test listing subdirectory // Test listing subdirectory
req = httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0?path=lib", nil) req = httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0?path=lib", nil)
w = httptest.NewRecorder() w = httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -137,44 +137,29 @@ func TestHandleBrowseFile(t *testing.T) {
t.Fatalf("failed to upsert artifact: %v", err) t.Fatalf("failed to upsert artifact: %v", err)
} }
files := []struct { // Test fetching a file
path string req := httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0/file/README.md", nil)
content string w := httptest.NewRecorder()
contentType string ts.handler.ServeHTTP(w, req)
}{
{"README.md", "# Test Package\n", contentTypePlainText},
{"notes.data", "short text\n", contentTypePlainText},
{"logo", "\x89PNG\r\n\x1a\nimage data", "image/png"},
{"page", "<!DOCTYPE html><html></html>", contentTypePlainText},
{"misleading.txt", "\x89PNG\r\n\x1a\nimage data", contentTypePlainText},
}
for _, file := range files {
t.Run(file.path, func(t *testing.T) {
req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/"+file.path, nil)
w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusOK { if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String())
} }
if w.Body.String() != file.content {
t.Errorf("unexpected file content: %q", w.Body.String()) body := w.Body.String()
} if body != "# Test Package\n" {
if got := w.Header().Get("Content-Type"); got != file.contentType { t.Errorf("unexpected file content: %q", body)
t.Errorf("Content-Type = %q, want %q", got, file.contentType) }
}
if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" { // Check content type
t.Errorf("Content-Security-Policy = %q, want sandbox", got) contentType := w.Header().Get("Content-Type")
} if contentType != contentTypePlainText {
if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" { t.Errorf("expected text/plain content type, got %q", contentType)
t.Errorf("X-Content-Type-Options = %q, want nosniff", got)
}
})
} }
// Test fetching non-existent file // Test fetching non-existent file
req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil) req = httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil)
w := httptest.NewRecorder() w = httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusNotFound { if w.Code != http.StatusNotFound {
@ -182,52 +167,36 @@ func TestHandleBrowseFile(t *testing.T) {
} }
} }
func TestBrowseContentTypePolicy(t *testing.T) { func TestDetectContentType(t *testing.T) {
tests := []struct { tests := []struct {
name string
filename string filename string
prefix []byte
expectedCT string expectedCT string
}{ }{
{"text extension", "file.txt", nil, contentTypePlainText}, {"file.txt", contentTypePlainText},
{"markdown extension", "file.md", nil, contentTypePlainText}, {"file.md", contentTypePlainText},
{"JSON extension", "file.json", nil, "application/json; charset=utf-8"}, {"file.json", "application/json; charset=utf-8"},
{"JavaScript extension", "file.js", nil, "application/javascript; charset=utf-8"}, {"file.js", "application/javascript; charset=utf-8"},
{"Go extension", "file.go", nil, "text/x-go; charset=utf-8"}, {"file.go", "text/x-go; charset=utf-8"},
{"Python extension", "file.py", nil, "text/x-python; charset=utf-8"}, {"file.py", "text/x-python; charset=utf-8"},
{"Rust extension", "file.rs", nil, "text/x-rust; charset=utf-8"}, {"file.rs", "text/x-rust; charset=utf-8"},
{"HTML extension", "file.html", nil, contentTypePlainText}, {"file.html", contentTypePlainText},
{"HTM extension", "file.htm", nil, contentTypePlainText}, {"file.htm", contentTypePlainText},
{"XHTML extension", "file.xhtml", nil, contentTypePlainText}, {"file.xhtml", contentTypePlainText},
{"SVG extension", "file.svg", nil, contentTypePlainText}, {"file.svg", contentTypePlainText},
{"PNG extension", "file.png", nil, "image/png"}, {"file.png", "image/png"},
{"JPEG extension", "file.jpg", nil, "image/jpeg"}, {"file.jpg", "image/jpeg"},
{"README", "README", nil, contentTypePlainText}, {"README", contentTypePlainText},
{"LICENSE", "LICENSE", nil, contentTypePlainText}, {"LICENSE", contentTypePlainText},
{"Makefile", "Makefile", nil, contentTypePlainText}, {"Makefile", contentTypePlainText},
{"gitignore", ".gitignore", nil, contentTypePlainText}, {".gitignore", contentTypePlainText},
{"unknown empty", "file.bin", nil, "application/octet-stream"}, {"file.bin", "application/octet-stream"},
{"extensionless PNG", "asset", []byte("\x89PNG\r\n\x1a\n"), "image/png"},
{"extensionless JPEG", "asset", []byte("\xff\xd8\xff"), "image/jpeg"},
{"extensionless GIF", "asset", []byte("GIF89a"), "image/gif"},
{"extensionless PDF", "asset", []byte("%PDF-1.7"), "application/pdf"},
{"extensionless text", "asset", []byte("plain text\n"), contentTypePlainText},
{"extensionless HTML", "asset", []byte("<!DOCTYPE html><html></html>"), contentTypePlainText},
{"extensionless XML", "asset", []byte("<?xml version=\"1.0\"?><root/>"), contentTypePlainText},
{"extensionless SVG", "asset", []byte("<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>"), contentTypePlainText},
{"extensionless ZIP", "asset", []byte("PK\x03\x04"), "application/octet-stream"},
{"extensionless binary", "asset", []byte{0, 1, 2}, "application/octet-stream"},
{"known path wins", "file.txt", []byte("\x89PNG\r\n\x1a\n"), contentTypePlainText},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.filename, func(t *testing.T) {
got, knownPath := detectContentTypeFromPath(tt.filename) got := detectContentType(tt.filename)
if !knownPath {
got = detectContentTypeFromPrefix(tt.prefix)
}
if got != tt.expectedCT { if got != tt.expectedCT {
t.Errorf("content type for %q with prefix %q = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT) t.Errorf("detectContentType(%q) = %q, want %q", tt.filename, got, tt.expectedCT)
} }
}) })
} }
@ -286,10 +255,6 @@ func createTestArchive(t *testing.T) []byte {
"package/lib/index.js": "module.exports = {};", "package/lib/index.js": "module.exports = {};",
"package/lib/helper.js": "module.exports.help = () => {};", "package/lib/helper.js": "module.exports.help = () => {};",
"package/test/index.test.js": "// tests", "package/test/index.test.js": "// tests",
"package/notes.data": "short text\n",
"package/logo": "\x89PNG\r\n\x1a\nimage data",
"package/page": "<!DOCTYPE html><html></html>",
"package/misleading.txt": "\x89PNG\r\n\x1a\nimage data",
} }
for path, content := range files { for path, content := range files {
@ -349,7 +314,7 @@ func TestBrowseNonCachedArtifact(t *testing.T) {
} }
// Try to browse // Try to browse
req := httptest.NewRequest("GET", "/ui/api/browse/npm/not-cached/1.0.0", nil) req := httptest.NewRequest("GET", "/api/browse/npm/not-cached/1.0.0", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -403,7 +368,7 @@ func TestHandleBrowseSourcePage(t *testing.T) {
} }
// Test the browse source page loads // Test the browse source page loads
req := httptest.NewRequest("GET", "/ui/package/npm/test-browse/1.0.0/browse", nil) req := httptest.NewRequest("GET", "/package/npm/test-browse/1.0.0/browse", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -430,10 +395,6 @@ func TestHandleBrowseSourcePage(t *testing.T) {
} }
} }
if !strings.Contains(body, "proxy test-version (test-commit)") {
t.Error("browse source footer should contain proxy build information, not the package version")
}
// Check that the escapeHTML function is present for XSS protection // Check that the escapeHTML function is present for XSS protection
if !strings.Contains(body, "function escapeHTML(str)") { if !strings.Contains(body, "function escapeHTML(str)") {
t.Error("browse source page missing escapeHTML function for XSS protection") t.Error("browse source page missing escapeHTML function for XSS protection")
@ -454,10 +415,8 @@ func TestHandleBrowseSourcePage(t *testing.T) {
if !strings.Contains(body, "const packageName = 'test-browse'") { if !strings.Contains(body, "const packageName = 'test-browse'") {
t.Error("browse source page missing packageName variable") t.Error("browse source page missing packageName variable")
} }
// The version reaches the browse API as one path segment, so the page holds if !strings.Contains(body, "const version = '1.0.0'") {
// its escaped form. t.Error("browse source page missing version variable")
if !strings.Contains(body, "const versionPath = '1.0.0'") {
t.Error("browse source page missing versionPath variable")
} }
// Verify content type // Verify content type
@ -542,7 +501,7 @@ func TestHandleCompareDiff(t *testing.T) {
} }
// Test the compare endpoint // Test the compare endpoint
req := httptest.NewRequest("GET", "/ui/api/compare/npm/test-compare/1.0.0/2.0.0", nil) req := httptest.NewRequest("GET", "/api/compare/npm/test-compare/1.0.0/2.0.0", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -613,7 +572,7 @@ func TestHandleComparePage(t *testing.T) {
defer ts.close() defer ts.close()
// Test valid format with ... separator // Test valid format with ... separator
req := httptest.NewRequest("GET", "/ui/package/npm/test/compare/1.0.0...2.0.0", nil) req := httptest.NewRequest("GET", "/package/npm/test/compare/1.0.0...2.0.0", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -623,17 +582,16 @@ func TestHandleComparePage(t *testing.T) {
body := w.Body.String() body := w.Body.String()
// Check that versions are set correctly in JavaScript. The compare API takes // Check that versions are set correctly in JavaScript
// each version as a path segment, so the page holds their escaped forms. if !strings.Contains(body, "const fromVersion = '1.0.0'") {
if !strings.Contains(body, "const fromVersionPath = '1.0.0'") { t.Error("page should set fromVersion")
t.Error("page should set fromVersionPath")
} }
if !strings.Contains(body, "const toVersionPath = '2.0.0'") { if !strings.Contains(body, "const toVersion = '2.0.0'") {
t.Error("page should set toVersionPath") t.Error("page should set toVersion")
} }
// Test invalid format (missing separator) // Test invalid format (missing separator)
req = httptest.NewRequest("GET", "/ui/package/npm/test/compare/invalid", nil) req = httptest.NewRequest("GET", "/package/npm/test/compare/invalid", nil)
w = httptest.NewRecorder() w = httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -642,7 +600,7 @@ func TestHandleComparePage(t *testing.T) {
} }
// Test with only one dot (should fail) // Test with only one dot (should fail)
req = httptest.NewRequest("GET", "/ui/package/npm/test/compare/1.0.0.2.0.0", nil) req = httptest.NewRequest("GET", "/package/npm/test/compare/1.0.0.2.0.0", nil)
w = httptest.NewRecorder() w = httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -651,19 +609,25 @@ func TestHandleComparePage(t *testing.T) {
} }
} }
func TestOpenArchiveDetectsExtensionlessTarGz(t *testing.T) { func TestArchiveFilename(t *testing.T) {
reader, err := openArchive("artifact", bytes.NewReader(createTestArchive(t)), "npm") tests := []struct {
if err != nil { input string
t.Fatalf("openArchive failed: %v", err) want string
}{
{"package.tar.gz", "package.tar.gz"},
{"d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856.zip"},
{"file.zip", "file.zip"},
{"archive.tgz", "archive.tgz"},
{"noext", "noext.zip"},
} }
defer func() { _ = reader.Close() }()
files, err := reader.List() for _, tt := range tests {
if err != nil { t.Run(tt.input, func(t *testing.T) {
t.Fatalf("List failed: %v", err) got := archiveFilename(tt.input)
} if got != tt.want {
if len(files) == 0 { t.Errorf("archiveFilename(%q) = %q, want %q", tt.input, got, tt.want)
t.Fatal("expected files in extensionless archive") }
})
} }
} }
@ -836,70 +800,3 @@ func createTarGzArchive(t *testing.T, files map[string]string) []byte {
} }
return buf.Bytes() return buf.Bytes()
} }
// TestFirstBrowsableArtifact guards artifact selection against PEP 658
// core-metadata sidecars. A sidecar resolves to the same version as the
// distribution it describes, so it is cached under that version, but it is plain
// text and openArchive cannot parse it.
func TestFirstBrowsableArtifact(t *testing.T) {
cached := func(filename string) database.Artifact {
return database.Artifact{
Filename: filename,
StoragePath: sql.NullString{String: "pypi/" + filename, Valid: true},
}
}
uncached := func(filename string) database.Artifact {
return database.Artifact{Filename: filename}
}
tests := []struct {
name string
artifacts []database.Artifact
want string
}{
{"no artifacts", nil, ""},
{
"sidecar only is not browsable",
[]database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata")},
"",
},
{
// '-' (0x2D) sorts before '.' (0x2E), so the sidecar precedes the
// sdist in the filename-ordered list the query returns.
"sidecar sorting ahead of the sdist is skipped",
[]database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0.tar.gz")},
"foo-1.0.tar.gz",
},
{
"sidecar skipped in favour of its own wheel",
[]database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0-py3-none-any.whl")},
"foo-1.0-py3-none-any.whl",
},
{
"uncached archive is still not selected",
[]database.Artifact{cached("foo-1.0.tar.gz.metadata"), uncached("foo-1.0.tar.gz")},
"",
},
{"plain sdist", []database.Artifact{cached("foo-1.0.tar.gz")}, "foo-1.0.tar.gz"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := firstBrowsableArtifact(tt.artifacts)
if tt.want == "" {
if got != nil {
t.Fatalf("firstBrowsableArtifact() = %q, want nil", got.Filename)
}
return
}
if got == nil {
t.Fatalf("firstBrowsableArtifact() = nil, want %q", tt.want)
}
if got.Filename != tt.want {
t.Errorf("firstBrowsableArtifact() = %q, want %q", got.Filename, tt.want)
}
})
}
}

View file

@ -8,7 +8,6 @@ import (
// DashboardData contains data for rendering the dashboard. // DashboardData contains data for rendering the dashboard.
type DashboardData struct { type DashboardData struct {
Layout
Stats DashboardStats Stats DashboardStats
EnrichmentStats EnrichmentStatsView EnrichmentStats EnrichmentStatsView
RecentPackages []PackageInfo RecentPackages []PackageInfo
@ -61,7 +60,6 @@ type RegistryConfig struct {
// PackageShowData contains data for rendering the package show page. // PackageShowData contains data for rendering the package show page.
type PackageShowData struct { type PackageShowData struct {
Layout
Package *database.Package Package *database.Package
Versions []database.Version Versions []database.Version
Vulnerabilities []database.Vulnerability Vulnerabilities []database.Vulnerability
@ -70,7 +68,6 @@ type PackageShowData struct {
// VersionShowData contains data for rendering the version show page. // VersionShowData contains data for rendering the version show page.
type VersionShowData struct { type VersionShowData struct {
Layout
Package *database.Package Package *database.Package
Version *database.Version Version *database.Version
Artifacts []database.Artifact Artifacts []database.Artifact
@ -82,7 +79,6 @@ type VersionShowData struct {
// SearchPageData contains data for rendering the search results page. // SearchPageData contains data for rendering the search results page.
type SearchPageData struct { type SearchPageData struct {
Layout
Query string Query string
Ecosystem string Ecosystem string
Results []SearchResultItem Results []SearchResultItem
@ -108,7 +104,6 @@ type SearchResultItem struct {
// PackagesListPageData contains data for rendering the packages list page. // PackagesListPageData contains data for rendering the packages list page.
type PackagesListPageData struct { type PackagesListPageData struct {
Layout
Ecosystem string Ecosystem string
SortBy string SortBy string
Results []SearchResultItem Results []SearchResultItem
@ -132,7 +127,6 @@ func supportedEcosystems() []string {
"gem", "gem",
"golang", "golang",
"hex", "hex",
"julia",
"maven", "maven",
"npm", "npm",
"nuget", "nuget",
@ -182,8 +176,6 @@ func ecosystemBadgeClasses(ecosystem string) string {
return base + " bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300" return base + " bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300"
case "cran": case "cran":
return base + " bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300" return base + " bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300"
case "julia":
return base + " bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300"
case "oci": case "oci":
return base + " bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300" return base + " bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300"
case "deb": case "deb":
@ -385,17 +377,6 @@ local({
r["CRAN"] &lt;- "` + baseURL + `/cran" r["CRAN"] &lt;- "` + baseURL + `/cran"
options(repos = r) options(repos = r)
})</code></pre>`), })</code></pre>`),
},
{
ID: "julia",
Name: "Julia",
Language: "Julia",
Endpoint: "/julia/",
Instructions: template.HTML(`<p class="config-note">Set the Pkg server before starting Julia:</p>
<pre><code>export JULIA_PKG_SERVER=` + baseURL + `/julia</code></pre>
<p class="config-note">Or inside a running session:</p>
<pre><code>ENV["JULIA_PKG_SERVER"] = "` + baseURL + `/julia"
using Pkg; Pkg.update()</code></pre>`),
}, },
{ {
ID: "oci", ID: "oci",

View file

@ -15,7 +15,7 @@ import (
"github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/proxy/internal/storage"
) )
func setupEvictionTest(t *testing.T) (*database.DB, *storage.Blob) { func setupEvictionTest(t *testing.T) (*database.DB, *storage.Filesystem) {
t.Helper() t.Helper()
tempDir := t.TempDir() tempDir := t.TempDir()
@ -27,7 +27,7 @@ func setupEvictionTest(t *testing.T) (*database.DB, *storage.Blob) {
t.Fatalf("failed to create database: %v", err) t.Fatalf("failed to create database: %v", err)
} }
store, err := storage.OpenBucket(context.Background(), "file://"+storagePath) store, err := storage.NewFilesystem(storagePath)
if err != nil { if err != nil {
_ = db.Close() _ = db.Close()
t.Fatalf("failed to create storage: %v", err) t.Fatalf("failed to create storage: %v", err)
@ -243,7 +243,7 @@ func TestStartEvictionLoop_UnlimitedSkips(t *testing.T) {
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
store, err := storage.OpenBucket(context.Background(), "file://"+storagePath) store, err := storage.NewFilesystem(storagePath)
if err != nil { if err != nil {
t.Fatalf("failed to create storage: %v", err) t.Fatalf("failed to create storage: %v", err)
} }
@ -280,7 +280,7 @@ func defaultTestConfig(storagePath, dbPath string) *config.Config {
return &config.Config{ return &config.Config{
Listen: ":8080", Listen: ":8080",
BaseURL: "http://localhost:8080", BaseURL: "http://localhost:8080",
Storage: config.StorageConfig{URL: "file://" + storagePath, MaxSize: ""}, Storage: config.StorageConfig{Path: storagePath, MaxSize: ""},
Database: config.DatabaseConfig{ Database: config.DatabaseConfig{
Driver: "sqlite", Driver: "sqlite",
Path: dbPath, Path: dbPath,

View file

@ -1,182 +0,0 @@
// Package server implements the proxy HTTP server.
package server
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"strconv"
"sync"
"time"
"github.com/git-pkgs/proxy/internal/metrics"
"github.com/git-pkgs/proxy/internal/storage"
)
const (
probePathPrefix = ".healthcheck/"
probeMarker = "proxy-healthcheck:"
probeSuffixBytes = 8
defaultProbeTTL = 30 * time.Second
defaultProbeTimeout = 10 * time.Second
)
// HealthResponse is the JSON payload returned by /health.
type HealthResponse struct {
Status string `json:"status"`
Checks map[string]HealthCheck `json:"checks"`
}
// HealthCheck reports the status of a single subsystem check.
type HealthCheck struct {
Status string `json:"status"`
Error string `json:"error,omitempty"`
Step string `json:"step,omitempty"`
}
// probeError tags a storage probe failure with the step that failed.
type probeError struct {
step string
err error
}
func (e *probeError) Error() string { return e.step + ": " + e.err.Error() }
func (e *probeError) Unwrap() error { return e.err }
// storageProbe runs a write → size-check → read → verify → delete round-trip
// against the storage backend. Returns nil on success or a *probeError on failure.
func storageProbe(ctx context.Context, s storage.Storage) (err error) {
suffix, suffixErr := randomSuffix()
if suffixErr != nil {
return &probeError{step: "write", err: fmt.Errorf("generating random suffix: %w", suffixErr)}
}
path := probePathPrefix + strconv.FormatInt(time.Now().UnixNano(), 10) + "-" + suffix
payload := []byte(probeMarker + suffix)
// 1. Store
size, _, storeErr := s.Store(ctx, path, bytes.NewReader(payload))
if storeErr != nil {
return &probeError{step: "write", err: storeErr}
}
// After Store succeeds, always attempt to delete on the way out so probe
// objects don't accumulate when a later step (size/open/read/verify) fails.
// Delete is reported as the primary error only if no earlier failure
// already set one.
defer func() {
if delErr := s.Delete(ctx, path); delErr != nil && err == nil {
err = &probeError{step: "delete", err: delErr}
}
}()
// 2. Size check
if size != int64(len(payload)) {
return &probeError{step: "size", err: fmt.Errorf("wrote %d bytes, expected %d", size, len(payload))}
}
// 3. Open
rc, openErr := s.Open(ctx, path)
if openErr != nil {
return &probeError{step: "read", err: openErr}
}
// 4. Read all (classify mid-stream errors as read, not verify).
// Close explicitly (not deferred) so the file handle is released before
// Delete — on Windows, an open handle prevents deletion.
data, readErr := io.ReadAll(rc)
_ = rc.Close()
if readErr != nil {
return &probeError{step: "read", err: readErr}
}
// 5. Verify
if !bytes.Equal(data, payload) {
return &probeError{step: "verify", err: fmt.Errorf("content mismatch")}
}
// 6. Delete is handled via the deferred cleanup above.
return nil
}
// randomSuffix returns 8 cryptographically random bytes hex-encoded.
func randomSuffix() (string, error) {
b := make([]byte, probeSuffixBytes)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// healthCache memoizes the result of storageProbe for a configurable TTL.
// It is safe for concurrent use.
type healthCache struct {
storage storage.Storage
interval time.Duration
probeTimeout time.Duration
logger *slog.Logger
mu sync.Mutex
lastAt time.Time
lastErr error
}
// newHealthCache builds a cache, parsing the interval from a duration string.
// Empty interval string defaults to 30s. "0" or "0s" disables caching.
func newHealthCache(s storage.Storage, intervalStr string, logger *slog.Logger) (*healthCache, error) {
interval := defaultProbeTTL
if intervalStr != "" {
d, err := time.ParseDuration(intervalStr)
if err != nil {
return nil, fmt.Errorf("parsing storage_probe_interval %q: %w", intervalStr, err)
}
interval = d
}
return &healthCache{
storage: s,
interval: interval,
probeTimeout: defaultProbeTimeout,
logger: logger,
}, nil
}
// Check returns the cached probe result if still fresh, otherwise runs a fresh probe.
// The probe runs under a context derived from context.Background() with a fixed
// timeout so that caller cancellation (e.g. client disconnect) cannot poison the
// cache with context.Canceled.
func (c *healthCache) Check() error {
c.mu.Lock()
defer c.mu.Unlock()
// Cache hit
if c.interval > 0 && !c.lastAt.IsZero() && time.Since(c.lastAt) < c.interval {
return c.lastErr
}
// Fresh probe under a detached context
probeCtx, cancel := context.WithTimeout(context.Background(), c.probeTimeout)
defer cancel()
err := storageProbe(probeCtx, c.storage)
// Transition logging and metric increment happen only on the fresh-probe path.
c.logTransition(c.lastErr, err)
if err != nil {
var pe *probeError
if errors.As(err, &pe) {
metrics.RecordHealthProbeFailure(pe.step)
} else {
metrics.RecordHealthProbeFailure("unknown")
}
}
c.lastErr = err
c.lastAt = time.Now()
return err
}
func (c *healthCache) logTransition(prev, curr error) {
switch {
case prev != nil && curr == nil:
c.logger.Info("storage probe recovered")
case prev == nil && curr != nil:
c.logger.Error("storage probe failed", "error", curr.Error())
}
}

View file

@ -1,448 +0,0 @@
package server
import (
"bytes"
"context"
"errors"
"io"
"log/slog"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/git-pkgs/proxy/internal/metrics"
"github.com/git-pkgs/proxy/internal/storage"
"github.com/prometheus/client_golang/prometheus/testutil"
)
// fakeStorage is a minimal storage.Storage for probe tests with per-step failure injection.
type fakeStorage struct {
mu sync.Mutex
storeCalls atomic.Int64
openCalls atomic.Int64
closeCalls atomic.Int64
deleteCalls atomic.Int64
paths []string
payloads [][]byte
// Failure injection.
storeErr error
openErr error
readErr error // returned by the io.ReadCloser.Read after partial bytes
deleteErr error
// Misbehavior knobs.
sizeDelta int64 // added to the reported size from Store
readOverride []byte // if non-nil, Open returns a reader yielding these bytes instead of stored content
// storeBlock, if non-nil, causes Store to block until the channel is closed or ctx is done.
storeBlock chan struct{}
stored map[string][]byte
}
func newFakeStorage() *fakeStorage { return &fakeStorage{stored: map[string][]byte{}} }
func (f *fakeStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
f.storeCalls.Add(1)
if f.storeErr != nil {
return 0, "", f.storeErr
}
if f.storeBlock != nil {
select {
case <-f.storeBlock:
case <-ctx.Done():
return 0, "", ctx.Err()
}
}
data, err := io.ReadAll(r)
if err != nil {
return 0, "", err
}
f.mu.Lock()
f.stored[path] = data
f.paths = append(f.paths, path)
f.payloads = append(f.payloads, data)
f.mu.Unlock()
return int64(len(data)) + f.sizeDelta, "fakehash", nil
}
type fakeReadCloser struct {
data []byte
pos int
readErr error
closed *atomic.Int64
}
func (rc *fakeReadCloser) Read(p []byte) (int, error) {
if rc.pos >= len(rc.data) {
if rc.readErr != nil {
return 0, rc.readErr
}
return 0, io.EOF
}
n := copy(p, rc.data[rc.pos:])
rc.pos += n
if rc.pos >= len(rc.data) && rc.readErr != nil {
return n, rc.readErr
}
return n, nil
}
func (rc *fakeReadCloser) Close() error { rc.closed.Add(1); return nil }
func (f *fakeStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) {
f.openCalls.Add(1)
if f.openErr != nil {
return nil, f.openErr
}
f.mu.Lock()
data := f.stored[path]
f.mu.Unlock()
if f.readOverride != nil {
data = f.readOverride
}
return &fakeReadCloser{data: data, readErr: f.readErr, closed: &f.closeCalls}, nil
}
func (f *fakeStorage) Exists(ctx context.Context, path string) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
_, ok := f.stored[path]
return ok, nil
}
func (f *fakeStorage) Delete(ctx context.Context, path string) error {
f.deleteCalls.Add(1)
if f.deleteErr != nil {
return f.deleteErr
}
f.mu.Lock()
delete(f.stored, path)
f.mu.Unlock()
return nil
}
func (f *fakeStorage) Size(ctx context.Context, path string) (int64, error) { return 0, nil }
func (f *fakeStorage) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
return "", storage.ErrSignedURLUnsupported
}
func (f *fakeStorage) UsedSpace(ctx context.Context) (int64, error) { return 0, nil }
func (f *fakeStorage) URL() string { return "fake://" }
func (f *fakeStorage) Close() error { return nil }
// --- Tests follow. First test: happy path ---
func TestStorageProbe_HappyPath(t *testing.T) {
fs := newFakeStorage()
if err := storageProbe(context.Background(), fs); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := fs.storeCalls.Load(); got != 1 {
t.Errorf("Store calls = %d, want 1", got)
}
if got := fs.openCalls.Load(); got != 1 {
t.Errorf("Open calls = %d, want 1", got)
}
if got := fs.closeCalls.Load(); got != 1 {
t.Errorf("Close calls = %d, want 1", got)
}
if got := fs.deleteCalls.Load(); got != 1 {
t.Errorf("Delete calls = %d, want 1", got)
}
if len(fs.paths) != 1 || !strings.HasPrefix(fs.paths[0], ".healthcheck/") {
t.Errorf("unexpected probe path: %v", fs.paths)
}
}
func TestStorageProbe_WriteFails(t *testing.T) {
fs := newFakeStorage()
fs.storeErr = errors.New("disk full")
err := storageProbe(context.Background(), fs)
var pe *probeError
if !errors.As(err, &pe) {
t.Fatalf("expected *probeError, got %T: %v", err, err)
}
if pe.step != "write" {
t.Errorf("step = %q, want write", pe.step)
}
if fs.openCalls.Load() != 0 {
t.Errorf("Open should not be called after write failure")
}
}
func TestStorageProbe_SizeMismatch(t *testing.T) {
fs := newFakeStorage()
fs.sizeDelta = -1 // Report 1 byte fewer than actually written
err := storageProbe(context.Background(), fs)
var pe *probeError
if !errors.As(err, &pe) || pe.step != "size" {
t.Fatalf("step = %v, want size; err = %v", pe, err)
}
if fs.openCalls.Load() != 0 {
t.Errorf("Open should not be called after size mismatch")
}
}
func TestStorageProbe_OpenFails(t *testing.T) {
fs := newFakeStorage()
fs.openErr = errors.New("access denied")
err := storageProbe(context.Background(), fs)
var pe *probeError
if !errors.As(err, &pe) || pe.step != "read" {
t.Fatalf("step = %v, want read; err = %v", pe, err)
}
}
func TestStorageProbe_ReadMidStreamFails(t *testing.T) {
fs := newFakeStorage()
fs.readErr = errors.New("connection reset")
err := storageProbe(context.Background(), fs)
var pe *probeError
if !errors.As(err, &pe) || pe.step != "read" {
t.Fatalf("step = %v, want read (NOT verify); err = %v", pe, err)
}
}
func TestStorageProbe_ContentMismatch(t *testing.T) {
fs := newFakeStorage()
fs.readOverride = []byte("wrong content")
err := storageProbe(context.Background(), fs)
var pe *probeError
if !errors.As(err, &pe) || pe.step != "verify" {
t.Fatalf("step = %v, want verify; err = %v", pe, err)
}
}
func TestStorageProbe_DeleteFails(t *testing.T) {
fs := newFakeStorage()
fs.deleteErr = errors.New("permission denied")
err := storageProbe(context.Background(), fs)
var pe *probeError
if !errors.As(err, &pe) || pe.step != "delete" {
t.Fatalf("step = %v, want delete; err = %v", pe, err)
}
}
// TestStorageProbe_CleanupOnNonDeleteFailure asserts that the probe object is
// deleted even when a step after Store (size/open/read/verify) fails, so
// probe artifacts don't accumulate in the storage backend.
func TestStorageProbe_CleanupOnNonDeleteFailure(t *testing.T) {
cases := []struct {
name string
inject func(*fakeStorage)
wantErr string
}{
{"size mismatch", func(fs *fakeStorage) { fs.sizeDelta = -1 }, "size"},
{"open fails", func(fs *fakeStorage) { fs.openErr = errors.New("open boom") }, "read"},
{"read mid-stream", func(fs *fakeStorage) { fs.readErr = errors.New("mid-stream boom") }, "read"},
{"content mismatch", func(fs *fakeStorage) { fs.readOverride = []byte("wrong") }, "verify"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
fs := newFakeStorage()
tc.inject(fs)
err := storageProbe(context.Background(), fs)
var pe *probeError
if !errors.As(err, &pe) || pe.step != tc.wantErr {
t.Fatalf("step = %v, want %q; err = %v", pe, tc.wantErr, err)
}
if got := fs.deleteCalls.Load(); got != 1 {
t.Errorf("deleteCalls = %d, want 1 (cleanup should run on non-delete failures)", got)
}
})
}
}
func TestStorageProbe_ReaderClosedOnReadFailure(t *testing.T) {
fs := newFakeStorage()
fs.readErr = errors.New("read error")
_ = storageProbe(context.Background(), fs)
if got := fs.closeCalls.Load(); got != fs.openCalls.Load() {
t.Errorf("closeCalls = %d, openCalls = %d (should match)", got, fs.openCalls.Load())
}
}
func TestStorageProbe_PathUniqueness(t *testing.T) {
fs := newFakeStorage()
for i := 0; i < 100; i++ {
if err := storageProbe(context.Background(), fs); err != nil {
t.Fatalf("probe %d: %v", i, err)
}
}
seen := make(map[string]bool)
for _, p := range fs.paths {
if !strings.HasPrefix(p, ".healthcheck/") {
t.Errorf("path missing prefix: %q", p)
}
if seen[p] {
t.Errorf("duplicate path: %q", p)
}
seen[p] = true
}
}
// helper: a healthCache wired to a fakeStorage and a discard logger.
func newTestCache(fs *fakeStorage, interval time.Duration) *healthCache {
return &healthCache{
storage: fs,
interval: interval,
probeTimeout: 5 * time.Second,
logger: discardLogger(),
}
}
func discardLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(io.Discard, nil))
}
func TestHealthCache_CacheHit(t *testing.T) {
fs := newFakeStorage()
c := newTestCache(fs, 30*time.Second)
if err := c.Check(); err != nil {
t.Fatalf("first check: %v", err)
}
if err := c.Check(); err != nil {
t.Fatalf("second check: %v", err)
}
if got := fs.storeCalls.Load(); got != 1 {
t.Errorf("storeCalls = %d, want 1 (second call should be cached)", got)
}
}
func TestHealthCache_MissAfterTTL(t *testing.T) {
fs := newFakeStorage()
c := newTestCache(fs, 10*time.Millisecond)
_ = c.Check()
time.Sleep(20 * time.Millisecond)
_ = c.Check()
if got := fs.storeCalls.Load(); got != 2 {
t.Errorf("storeCalls = %d, want 2", got)
}
}
func TestHealthCache_Disabled(t *testing.T) {
fs := newFakeStorage()
c := newTestCache(fs, 0) // interval = 0 means probe every call
_ = c.Check()
_ = c.Check()
if got := fs.storeCalls.Load(); got != 2 {
t.Errorf("storeCalls = %d, want 2", got)
}
}
func TestHealthCache_LastAtNotAdvancedOnHit(t *testing.T) {
fs := newFakeStorage()
c := newTestCache(fs, 30*time.Second)
for i := 0; i < 100; i++ {
_ = c.Check()
}
if got := fs.storeCalls.Load(); got != 1 {
t.Errorf("storeCalls = %d, want 1 across 100 hits", got)
}
}
func TestHealthCache_ConcurrentSingleFlight(t *testing.T) {
fs := newFakeStorage()
c := newTestCache(fs, 30*time.Second)
var wg sync.WaitGroup
for i := 0; i < 20; i++ {
wg.Add(1)
go func() { defer wg.Done(); _ = c.Check() }()
}
wg.Wait()
if got := fs.storeCalls.Load(); got != 1 {
t.Errorf("storeCalls = %d, want 1 with 20 concurrent callers", got)
}
}
func TestHealthCache_FailureCounterIncrement(t *testing.T) {
fs := newFakeStorage()
fs.storeErr = errors.New("boom")
c := newTestCache(fs, 30*time.Second)
before := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
// First call: fresh probe → counter +1
_ = c.Check()
afterFirst := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
if afterFirst-before != 1 {
t.Errorf("counter delta after first call = %v, want 1", afterFirst-before)
}
// Second call: cache hit → counter NOT re-incremented
_ = c.Check()
afterSecond := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
if afterSecond != afterFirst {
t.Errorf("counter changed on cache hit: %v → %v", afterFirst, afterSecond)
}
}
func TestHealthCache_ProbeTimeout(t *testing.T) {
fs := newFakeStorage()
fs.storeBlock = make(chan struct{}) // Store will block until channel is closed (or never)
t.Cleanup(func() { close(fs.storeBlock) })
c := &healthCache{
storage: fs,
interval: 30 * time.Second,
probeTimeout: 50 * time.Millisecond,
logger: discardLogger(),
}
start := time.Now()
err := c.Check()
elapsed := time.Since(start)
if err == nil {
t.Fatal("expected timeout error, got nil")
}
if elapsed > 500*time.Millisecond {
t.Errorf("probe took %v, expected ~50ms (timeout not respected)", elapsed)
}
}
func TestHealthCache_TransitionLogging(t *testing.T) {
fs := newFakeStorage()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
c := &healthCache{
storage: fs,
interval: 0, // probe every call
probeTimeout: 5 * time.Second,
logger: logger,
}
// Steady ok state — should not log
_ = c.Check()
_ = c.Check()
if got := strings.Count(buf.String(), "storage probe"); got != 0 {
t.Errorf("steady-state logs = %d, want 0; output: %s", got, buf.String())
}
// ok → err transition: exactly one Error log
buf.Reset()
fs.storeErr = errors.New("boom")
_ = c.Check()
if !strings.Contains(buf.String(), "storage probe failed") {
t.Errorf("missing failure log on transition; output: %s", buf.String())
}
// err steady state — should not log again
buf.Reset()
_ = c.Check()
if buf.Len() != 0 {
t.Errorf("steady-err logs = %q, want empty", buf.String())
}
// err → ok transition: exactly one Info log
buf.Reset()
fs.storeErr = nil
_ = c.Check()
if !strings.Contains(buf.String(), "storage probe recovered") {
t.Errorf("missing recovery log on transition; output: %s", buf.String())
}
}

View file

@ -1,26 +0,0 @@
package server
import "net/http"
// BuildInfo identifies the running proxy binary.
type BuildInfo struct {
Version string
Commit string
}
// Layout carries shared fields consumed by the base template. It is embedded
// in every page data struct so templates can access canonical URL and build
// information alongside the page's own fields.
type Layout struct {
BuildInfo BuildInfo
UIBaseURL string
CanonicalPath string
}
func (s *Server) layoutFor(r *http.Request) Layout {
return Layout{
BuildInfo: s.buildInfo,
UIBaseURL: s.cfg.UIBaseURL,
CanonicalPath: r.URL.Path,
}
}

View file

@ -3,15 +3,16 @@ package server
import ( import (
"context" "context"
"net/http" "net/http"
"strings"
"sync/atomic" "sync/atomic"
"time" "time"
"github.com/git-pkgs/proxy/internal/accesslog"
"github.com/git-pkgs/proxy/internal/metrics"
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
) )
type contextKey string
const requestIDKey contextKey = "request_id"
var requestCounter atomic.Uint64 var requestCounter atomic.Uint64
// RequestIDMiddleware adds a sequential request ID to the context and response headers. // RequestIDMiddleware adds a sequential request ID to the context and response headers.
@ -22,7 +23,7 @@ func RequestIDMiddleware(next http.Handler) http.Handler {
requestID := middleware.GetReqID(r.Context()) requestID := middleware.GetReqID(r.Context())
// Store formatted ID in context // Store formatted ID in context
ctx := accesslog.WithRequestID(r.Context(), requestID) ctx := context.WithValue(r.Context(), requestIDKey, requestID)
// Add to response header for client tracking // Add to response header for client tracking
w.Header().Set("X-Request-ID", requestID) w.Header().Set("X-Request-ID", requestID)
@ -33,7 +34,10 @@ func RequestIDMiddleware(next http.Handler) http.Handler {
// GetRequestID retrieves the request ID from context. // GetRequestID retrieves the request ID from context.
func GetRequestID(ctx context.Context) string { func GetRequestID(ctx context.Context) string {
return accesslog.RequestID(ctx) if id, ok := ctx.Value(requestIDKey).(string); ok {
return id
}
return ""
} }
// LoggerMiddleware logs HTTP requests with request ID correlation. // LoggerMiddleware logs HTTP requests with request ID correlation.
@ -44,51 +48,27 @@ func (s *Server) LoggerMiddleware(next http.Handler) http.Handler {
rw := &responseWriter{ResponseWriter: w, status: http.StatusOK} rw := &responseWriter{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(rw, r) next.ServeHTTP(rw, r)
duration := time.Since(start)
s.logger.Info("request", s.logger.Info("request",
"request_id", requestID, "request_id", requestID,
"method", r.Method, "method", r.Method,
"path", r.URL.Path, "path", r.URL.Path,
"status", rw.status, "status", rw.status,
"duration", duration, "duration", time.Since(start),
"remote", r.RemoteAddr) "remote", r.RemoteAddr)
if r.URL.Path != "/metrics" {
metrics.RecordRequest(requestEcosystem(r.URL.Path), rw.status, duration)
}
if s.accessLog != nil {
if err := s.accessLog.Write(accesslog.Entry{
Event: accesslog.EventRequest,
RequestID: requestID,
Method: r.Method,
Path: r.URL.EscapedPath(),
StatusCode: rw.status,
DurationMS: duration.Milliseconds(),
RemoteAddr: r.RemoteAddr,
}); err != nil {
s.logger.Error("failed to write access log", "error", err)
}
}
}) })
} }
func requestEcosystem(path string) string { // ActiveRequestsMiddleware tracks the number of active requests using Prometheus metrics.
segment, _, _ := strings.Cut(strings.TrimPrefix(path, "/"), "/") func ActiveRequestsMiddleware(next http.Handler) http.Handler {
switch segment { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", // Don't track metrics endpoint itself
"conan", "conda", "cran", "julia", "debian", "rpm": if r.URL.Path == "/metrics" {
return segment next.ServeHTTP(w, r)
case "gem": return
return "rubygems" }
case "go":
return "golang" // Implemented in server.go where metrics package is imported
case "composer": next.ServeHTTP(w, r)
return "packagist" })
case "v2":
return "oci"
default:
return "other"
}
} }

View file

@ -2,21 +2,13 @@ package server
import ( import (
"context" "context"
"encoding/json"
"io" "io"
"log/slog" "log/slog"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os"
"path/filepath"
"testing" "testing"
"github.com/git-pkgs/proxy/internal/accesslog"
"github.com/git-pkgs/proxy/internal/metrics"
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
dto "github.com/prometheus/client_model/go"
) )
func TestRequestIDMiddleware(t *testing.T) { func TestRequestIDMiddleware(t *testing.T) {
@ -53,7 +45,7 @@ func TestGetRequestID(t *testing.T) {
}{ }{
{ {
name: "with request ID", name: "with request ID",
ctx: accesslog.WithRequestID(context.Background(), "test-123"), ctx: context.WithValue(context.Background(), requestIDKey, "test-123"),
expected: "test-123", expected: "test-123",
}, },
{ {
@ -73,6 +65,36 @@ func TestGetRequestID(t *testing.T) {
} }
} }
func TestActiveRequestsMiddleware(t *testing.T) {
handler := ActiveRequestsMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
req := httptest.NewRequest(http.MethodGet, "/test", nil)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Errorf("expected status 200, got %d", rec.Code)
}
}
func TestActiveRequestsMiddleware_SkipsMetricsEndpoint(t *testing.T) {
handler := ActiveRequestsMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Errorf("expected status 200, got %d", rec.Code)
}
}
func TestLoggerMiddleware(t *testing.T) { func TestLoggerMiddleware(t *testing.T) {
logger := slog.New(slog.NewTextHandler(io.Discard, nil)) logger := slog.New(slog.NewTextHandler(io.Discard, nil))
s := &Server{logger: logger} s := &Server{logger: logger}
@ -99,133 +121,6 @@ func TestLoggerMiddleware(t *testing.T) {
} }
} }
func TestLoggerMiddlewareRecordsRequestMetrics(t *testing.T) {
before := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("rubygems", "404"))
durationMetric := metrics.RequestDuration.WithLabelValues("rubygems", "404")
beforeDurationCount := histogramSampleCount(t, durationMetric)
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
s := &Server{logger: logger}
handler := s.LoggerMiddleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
req := httptest.NewRequest(http.MethodGet, "/gem/downloads/missing.gem", nil)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
after := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("rubygems", "404"))
if got := after - before; got != 1 {
t.Errorf("request counter delta = %.0f, want 1", got)
}
afterDurationCount := histogramSampleCount(t, durationMetric)
if got := afterDurationCount - beforeDurationCount; got != 1 {
t.Errorf("request duration sample delta = %d, want 1", got)
}
}
func histogramSampleCount(t *testing.T, observer prometheus.Observer) uint64 {
t.Helper()
metric, ok := observer.(prometheus.Metric)
if !ok {
t.Fatal("histogram observer does not implement prometheus.Metric")
}
var value dto.Metric
if err := metric.Write(&value); err != nil {
t.Fatalf("writing histogram metric: %v", err)
}
return value.GetHistogram().GetSampleCount()
}
func TestLoggerMiddlewareSkipsMetricsEndpointMetrics(t *testing.T) {
before := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("other", "200"))
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
s := &Server{logger: logger}
handler := s.LoggerMiddleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
after := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("other", "200"))
if got := after - before; got != 0 {
t.Errorf("request counter delta = %.0f, want 0", got)
}
}
func TestRequestEcosystem(t *testing.T) {
tests := []struct {
path string
want string
}{
{path: "/npm/lodash", want: "npm"},
{path: "/gem/downloads/rails.gem", want: "rubygems"},
{path: "/go/example.com/module/@v/list", want: "golang"},
{path: "/composer/vendor/package", want: "packagist"},
{path: "/v2/library/alpine/manifests/latest", want: "oci"},
{path: "/ui/", want: "other"},
{path: "/api/package/npm/lodash", want: "other"},
{path: "/", want: "other"},
}
for _, tt := range tests {
t.Run(tt.path, func(t *testing.T) {
if got := requestEcosystem(tt.path); got != tt.want {
t.Errorf("requestEcosystem(%q) = %q, want %q", tt.path, got, tt.want)
}
})
}
}
func TestLoggerMiddlewareWritesAccessLog(t *testing.T) {
path := filepath.Join(t.TempDir(), "access.jsonl")
activityLog, err := accesslog.Open(path)
if err != nil {
t.Fatal(err)
}
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
s := &Server{logger: logger, accessLog: activityLog}
next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
})
handler := middleware.RequestID(RequestIDMiddleware(s.LoggerMiddleware(next)))
req := httptest.NewRequest(http.MethodGet, "/packages/example?token=secret", nil)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if err := activityLog.Close(); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var entry accesslog.Entry
if err := json.Unmarshal(data, &entry); err != nil {
t.Fatalf("decoding access log: %v", err)
}
if entry.Event != accesslog.EventRequest {
t.Errorf("event = %q, want %q", entry.Event, accesslog.EventRequest)
}
if entry.RequestID == "" {
t.Error("request_id is empty")
}
if entry.Path != "/packages/example" {
t.Errorf("path = %q, want query string omitted", entry.Path)
}
if entry.StatusCode != http.StatusNotFound {
t.Errorf("status_code = %d, want %d", entry.StatusCode, http.StatusNotFound)
}
}
func TestResponseWriter_WriteHeader(t *testing.T) { func TestResponseWriter_WriteHeader(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View file

@ -2,13 +2,10 @@ package server
import ( import (
"fmt" "fmt"
"net/http"
"net/url"
"strings" "strings"
"unicode" "unicode"
"github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/database"
"github.com/go-chi/chi/v5"
) )
// maxPackagePathLen bounds the wildcard portion of package routes (name plus // maxPackagePathLen bounds the wildcard portion of package routes (name plus
@ -16,69 +13,22 @@ import (
// longer, so 512 leaves room without admitting pathological inputs. // longer, so 512 leaves room without admitting pathological inputs.
const maxPackagePathLen = 512 const maxPackagePathLen = 512
// packagePathSegments validates the wildcard portion of a package route and
// splits it into decoded path segments.
func packagePathSegments(r *http.Request) ([]string, error) {
wildcard := chi.URLParam(r, "*")
encoded := wildcardIsEncoded(r)
if err := validatePackagePath(wildcard, encoded); err != nil {
return nil, err
}
return splitWildcardPath(wildcard, encoded), nil
}
// wildcardIsEncoded reports whether the chi wildcard for this request is still
// percent-encoded.
//
// chi routes on r.URL.RawPath when it is set and on r.URL.Path otherwise, and
// net/url only sets RawPath when the request's escaping differs from the
// canonical encoding of the decoded path. A version such as "release%2F1" is
// therefore routed raw, while "1.0%252B" (a version whose text contains a
// literal "%2B") encodes canonically and arrives already decoded once. The
// distinction decides whether the segments still need decoding: decoding the
// second case again would turn it into "1.0+" and resolve a different version.
func wildcardIsEncoded(r *http.Request) bool {
return r.URL.RawPath != ""
}
// validatePackagePath rejects wildcard package paths that cannot be valid in // validatePackagePath rejects wildcard package paths that cannot be valid in
// any supported ecosystem. It is a coarse filter applied before database or // any supported ecosystem. It is a coarse filter applied before database or
// enrichment lookups; ecosystem-specific name rules are layered on top. // enrichment lookups; ecosystem-specific name rules are layered on top.
// func validatePackagePath(path string) error {
// encoded has the meaning described on wildcardIsEncoded.
func validatePackagePath(path string, encoded bool) error {
if path == "" { if path == "" {
return fmt.Errorf("package name required") return fmt.Errorf("package name required")
} }
if len(path) > maxPackagePathLen { if len(path) > maxPackagePathLen {
return fmt.Errorf("package path exceeds %d bytes", maxPackagePathLen) return fmt.Errorf("package path exceeds %d bytes", maxPackagePathLen)
} }
// Validate the decoded segments: the handlers work with decoded values, so for _, r := range path {
// an escape such as "%00" or "%2E%2E" must not slip past these checks. if r == 0 {
for _, seg := range splitWildcardPath(path, encoded) { return fmt.Errorf("package path contains null byte")
// Each segment is checked both as the handlers see it and decoded once }
// more: a segment can reach a handler with escapes intact, and the if unicode.IsControl(r) {
// upstream registry is then the one that decodes them. return fmt.Errorf("package path contains control character %#U", r)
for _, value := range []string{seg, decodePathSegment(seg)} {
// A decoded segment can itself contain slashes (from "%2F"), and
// the segments are later rejoined into a package name that
// registries interpolate straight into an upstream URL. Check every
// path element, not just the segment as a whole, or
// "a%2F..%2F..%2Fb" traverses.
for _, elem := range strings.Split(value, "/") {
if elem == ".." {
return fmt.Errorf("package path contains parent directory segment")
}
}
for _, r := range value {
if r == 0 {
return fmt.Errorf("package path contains null byte")
}
if unicode.IsControl(r) {
return fmt.Errorf("package path contains control character %#U", r)
}
}
} }
} }
return nil return nil
@ -110,37 +60,10 @@ func resolvePackageName(db *database.DB, ecosystem string, segments []string) (n
// splitWildcardPath splits a chi wildcard path value into segments, // splitWildcardPath splits a chi wildcard path value into segments,
// trimming any leading/trailing slashes. // trimming any leading/trailing slashes.
// func splitWildcardPath(path string) []string {
// When encoded is set the value is still percent-encoded (see
// wildcardIsEncoded), so each segment is decoded after splitting. Splitting
// first keeps an encoded "%2F" inside a name from being mistaken for a
// separator. Decoding matters for versions such as "1.0%2Bbuild1", which must
// reach the handlers as "1.0+build1" so that rebuilding the PURL yields the
// value that was stored rather than a double-encoded one.
func splitWildcardPath(path string, encoded bool) []string {
path = strings.Trim(path, "/") path = strings.Trim(path, "/")
if path == "" { if path == "" {
return nil return nil
} }
segments := strings.Split(path, "/") return strings.Split(path, "/")
if !encoded {
return segments
}
for i, seg := range segments {
segments[i] = decodePathSegment(seg)
}
return segments
}
// decodePathSegment percent-decodes a single URL path segment, returning it
// unchanged if it is not valid percent-encoding.
func decodePathSegment(seg string) string {
if !strings.Contains(seg, "%") {
return seg
}
decoded, err := url.PathUnescape(seg)
if err != nil {
return seg
}
return decoded
} }

View file

@ -1,15 +1,12 @@
package server package server
import ( import (
"net/http"
"net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/database"
"github.com/go-chi/chi/v5"
) )
func newTestDB(t *testing.T) (*database.DB, func()) { func newTestDB(t *testing.T) (*database.DB, func()) {
@ -98,44 +95,26 @@ func TestResolvePackageName(t *testing.T) {
func TestSplitWildcardPath(t *testing.T) { func TestSplitWildcardPath(t *testing.T) {
tests := []struct { tests := []struct {
input string input string
encoded bool want []string
want []string
}{ }{
{"lodash", false, []string{"lodash"}}, {"lodash", []string{"lodash"}},
{"lodash/4.17.21", false, []string{"lodash", "4.17.21"}}, {"lodash/4.17.21", []string{"lodash", "4.17.21"}},
{"monolog/monolog", false, []string{"monolog", "monolog"}}, {"monolog/monolog", []string{"monolog", "monolog"}},
{"symfony/console/6.0.0/browse", false, []string{"symfony", "console", "6.0.0", "browse"}}, {"symfony/console/6.0.0/browse", []string{"symfony", "console", "6.0.0", "browse"}},
{"", false, nil}, {"", nil},
{"/", false, nil}, {"/", nil},
// chi routes on the raw path when it differs from the canonical
// encoding of the decoded path, so segments arrive percent-encoded and
// must be decoded.
{
"nmap/7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", true,
[]string{"nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"},
},
{"%40babel/core/7.0.0", true, []string{"@babel", "core", "7.0.0"}},
// An encoded separator stays inside its segment rather than splitting.
{"vendor%2Fname/1.0.0", true, []string{"vendor/name", "1.0.0"}},
// Invalid escapes are passed through untouched.
{"lodash/1.0%zz", true, []string{"lodash", "1.0%zz"}},
// When chi routed on the already-decoded path, an escape that survived
// is part of the value: a version whose text is "1.0%2B" reaches here
// as "1.0%2B" and decoding it again would yield "1.0+".
{"nmap/1.0%2B", false, []string{"nmap", "1.0%2B"}},
} }
for _, tt := range tests { for _, tt := range tests {
got := splitWildcardPath(tt.input, tt.encoded) got := splitWildcardPath(tt.input)
if len(got) != len(tt.want) { if len(got) != len(tt.want) {
t.Errorf("splitWildcardPath(%q, %v) = %v, want %v", tt.input, tt.encoded, got, tt.want) t.Errorf("splitWildcardPath(%q) = %v, want %v", tt.input, got, tt.want)
continue continue
} }
for i := range got { for i := range got {
if got[i] != tt.want[i] { if got[i] != tt.want[i] {
t.Errorf("splitWildcardPath(%q, %v)[%d] = %q, want %q", t.Errorf("splitWildcardPath(%q)[%d] = %q, want %q", tt.input, i, got[i], tt.want[i])
tt.input, tt.encoded, i, got[i], tt.want[i])
} }
} }
} }
@ -153,19 +132,8 @@ func TestValidatePackagePath(t *testing.T) {
{"composer namespaced", "symfony/console/6.0.0", false}, {"composer namespaced", "symfony/console/6.0.0", false},
{"maven coordinates", "org.apache.commons/commons-lang3/3.12.0", false}, {"maven coordinates", "org.apache.commons/commons-lang3/3.12.0", false},
{"unicode", "café/1.0.0", false}, {"unicode", "café/1.0.0", false},
{"encoded plus in version", "nmap/7.91%2Bdfsg1-2ubuntu0.1", false},
{"empty", "", true}, {"empty", "", true},
{"null byte", "lodash\x00/4.17.21", true}, {"null byte", "lodash\x00/4.17.21", true},
{"encoded null byte", "lodash/%00", true},
{"encoded newline", "lodash/1.0%0A", true},
{"parent segment", "lodash/../4.17.21", true},
{"encoded parent segment", "lodash/%2E%2E/4.17.21", true},
// A decoded segment can contain slashes, so traversal can hide inside
// one segment. Registries interpolate the resolved name straight into
// an upstream URL, and Go sends dot-segments verbatim.
{"traversal inside one segment", "pkg%2F..%2F..%2Fadmin", true},
{"traversal via encoded dots and slash", "pkg%2f%2e%2e%2fadmin", true},
{"encoded slash alone is allowed", "vendor%2Fname/1.0.0", false},
{"null byte suffix", "lodash\x00", true}, {"null byte suffix", "lodash\x00", true},
{"newline", "lodash\n4.17.21", true}, {"newline", "lodash\n4.17.21", true},
{"carriage return", "lodash\r", true}, {"carriage return", "lodash\r", true},
@ -177,64 +145,9 @@ func TestValidatePackagePath(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
// The verdict must not depend on whether chi routed on the raw or err := validatePackagePath(tt.path)
// on the already-decoded path: an escape that reaches a handler if (err != nil) != tt.wantErr {
// undecoded is decoded by the upstream registry instead, so it is t.Errorf("validatePackagePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
// rejected either way.
for _, encoded := range []bool{false, true} {
err := validatePackagePath(tt.path, encoded)
if (err != nil) != tt.wantErr {
t.Errorf("validatePackagePath(%q, %v) error = %v, wantErr %v",
tt.path, encoded, err, tt.wantErr)
}
}
})
}
}
// TestPackagePathSegments drives the real router, which is what decides whether
// the wildcard still carries percent-encoding. Go decodes the request path
// itself unless the escaping is non-canonical, so the same version can arrive
// either way and only one of the two forms may be decoded again.
func TestPackagePathSegments(t *testing.T) {
tests := []struct {
name string
target string
want []string
}{
{"plain", "/pkg/npm/lodash/4.17.21", []string{"lodash", "4.17.21"}},
{"encoded plus", "/pkg/deb/nmap/7.91%2Bdfsg1-2ubuntu0.1", []string{"nmap", "7.91+dfsg1-2ubuntu0.1"}},
{"decoded plus", "/pkg/deb/nmap/7.91+dfsg1-2ubuntu0.1", []string{"nmap", "7.91+dfsg1-2ubuntu0.1"}},
// An encoded slash is one segment, not a separator.
{"encoded slash", "/pkg/composer/vendor%2Fname/1.0.0", []string{"vendor/name", "1.0.0"}},
{"question mark", "/pkg/npm/example/v1%3Fbuild", []string{"example", "v1?build"}},
// "1.0%252B" is the escaped form of the version "1.0%2B"; net/url
// already decoded it once, so it must not be decoded again.
{"literal percent escape", "/pkg/npm/example/1.0%252B", []string{"example", "1.0%2B"}},
{"browse suffix", "/pkg/deb/nmap/7.91%2Bdfsg1/browse", []string{"nmap", "7.91+dfsg1", "browse"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var got []string
var gotErr error
router := chi.NewRouter()
router.Get("/pkg/{ecosystem}/*", func(_ http.ResponseWriter, r *http.Request) {
got, gotErr = packagePathSegments(r)
})
router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", tt.target, nil))
if gotErr != nil {
t.Fatalf("packagePathSegments(%q) failed: %v", tt.target, gotErr)
}
if len(got) != len(tt.want) {
t.Fatalf("segments for %q = %v, want %v", tt.target, got, tt.want)
}
for i := range got {
if got[i] != tt.want[i] {
t.Errorf("segments for %q [%d] = %q, want %q", tt.target, i, got[i], tt.want[i])
}
} }
}) })
} }

View file

@ -15,26 +15,16 @@
// - /conan/* - Conan C/C++ protocol // - /conan/* - Conan C/C++ protocol
// - /conda/* - Conda/Anaconda protocol // - /conda/* - Conda/Anaconda protocol
// - /cran/* - CRAN (R) protocol // - /cran/* - CRAN (R) protocol
// - /julia/* - Julia Pkg server protocol
// - /v2/* - OCI/Docker container registry protocol // - /v2/* - OCI/Docker container registry protocol
// - /debian/* - Debian/APT repository protocol // - /debian/* - Debian/APT repository protocol
// - /rpm/* - RPM/Yum repository protocol // - /rpm/* - RPM/Yum repository protocol
// //
// Additional endpoints: // Additional endpoints:
// - /health - Health check endpoint // - /health - Health check endpoint
// - /stats - Cache statistics (JSON) // - /stats - Cache statistics (JSON)
// - /openapi.json - OpenAPI spec (JSON) // - /openapi.json - OpenAPI spec (JSON)
// - /metrics - Prometheus metrics // - /packages - List all cached packages (HTML)
// // - /search - Search packages (HTML)
// Web UI (HTML), mounted under /ui so reverse proxies can gate it
// separately from the package endpoints:
// - /ui/ - Dashboard
// - /ui/install - Client configuration guide
// - /ui/packages - List all cached packages
// - /ui/search - Search packages
// - /ui/package/... - Package and version detail pages
// - /ui/api/browse/... - Archive browsing (used by the UI)
// - /ui/api/compare/... - Archive diffing (used by the UI)
// //
// API endpoints for enrichment data: // API endpoints for enrichment data:
// - GET /api/package/{ecosystem}/{name} - Package metadata // - GET /api/package/{ecosystem}/{name} - Package metadata
@ -50,29 +40,24 @@ import (
"context" "context"
"database/sql" "database/sql"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net/http" "net/http"
"net/url"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/git-pkgs/cooldown"
swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" swaggerdoc "github.com/git-pkgs/proxy/docs/swagger"
"github.com/git-pkgs/proxy/internal/accesslog"
"github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/config"
"github.com/git-pkgs/proxy/internal/cooldown"
"github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/proxy/internal/enrichment" "github.com/git-pkgs/proxy/internal/enrichment"
"github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/handler"
upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient"
"github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/metrics"
"github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/mirror"
"github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/proxy/internal/storage"
"github.com/git-pkgs/purl" "github.com/git-pkgs/purl"
"github.com/git-pkgs/registries/fetch" "github.com/git-pkgs/registries/fetch"
"github.com/git-pkgs/registries/safehttp"
"github.com/git-pkgs/spdx" "github.com/git-pkgs/spdx"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
@ -88,35 +73,17 @@ const (
// Server is the main proxy server. // Server is the main proxy server.
type Server struct { type Server struct {
cfg *config.Config cfg *config.Config
db *database.DB db *database.DB
storage storage.Storage storage storage.Storage
logger *slog.Logger logger *slog.Logger
buildInfo BuildInfo http *http.Server
http *http.Server templates *Templates
templates *Templates cancel context.CancelFunc
cancel context.CancelFunc
healthCache *healthCache
accessLog *accesslog.Logger
} }
// New creates a new Server with the given configuration. // New creates a new Server with the given configuration.
func New(cfg *config.Config, logger *slog.Logger, buildInfo BuildInfo) (*Server, error) { func New(cfg *config.Config, logger *slog.Logger) (*Server, error) {
var activityLog *accesslog.Logger
if cfg.AccessLog.Path != "" {
var err error
activityLog, err = accesslog.Open(cfg.AccessLog.Path)
if err != nil {
return nil, fmt.Errorf("initializing access log: %w", err)
}
}
closeAccessLog := true
defer func() {
if closeAccessLog && activityLog != nil {
_ = activityLog.Close()
}
}()
// Initialize database // Initialize database
var db *database.DB var db *database.DB
var err error var err error
@ -158,59 +125,30 @@ func New(cfg *config.Config, logger *slog.Logger, buildInfo BuildInfo) (*Server,
return nil, fmt.Errorf("verifying storage connectivity: %w", err) return nil, fmt.Errorf("verifying storage connectivity: %w", err)
} }
hc, err := newHealthCache(store, cfg.Health.StorageProbeInterval, logger) return &Server{
if err != nil { cfg: cfg,
_ = store.Close() db: db,
_ = db.Close() storage: store,
return nil, fmt.Errorf("initializing health cache: %w", err) logger: logger,
} templates: &Templates{},
}, nil
server := &Server{
cfg: cfg,
db: db,
storage: store,
logger: logger,
buildInfo: buildInfo,
templates: &Templates{},
healthCache: hc,
accessLog: activityLog,
}
closeAccessLog = false
return server, nil
} }
// Start starts the HTTP server. // Start starts the HTTP server.
func (s *Server) Start() error { func (s *Server) Start() error {
// Use one authentication-aware transport for metadata and artifacts so // Create shared components with circuit breaker
// configured credentials and cached OCI challenges apply consistently. baseFetcher := fetch.NewFetcher(fetch.WithAuthFunc(s.authForURL))
safeClient := safehttp.New(nil, safehttp.Options{})
baseTransport := safeClient.Transport
if s.accessLog != nil {
baseTransport = upstreamhttp.NewAccessLogTransport(baseTransport, s.accessLog, s.logger)
}
authTransport := upstreamhttp.NewTransport(baseTransport, upstreamhttp.AuthFunc(s.authForURL))
metadataClient := *safeClient
metadataClient.Timeout = s.cfg.ParseHTTPTimeout()
metadataClient.Transport = authTransport
artifactClient := metadataClient
artifactClient.Timeout = serverWriteTimeout
// Create shared components with circuit breaker.
baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(&artifactClient))
fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher) fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher)
resolver := fetch.NewResolver() resolver := fetch.NewResolver()
cd := &cooldown.Config{ cd := &cooldown.Config{
Default: s.cfg.Cooldown.Default, Default: s.cfg.Cooldown.Default,
Ecosystems: s.cfg.Cooldown.Ecosystems, Ecosystems: s.cfg.Cooldown.Ecosystems,
Packages: s.cfg.Cooldown.NormalizedPackages(), Packages: s.cfg.Cooldown.Packages,
} }
proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger)
proxy.HTTPClient = &metadataClient
proxy.AuthForURL = s.authForURL
proxy.Cooldown = cd proxy.Cooldown = cd
proxy.CacheMetadata = s.cfg.CacheMetadata proxy.CacheMetadata = s.cfg.CacheMetadata
proxy.MetadataTTL = s.cfg.ParseMetadataTTL() proxy.MetadataTTL = s.cfg.ParseMetadataTTL()
proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize()
proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly
proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize() proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize()
proxy.DirectServe = s.cfg.Storage.DirectServe proxy.DirectServe = s.cfg.Storage.DirectServe
@ -223,6 +161,7 @@ func (s *Server) Start() error {
// Add middleware // Add middleware
r.Use(middleware.RequestID) r.Use(middleware.RequestID)
r.Use(RequestIDMiddleware) r.Use(RequestIDMiddleware)
r.Use(middleware.RealIP)
r.Use(s.LoggerMiddleware) r.Use(s.LoggerMiddleware)
r.Use(middleware.Recoverer) r.Use(middleware.Recoverer)
r.Use(func(next http.Handler) http.Handler { r.Use(func(next http.Handler) http.Handler {
@ -236,34 +175,22 @@ func (s *Server) Start() error {
}) })
// Mount protocol handlers // Mount protocol handlers
npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM) npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL)
cargoHandler := handler.NewCargoHandler( cargoHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL)
proxy,
s.cfg.BaseURL,
s.cfg.Upstream.Cargo,
s.cfg.Upstream.CargoDownload,
)
gemHandler := handler.NewGemHandler(proxy, s.cfg.BaseURL) gemHandler := handler.NewGemHandler(proxy, s.cfg.BaseURL)
goHandler := handler.NewGoHandler(proxy, s.cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, s.cfg.BaseURL)
hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL) hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL)
pubHandler := handler.NewPubHandler(proxy, s.cfg.BaseURL) pubHandler := handler.NewPubHandler(proxy, s.cfg.BaseURL)
pypiHandler := handler.NewPyPIHandler(proxy, s.cfg.BaseURL) pypiHandler := handler.NewPyPIHandler(proxy, s.cfg.BaseURL)
mavenHandler := handler.NewMavenHandler( mavenHandler := handler.NewMavenHandler(proxy, s.cfg.BaseURL)
proxy,
s.cfg.BaseURL,
s.cfg.Upstream.Maven,
s.cfg.Upstream.GradlePluginPortal,
)
gradleHandler := handler.NewGradleBuildCacheHandler(proxy) gradleHandler := handler.NewGradleBuildCacheHandler(proxy)
nugetHandler := handler.NewNuGetHandler(proxy, s.cfg.BaseURL) nugetHandler := handler.NewNuGetHandler(proxy, s.cfg.BaseURL)
composerHandler := handler.NewComposerHandler(proxy, s.cfg.BaseURL) composerHandler := handler.NewComposerHandler(proxy, s.cfg.BaseURL)
conanHandler := handler.NewConanHandler(proxy, s.cfg.BaseURL) conanHandler := handler.NewConanHandler(proxy, s.cfg.BaseURL)
condaHandler := handler.NewCondaHandler(proxy, s.cfg.BaseURL) condaHandler := handler.NewCondaHandler(proxy, s.cfg.BaseURL)
cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL) cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL)
juliaHandler := handler.NewJuliaHandler(proxy, s.cfg.BaseURL) containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL)
containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.OCI) debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL)
helmHandler := handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm)
debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Debian)
rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL) rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL)
r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))
@ -280,35 +207,23 @@ func (s *Server) Start() error {
r.Mount("/conan", http.StripPrefix("/conan", conanHandler.Routes())) r.Mount("/conan", http.StripPrefix("/conan", conanHandler.Routes()))
r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes())) r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes()))
r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes())) r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes()))
r.Mount("/julia", http.StripPrefix("/julia", juliaHandler.Routes()))
r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes())) r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes()))
r.Mount("/helm", http.StripPrefix("/helm", helmHandler.Routes()))
r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes()))
r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes()))
// Health, stats, and metrics endpoints // Health, stats, and static endpoints
r.Get("/health", s.handleHealth) r.Get("/health", s.handleHealth)
r.Get("/stats", s.handleStats) r.Get("/stats", s.handleStats)
r.Get("/openapi.json", s.handleOpenAPIJSON) r.Get("/openapi.json", s.handleOpenAPIJSON)
r.Get("/metrics", func(w http.ResponseWriter, r *http.Request) { r.Get("/metrics", func(w http.ResponseWriter, r *http.Request) {
metrics.Handler().ServeHTTP(w, r) metrics.Handler().ServeHTTP(w, r)
}) })
r.Mount("/static", http.StripPrefix("/static/", staticHandler()))
// Web UI. Mounted under /ui so a reverse proxy can apply different r.Get("/", s.handleRoot)
// access rules to it than to the package endpoints above (#123). r.Get("/install", s.handleInstall)
r.Route("/ui", func(ui chi.Router) { r.Get("/search", s.handleSearch)
ui.Mount("/static", http.StripPrefix("/ui/static/", staticHandler())) r.Get("/packages", s.handlePackagesList)
ui.Get("/", s.handleRoot) r.Get("/package/{ecosystem}/*", s.handlePackagePath)
ui.Get("/install", s.handleInstall)
ui.Get("/search", s.handleSearch)
ui.Get("/packages", s.handlePackagesList)
ui.Get("/package/{ecosystem}/*", s.handlePackagePath)
ui.Get("/api/browse/{ecosystem}/*", s.handleBrowsePath)
ui.Get("/api/compare/{ecosystem}/*", s.handleComparePath)
})
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/ui/", http.StatusFound)
})
// API endpoints for enrichment data // API endpoints for enrichment data
enrichSvc := enrichment.New(s.logger) enrichSvc := enrichment.New(s.logger)
@ -321,6 +236,10 @@ func (s *Server) Start() error {
r.Get("/api/search", apiHandler.HandleSearch) r.Get("/api/search", apiHandler.HandleSearch)
r.Get("/api/packages", apiHandler.HandlePackagesList) r.Get("/api/packages", apiHandler.HandlePackagesList)
// Archive browsing and comparison endpoints also use wildcard for namespaced packages
r.Get("/api/browse/{ecosystem}/*", s.handleBrowsePath)
r.Get("/api/compare/{ecosystem}/*", s.handleComparePath)
// Start background context (used by mirror jobs and cleanup) // Start background context (used by mirror jobs and cleanup)
bgCtx, bgCancel := context.WithCancel(context.Background()) bgCtx, bgCancel := context.WithCancel(context.Background())
s.cancel = bgCancel s.cancel = bgCancel
@ -348,9 +267,8 @@ func (s *Server) Start() error {
s.logger.Info("starting server", s.logger.Info("starting server",
"listen", s.cfg.Listen, "listen", s.cfg.Listen,
"base_url", s.cfg.BaseURL, "base_url", s.cfg.BaseURL,
"ui_url", s.cfg.UIBaseURL,
"storage", s.storage.URL(), "storage", s.storage.URL(),
"database", s.cfg.Database.String()) "database", s.cfg.Database.Path)
go s.updateCacheStatsMetrics() go s.updateCacheStatsMetrics()
go s.startEvictionLoop(bgCtx) go s.startEvictionLoop(bgCtx)
@ -401,12 +319,6 @@ func (s *Server) Shutdown(ctx context.Context) error {
} }
} }
if s.accessLog != nil {
if err := s.accessLog.Close(); err != nil {
errs = append(errs, fmt.Errorf("access log close: %w", err))
}
}
if s.db != nil { if s.db != nil {
if err := s.db.Close(); err != nil { if err := s.db.Close(); err != nil {
errs = append(errs, fmt.Errorf("database close: %w", err)) errs = append(errs, fmt.Errorf("database close: %w", err))
@ -457,7 +369,6 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) {
// Build dashboard data // Build dashboard data
data := DashboardData{ data := DashboardData{
Layout: s.layoutFor(r),
Stats: DashboardStats{ Stats: DashboardStats{
CachedArtifacts: stats.TotalArtifacts, CachedArtifacts: stats.TotalArtifacts,
TotalSize: formatSize(stats.TotalSize), TotalSize: formatSize(stats.TotalSize),
@ -544,12 +455,8 @@ func (s *Server) handleOpenAPIJSON(w http.ResponseWriter, _ *http.Request) {
func (s *Server) handleInstall(w http.ResponseWriter, r *http.Request) { func (s *Server) handleInstall(w http.ResponseWriter, r *http.Request) {
data := struct { data := struct {
Layout
BaseURL string
Registries []RegistryConfig Registries []RegistryConfig
}{ }{
Layout: s.layoutFor(r),
BaseURL: s.cfg.BaseURL,
Registries: getRegistryConfigs(s.cfg.BaseURL), Registries: getRegistryConfigs(s.cfg.BaseURL),
} }
@ -563,7 +470,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
ecosystem := r.URL.Query().Get("ecosystem") ecosystem := r.URL.Query().Get("ecosystem")
if query == "" { if query == "" {
http.Redirect(w, r, "/ui/", http.StatusSeeOther) http.Redirect(w, r, "/", http.StatusSeeOther)
return return
} }
@ -612,7 +519,6 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
totalPages := int((total + int64(limit) - 1) / int64(limit)) totalPages := int((total + int64(limit) - 1) / int64(limit))
data := SearchPageData{ data := SearchPageData{
Layout: s.layoutFor(r),
Query: query, Query: query,
Ecosystem: ecosystem, Ecosystem: ecosystem,
Results: items, Results: items,
@ -688,7 +594,6 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) {
totalPages := int((total + int64(limit) - 1) / int64(limit)) totalPages := int((total + int64(limit) - 1) / int64(limit))
data := PackagesListPageData{ data := PackagesListPageData{
Layout: s.layoutFor(r),
Ecosystem: ecosystem, Ecosystem: ecosystem,
SortBy: sortBy, SortBy: sortBy,
Results: items, Results: items,
@ -715,11 +620,12 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) {
// {name}/compare/{v1}...{v2} -> compare versions // {name}/compare/{v1}...{v2} -> compare versions
func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) { func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem") ecosystem := chi.URLParam(r, "ecosystem")
segments, err := packagePathSegments(r) wildcard := chi.URLParam(r, "*")
if err != nil { if err := validatePackagePath(wildcard); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) == 0 { if ecosystem == "" || len(segments) == 0 {
http.Error(w, "ecosystem and package name required", http.StatusBadRequest) http.Error(w, "ecosystem and package name required", http.StatusBadRequest)
@ -731,7 +637,7 @@ func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) {
if seg == "compare" && i > 0 && i < len(segments)-1 { if seg == "compare" && i > 0 && i < len(segments)-1 {
name := strings.Join(segments[:i], "/") name := strings.Join(segments[:i], "/")
versions := strings.Join(segments[i+1:], "/") versions := strings.Join(segments[i+1:], "/")
s.showComparePage(w, r, ecosystem, name, versions) s.showComparePage(w, ecosystem, name, versions)
return return
} }
} }
@ -751,7 +657,7 @@ func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) {
// segment is a version (if present) and everything else is the name. // segment is a version (if present) and everything else is the name.
if len(segments) == 1 { if len(segments) == 1 {
// Single segment, no DB match: try package show (will 404). // Single segment, no DB match: try package show (will 404).
s.showPackage(w, r, ecosystem, segments[0]) s.showPackage(w, ecosystem, segments[0])
return return
} }
name = strings.Join(segments[:len(segments)-1], "/") name = strings.Join(segments[:len(segments)-1], "/")
@ -760,17 +666,17 @@ func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) {
switch { switch {
case len(rest) == 0 && !browse: case len(rest) == 0 && !browse:
s.showPackage(w, r, ecosystem, name) s.showPackage(w, ecosystem, name)
case len(rest) == 1 && browse: case len(rest) == 1 && browse:
s.showBrowseSource(w, r, ecosystem, name, rest[0]) s.showBrowseSource(w, ecosystem, name, rest[0])
case len(rest) == 1: case len(rest) == 1:
s.showVersion(w, r, ecosystem, name, rest[0]) s.showVersion(w, ecosystem, name, rest[0])
default: default:
http.Error(w, "not found", http.StatusNotFound) http.Error(w, "not found", http.StatusNotFound)
} }
} }
func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) { func (s *Server) showPackage(w http.ResponseWriter, ecosystem, name string) {
pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name) pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name)
if err != nil { if err != nil {
s.logger.Error("failed to get package", "error", err, "ecosystem", ecosystem, "name", name) s.logger.Error("failed to get package", "error", err, "ecosystem", ecosystem, "name", name)
@ -795,7 +701,6 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem,
} }
data := PackageShowData{ data := PackageShowData{
Layout: s.layoutFor(r),
Package: pkg, Package: pkg,
Versions: versions, Versions: versions,
Vulnerabilities: vulns, Vulnerabilities: vulns,
@ -807,7 +712,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem,
} }
} }
func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { func (s *Server) showVersion(w http.ResponseWriter, ecosystem, name, version string) {
pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name) pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name)
if err != nil || pkg == nil { if err != nil || pkg == nil {
s.logger.Error("failed to get package", "error", err) s.logger.Error("failed to get package", "error", err)
@ -837,12 +742,15 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem,
isOutdated := pkg.LatestVersion.Valid && pkg.LatestVersion.String != version isOutdated := pkg.LatestVersion.Valid && pkg.LatestVersion.String != version
// A version whose only cached artifact is a metadata sidecar cannot be hasCached := false
// browsed, so it must not be advertised as cached. for _, art := range artifacts {
hasCached := firstBrowsableArtifact(artifacts) != nil if art.StoragePath.Valid {
hasCached = true
break
}
}
data := VersionShowData{ data := VersionShowData{
Layout: s.layoutFor(r),
Package: pkg, Package: pkg,
Version: ver, Version: ver,
Artifacts: artifacts, Artifacts: artifacts,
@ -857,13 +765,11 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem,
} }
} }
func (s *Server) showBrowseSource(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { func (s *Server) showBrowseSource(w http.ResponseWriter, ecosystem, name, version string) {
data := BrowseSourceData{ data := BrowseSourceData{
Layout: s.layoutFor(r), Ecosystem: ecosystem,
Ecosystem: ecosystem, PackageName: name,
PackageName: name, Version: version,
Version: version,
EscapedVersion: url.PathEscape(version),
} }
if err := s.templates.Render(w, "browse_source", data); err != nil { if err := s.templates.Render(w, "browse_source", data); err != nil {
@ -872,7 +778,7 @@ func (s *Server) showBrowseSource(w http.ResponseWriter, r *http.Request, ecosys
} }
} }
func (s *Server) showComparePage(w http.ResponseWriter, r *http.Request, ecosystem, name, versions string) { func (s *Server) showComparePage(w http.ResponseWriter, ecosystem, name, versions string) {
const compareVersionParts = 2 const compareVersionParts = 2
parts := strings.Split(versions, "...") parts := strings.Split(versions, "...")
if len(parts) != compareVersionParts { if len(parts) != compareVersionParts {
@ -881,13 +787,10 @@ func (s *Server) showComparePage(w http.ResponseWriter, r *http.Request, ecosyst
} }
data := ComparePageData{ data := ComparePageData{
Layout: s.layoutFor(r), Ecosystem: ecosystem,
Ecosystem: ecosystem, PackageName: name,
PackageName: name, FromVersion: parts[0],
FromVersion: parts[0], ToVersion: parts[1],
ToVersion: parts[1],
EscapedFromVersion: url.PathEscape(parts[0]),
EscapedToVersion: url.PathEscape(parts[1]),
} }
if err := s.templates.Render(w, "compare_versions", data); err != nil { if err := s.templates.Render(w, "compare_versions", data); err != nil {
@ -896,49 +799,23 @@ func (s *Server) showComparePage(w http.ResponseWriter, r *http.Request, ecosyst
} }
} }
// handleHealth responds with a structured JSON health report. // handleHealth responds with a simple health check.
//
// @Summary Health check // @Summary Health check
// @Tags meta // @Tags meta
// @Produce json // @Produce plain
// @Success 200 {object} HealthResponse // @Success 200 {string} string
// @Failure 503 {object} HealthResponse // @Failure 503 {string} string
// @Router /health [get] // @Router /health [get]
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") // Check database connectivity
resp := HealthResponse{Status: "ok", Checks: map[string]HealthCheck{}}
// Database check (short-circuit; do not waste a storage probe call when DB is down).
// On DB failure the storage entry reports "skipped" rather than being omitted so
// the response always carries the same key set for monitors that expect it.
if _, err := s.db.SchemaVersion(); err != nil { if _, err := s.db.SchemaVersion(); err != nil {
resp.Status = "error"
resp.Checks["database"] = HealthCheck{Status: "error", Error: err.Error()}
resp.Checks["storage"] = HealthCheck{Status: "skipped"}
w.WriteHeader(http.StatusServiceUnavailable) w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(resp) _, _ = fmt.Fprint(w, "database error")
return return
} }
resp.Checks["database"] = HealthCheck{Status: "ok"}
// Storage probe (via cache).
if err := s.healthCache.Check(); err != nil {
resp.Status = "error"
sc := HealthCheck{Status: "error", Error: err.Error()}
var pe *probeError
if errors.As(err, &pe) {
sc.Step = pe.step
}
resp.Checks["storage"] = sc
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(resp)
return
}
resp.Checks["storage"] = HealthCheck{Status: "ok"}
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(resp) _, _ = fmt.Fprint(w, "ok")
} }
// StatsResponse contains cache statistics. // StatsResponse contains cache statistics.
@ -979,7 +856,7 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
TotalSize: size, TotalSize: size,
TotalSizeHuman: formatSize(size), TotalSizeHuman: formatSize(size),
StorageURL: s.storage.URL(), StorageURL: s.storage.URL(),
DatabasePath: s.cfg.Database.String(), DatabasePath: s.cfg.Database.Path,
} }
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")

View file

@ -1,20 +1,15 @@
package server package server
import ( import (
"context"
"database/sql" "database/sql"
"encoding/json" "encoding/json"
"fmt" "fmt"
"html"
"io" "io"
"log/slog" "log/slog"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -23,7 +18,6 @@ import (
"github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/database"
"github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/handler"
"github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/proxy/internal/storage"
"github.com/git-pkgs/purl"
"github.com/git-pkgs/registries/fetch" "github.com/git-pkgs/registries/fetch"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
) )
@ -52,7 +46,7 @@ func newTestServer(t *testing.T) *testServer {
t.Fatalf("failed to create database: %v", err) t.Fatalf("failed to create database: %v", err)
} }
store, err := storage.OpenBucket(context.Background(), "file://"+storagePath) store, err := storage.NewFilesystem(storagePath)
if err != nil { if err != nil {
_ = db.Close() _ = db.Close()
_ = os.RemoveAll(tempDir) _ = os.RemoveAll(tempDir)
@ -66,20 +60,15 @@ func newTestServer(t *testing.T) *testServer {
cfg := &config.Config{ cfg := &config.Config{
BaseURL: "http://localhost:8080", BaseURL: "http://localhost:8080",
Storage: config.StorageConfig{URL: "file://" + storagePath}, Storage: config.StorageConfig{Path: storagePath},
Database: config.DatabaseConfig{Path: dbPath}, Database: config.DatabaseConfig{Path: dbPath},
} }
r := chi.NewRouter() r := chi.NewRouter()
// Mount handlers // Mount handlers
npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL, cfg.Upstream.NPM) npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL)
cargoHandler := handler.NewCargoHandler( cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL)
proxy,
cfg.BaseURL,
cfg.Upstream.Cargo,
cfg.Upstream.CargoDownload,
)
gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL) gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL)
goHandler := handler.NewGoHandler(proxy, cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, cfg.BaseURL)
pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL) pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL)
@ -92,40 +81,26 @@ func newTestServer(t *testing.T) *testServer {
r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes())) r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes()))
r.Mount("/gradle", http.StripPrefix("/gradle", gradleHandler.Routes())) r.Mount("/gradle", http.StripPrefix("/gradle", gradleHandler.Routes()))
hc, err := newHealthCache(store, "30s", logger)
if err != nil {
_ = db.Close()
_ = os.RemoveAll(tempDir)
t.Fatalf("failed to create health cache: %v", err)
}
// Create a minimal server struct for the handlers // Create a minimal server struct for the handlers
s := &Server{ s := &Server{
cfg: cfg, cfg: cfg,
db: db, db: db,
storage: store, storage: store,
logger: logger, logger: logger,
buildInfo: BuildInfo{Version: "test-version", Commit: "test-commit"}, templates: &Templates{},
templates: &Templates{},
healthCache: hc,
} }
r.Get("/health", s.handleHealth) r.Get("/health", s.handleHealth)
r.Get("/stats", s.handleStats) r.Get("/stats", s.handleStats)
r.Get("/openapi.json", s.handleOpenAPIJSON) r.Get("/openapi.json", s.handleOpenAPIJSON)
r.Route("/ui", func(ui chi.Router) { r.Mount("/static", http.StripPrefix("/static/", staticHandler()))
ui.Mount("/static", http.StripPrefix("/ui/static/", staticHandler())) r.Get("/search", s.handleSearch)
ui.Get("/", s.handleRoot) r.Get("/package/{ecosystem}/*", s.handlePackagePath)
ui.Get("/install", s.handleInstall) r.Get("/api/browse/{ecosystem}/*", s.handleBrowsePath)
ui.Get("/search", s.handleSearch) r.Get("/api/compare/{ecosystem}/*", s.handleComparePath)
ui.Get("/packages", s.handlePackagesList) r.Get("/", s.handleRoot)
ui.Get("/package/{ecosystem}/*", s.handlePackagePath) r.Get("/install", s.handleInstall)
ui.Get("/api/browse/{ecosystem}/*", s.handleBrowsePath) r.Get("/packages", s.handlePackagesList)
ui.Get("/api/compare/{ecosystem}/*", s.handleComparePath)
})
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/ui/", http.StatusFound)
})
return &testServer{ return &testServer{
handler: r, handler: r,
@ -204,55 +179,12 @@ func TestHealthEndpoint(t *testing.T) {
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusOK { if w.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) t.Errorf("expected status 200, got %d", w.Code)
} }
if got := w.Header().Get("Content-Type"); got != "application/json" {
t.Errorf("Content-Type = %q, want application/json", got)
}
var resp HealthResponse
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
t.Fatalf("decoding response: %v", err)
}
if resp.Status != "ok" {
t.Errorf("status = %q, want ok", resp.Status)
}
if resp.Checks["database"].Status != "ok" {
t.Errorf("database check = %+v, want ok", resp.Checks["database"])
}
if resp.Checks["storage"].Status != "ok" {
t.Errorf("storage check = %+v, want ok", resp.Checks["storage"])
}
}
func TestHealthEndpoint_DBFailureShortCircuits(t *testing.T) { body := w.Body.String()
ts := newTestServer(t) if body != "ok" {
defer ts.close() t.Errorf("expected body 'ok', got %q", body)
// Force DB failure by closing the connection.
_ = ts.db.Close()
req := httptest.NewRequest("GET", "/health", nil)
w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want 503; body: %s", w.Code, w.Body.String())
}
var resp HealthResponse
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
t.Fatalf("decoding: %v", err)
}
if resp.Status != "error" {
t.Errorf("status = %q, want error", resp.Status)
}
if resp.Checks["database"].Status != "error" {
t.Errorf("database check = %+v, want error", resp.Checks["database"])
}
storage, present := resp.Checks["storage"]
if !present {
t.Error("storage key should be present (with status=skipped) on DB short-circuit")
} else if storage.Status != "skipped" {
t.Errorf("storage check = %+v, want status=skipped", storage)
} }
} }
@ -291,7 +223,7 @@ func TestDashboard(t *testing.T) {
ts := newTestServer(t) ts := newTestServer(t)
defer ts.close() defer ts.close()
req := httptest.NewRequest("GET", "/ui/", nil) req := httptest.NewRequest("GET", "/", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -315,9 +247,6 @@ func TestDashboard(t *testing.T) {
if !strings.Contains(body, "Cached Artifacts") { if !strings.Contains(body, "Cached Artifacts") {
t.Error("dashboard should contain stats") t.Error("dashboard should contain stats")
} }
if !strings.Contains(body, "proxy test-version (test-commit)") {
t.Error("dashboard footer should contain build information")
}
if !strings.Contains(body, "Popular Packages") { if !strings.Contains(body, "Popular Packages") {
t.Error("dashboard should contain popular packages section") t.Error("dashboard should contain popular packages section")
} }
@ -465,9 +394,8 @@ func TestStaticFiles(t *testing.T) {
path string path string
contentTypes []string contentTypes []string
}{ }{
{"/ui/static/vendor/tailwind.js", []string{"text/javascript", "application/javascript"}}, {"/static/tailwind.js", []string{"text/javascript", "application/javascript"}},
{"/ui/static/vendor/lucide.min.js", []string{"text/javascript", "application/javascript"}}, {"/static/style.css", []string{"text/css"}},
{"/ui/static/style.css", []string{"text/css"}},
} }
for _, tc := range tests { for _, tc := range tests {
@ -518,27 +446,11 @@ func TestCategorizeLicenseCSS(t *testing.T) {
} }
} }
func TestRootRedirectsToUI(t *testing.T) {
ts := newTestServer(t)
defer ts.close()
req := httptest.NewRequest("GET", "/", nil)
w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusFound {
t.Errorf("expected status 302, got %d", w.Code)
}
if loc := w.Header().Get("Location"); loc != "/ui/" {
t.Errorf("expected redirect to /ui/, got %q", loc)
}
}
func TestDashboardWithEnrichmentStats(t *testing.T) { func TestDashboardWithEnrichmentStats(t *testing.T) {
ts := newTestServer(t) ts := newTestServer(t)
defer ts.close() defer ts.close()
req := httptest.NewRequest("GET", "/ui/", nil) req := httptest.NewRequest("GET", "/", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -549,7 +461,7 @@ func TestDashboardWithEnrichmentStats(t *testing.T) {
body := w.Body.String() body := w.Body.String()
// Dashboard should link to Tailwind JS // Dashboard should link to Tailwind JS
if !strings.Contains(body, "/ui/static/vendor/tailwind.js") { if !strings.Contains(body, "/static/tailwind.js") {
t.Error("dashboard should link to Tailwind JS") t.Error("dashboard should link to Tailwind JS")
} }
@ -590,7 +502,7 @@ func TestVersionShowWithHitCount(t *testing.T) {
t.Fatalf("failed to upsert artifact: %v", err) t.Fatalf("failed to upsert artifact: %v", err)
} }
req := httptest.NewRequest("GET", "/ui/package/npm/test/1.0.0", nil) req := httptest.NewRequest("GET", "/package/npm/test/1.0.0", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -603,9 +515,6 @@ func TestVersionShowWithHitCount(t *testing.T) {
if !strings.Contains(body, "42 cache hits") { if !strings.Contains(body, "42 cache hits") {
t.Error("expected page to show hit count") t.Error("expected page to show hit count")
} }
if !strings.Contains(body, "proxy test-version (test-commit)") {
t.Error("version show footer should contain proxy build information, not the package version")
}
} }
func TestSearchWithNullValues(t *testing.T) { func TestSearchWithNullValues(t *testing.T) {
@ -645,7 +554,7 @@ func TestSearchWithNullValues(t *testing.T) {
t.Fatalf("failed to upsert artifact: %v", err) t.Fatalf("failed to upsert artifact: %v", err)
} }
req := httptest.NewRequest("GET", "/ui/search?q=test", nil) req := httptest.NewRequest("GET", "/search?q=test", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -737,7 +646,7 @@ func TestSearchRedirectsWhenEmpty(t *testing.T) {
ts := newTestServer(t) ts := newTestServer(t)
defer ts.close() defer ts.close()
req := httptest.NewRequest("GET", "/ui/search", nil) req := httptest.NewRequest("GET", "/search", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -746,8 +655,8 @@ func TestSearchRedirectsWhenEmpty(t *testing.T) {
} }
loc := w.Header().Get("Location") loc := w.Header().Get("Location")
if loc != "/ui/" { if loc != "/" {
t.Errorf("expected redirect to /ui/, got %q", loc) t.Errorf("expected redirect to /, got %q", loc)
} }
} }
@ -755,7 +664,7 @@ func TestPackageShowPage_NotFoundServer(t *testing.T) {
ts := newTestServer(t) ts := newTestServer(t)
defer ts.close() defer ts.close()
req := httptest.NewRequest("GET", "/ui/package/npm/nonexistent-srv", nil) req := httptest.NewRequest("GET", "/package/npm/nonexistent-srv", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -768,7 +677,7 @@ func TestVersionShowPage_NotFoundServer(t *testing.T) {
ts := newTestServer(t) ts := newTestServer(t)
defer ts.close() defer ts.close()
req := httptest.NewRequest("GET", "/ui/package/npm/nonexistent-srv/1.0.0", nil) req := httptest.NewRequest("GET", "/package/npm/nonexistent-srv/1.0.0", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -777,236 +686,6 @@ func TestVersionShowPage_NotFoundServer(t *testing.T) {
} }
} }
// TestVersionShowPage_PlusInVersion covers Debian/Ubuntu style versions such as
// nmap's "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1". PURL percent-encodes "+" as
// "%2B", so the UI must show the decoded version and resolve both the decoded
// and the still-encoded form of the URL back to the same version.
func TestVersionShowPage_PlusInVersion(t *testing.T) {
ts := newTestServer(t)
defer ts.close()
const version = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"
const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1"
pkg := &database.Package{PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap"}
if err := ts.db.UpsertPackage(pkg); err != nil {
t.Fatalf("failed to upsert package: %v", err)
}
if err := ts.db.UpsertVersion(&database.Version{
PURL: versionPURL, PackagePURL: pkg.PURL,
}); err != nil {
t.Fatalf("failed to upsert version: %v", err)
}
// The package page must link to and display the decoded version.
req := httptest.NewRequest("GET", "/ui/package/deb/nmap", nil)
w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("package page: expected status 200, got %d", w.Code)
}
body := w.Body.String()
if strings.Contains(body, "%2B") {
t.Error("package page leaks PURL percent-encoding into the UI")
}
// html/template renders "+" as the "&#43;" entity inside attributes and text.
if !strings.Contains(body, "7.91&#43;dfsg1&#43;really7.80&#43;dfsg1-2ubuntu0.1") {
t.Error("expected package page to show the decoded version")
}
// Both the decoded and the encoded URL must reach the version page.
for _, path := range []string{
"/ui/package/deb/nmap/" + version,
"/ui/package/deb/nmap/7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1",
} {
req := httptest.NewRequest("GET", path, nil)
w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Errorf("GET %s: expected status 200, got %d", path, w.Code)
}
}
}
// TestVersionURLEscaping covers versions whose characters are significant in a
// URL path: "/" splits off another path segment, "?" starts a query string, and
// a literal "%xx" is read back as the character it encodes. The pages show the
// decoded version but must build every link from a separately escaped value,
// and those links have to resolve back to the same version.
func TestVersionURLEscaping(t *testing.T) {
// A second version is needed for the compare controls to be rendered.
const otherVersion = "1.0.0"
tests := []struct {
name string
version string
}{
{"slash", "release/1"},
{"question mark", "v1?build"},
{"literal percent escape", "1.0%2B"},
{"plus", "7.91+dfsg1-2ubuntu0.1"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ts := newTestServer(t)
defer ts.close()
seedEscapingVersions(t, ts.db, tt.version, otherVersion)
// The package page links to the escaped version.
escaped := url.PathEscape(tt.version)
versionPath := "/ui/package/deb/nmap/" + escaped
packagePage := ts.getOK(t, "/ui/package/deb/nmap")
if !containsValue(attrValues(packagePage, "href"), versionPath) {
t.Fatalf("package page has no link to %q; hrefs: %v",
versionPath, attrValues(packagePage, "href"))
}
ts.checkVersionAndBrowsePages(t, versionPath, tt.version, escaped)
ts.checkComparePage(t, packagePage, tt.version, escaped, otherVersion)
})
}
}
// seedEscapingVersions stores a Debian package with the given versions, each
// with a cached artifact so that the version page offers its browse link.
func seedEscapingVersions(t *testing.T, db *database.DB, versions ...string) {
t.Helper()
pkg := &database.Package{PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap"}
if err := db.UpsertPackage(pkg); err != nil {
t.Fatalf("failed to upsert package: %v", err)
}
for _, v := range versions {
versionPURL := purl.MakePURLString("deb", "nmap", v)
if err := db.UpsertVersion(&database.Version{
PURL: versionPURL, PackagePURL: pkg.PURL,
}); err != nil {
t.Fatalf("failed to upsert version %q: %v", v, err)
}
if err := db.UpsertArtifact(&database.Artifact{
VersionPURL: versionPURL,
Filename: "nmap.deb",
UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb",
StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
}); err != nil {
t.Fatalf("failed to upsert artifact for %q: %v", v, err)
}
}
}
// checkVersionAndBrowsePages follows a version link from the package page and
// then the browse link from the version page, checking that both resolve to the
// stored version and display it decoded.
func (ts *testServer) checkVersionAndBrowsePages(t *testing.T, versionPath, version, escaped string) {
t.Helper()
versionPage := ts.getOK(t, versionPath)
wantPURL := "pkg:deb/nmap@" + version
if !strings.Contains(html.UnescapeString(versionPage), wantPURL) {
t.Errorf("version page does not show %q", wantPURL)
}
browsePath := versionPath + "/browse"
if !containsValue(attrValues(versionPage, "href"), browsePath) {
t.Fatalf("version page has no browse link to %q; hrefs: %v",
browsePath, attrValues(versionPage, "href"))
}
browsePage := ts.getOK(t, browsePath)
if !strings.Contains(html.UnescapeString(browsePage), "nmap@"+version) {
t.Errorf("browse page does not show the decoded version %q", version)
}
// The browse API is called with the escaped version, not with the text shown
// in the heading.
if got := jsConstant(t, browsePage, "versionPath"); got != escaped {
t.Errorf("browse page passes %q to the browse API, want %q", got, escaped)
}
}
// checkComparePage builds the compare URL the way the package page's script
// does, from the values its checkboxes carry, and checks the page it reaches.
func (ts *testServer) checkComparePage(t *testing.T, packagePage, version, escaped, otherVersion string) {
t.Helper()
selectable := attrValues(packagePage, "data-version-path")
if !containsValue(selectable, escaped) {
t.Fatalf("package page compare data holds %v, want %q", selectable, escaped)
}
comparePage := ts.getOK(t, "/ui/package/deb/nmap/compare/"+escaped+"..."+otherVersion)
decoded := html.UnescapeString(comparePage)
for _, want := range []string{version, otherVersion} {
if !strings.Contains(decoded, want) {
t.Errorf("compare page does not show version %q", want)
}
}
if got := jsConstant(t, comparePage, "fromVersionPath"); got != escaped {
t.Errorf("compare page passes %q to the compare API, want %q", got, escaped)
}
}
// getOK performs a GET against the server and fails the test unless it returns
// 200, returning the response body.
func (ts *testServer) getOK(t *testing.T, path string) string {
t.Helper()
req := httptest.NewRequest("GET", path, nil)
w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("GET %s: expected status 200, got %d", path, w.Code)
}
return w.Body.String()
}
// attrValues returns the value of every occurrence of an HTML attribute in a
// rendered page, with HTML entities resolved so that values can be compared
// against the raw strings they were built from.
func attrValues(body, attr string) []string {
re := regexp.MustCompile(regexp.QuoteMeta(attr) + `="([^"]*)"`)
var values []string
for _, match := range re.FindAllStringSubmatch(body, -1) {
values = append(values, html.UnescapeString(match[1]))
}
return values
}
// jsConstant returns the value of a single-quoted JavaScript string constant in
// a rendered page. html/template escapes characters that are significant in
// JavaScript, rendering "+" as "\\u002b" for instance, so the escapes are
// resolved to recover the value the page actually uses.
func jsConstant(t *testing.T, body, name string) string {
t.Helper()
re := regexp.MustCompile(`const ` + regexp.QuoteMeta(name) + ` = '([^']*)'`)
match := re.FindStringSubmatch(body)
if match == nil {
t.Fatalf("page does not declare the constant %q", name)
}
unescaped, err := strconv.Unquote(`"` + match[1] + `"`)
if err != nil {
t.Fatalf("cannot unescape %q: %v", match[1], err)
}
return unescaped
}
func containsValue(values []string, want string) bool {
for _, v := range values {
if v == want {
return true
}
}
return false
}
func TestPackageShowPage_WithLicense(t *testing.T) { func TestPackageShowPage_WithLicense(t *testing.T) {
ts := newTestServer(t) ts := newTestServer(t)
defer ts.close() defer ts.close()
@ -1029,7 +708,7 @@ func TestPackageShowPage_WithLicense(t *testing.T) {
t.Fatalf("failed to upsert version: %v", err) t.Fatalf("failed to upsert version: %v", err)
} }
req := httptest.NewRequest("GET", "/ui/package/npm/show-test-lic", nil) req := httptest.NewRequest("GET", "/package/npm/show-test-lic", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -1071,8 +750,8 @@ func TestComposerNamespacedPackageRoutes(t *testing.T) {
url string url string
want string want string
}{ }{
{"package show", "/ui/package/composer/monolog/monolog", "monolog/monolog"}, {"package show", "/package/composer/monolog/monolog", "monolog/monolog"},
{"version show", "/ui/package/composer/symfony/console/6.0.0", "symfony/console"}, {"version show", "/package/composer/symfony/console/6.0.0", "symfony/console"},
} }
for _, tt := range tests { for _, tt := range tests {
@ -1129,11 +808,11 @@ func TestNamespacedPackageRoutes(t *testing.T) {
url string url string
want int want int
}{ }{
{"npm scoped package show", "/ui/package/npm/@babel/core", http.StatusOK}, {"npm scoped package show", "/package/npm/@babel/core", http.StatusOK},
{"golang module show", "/ui/package/golang/github.com/stretchr/testify", http.StatusOK}, {"golang module show", "/package/golang/github.com/stretchr/testify", http.StatusOK},
{"oci image show", "/ui/package/oci/library/nginx", http.StatusOK}, {"oci image show", "/package/oci/library/nginx", http.StatusOK},
{"conda package show", "/ui/package/conda/conda-forge/numpy", http.StatusOK}, {"conda package show", "/package/conda/conda-forge/numpy", http.StatusOK},
{"conan package show", "/ui/package/conan/zlib/1.2.13@demo/stable", http.StatusOK}, {"conan package show", "/package/conan/zlib/1.2.13@demo/stable", http.StatusOK},
} }
for _, tt := range tests { for _, tt := range tests {
@ -1156,7 +835,7 @@ func TestSearchPage_WithSeededResults(t *testing.T) {
seedTestPackage(t, ts.db, "searchable-pkg") seedTestPackage(t, ts.db, "searchable-pkg")
req := httptest.NewRequest("GET", "/ui/search?q=searchable", nil) req := httptest.NewRequest("GET", "/search?q=searchable", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -1204,7 +883,7 @@ func TestSearchPage_PaginationMultiPage(t *testing.T) {
} }
// First page // First page
req := httptest.NewRequest("GET", "/ui/search?q=page-test", nil) req := httptest.NewRequest("GET", "/search?q=page-test", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -1218,7 +897,7 @@ func TestSearchPage_PaginationMultiPage(t *testing.T) {
} }
// Second page // Second page
req = httptest.NewRequest("GET", "/ui/search?q=page-test&page=2", nil) req = httptest.NewRequest("GET", "/search?q=page-test&page=2", nil)
w = httptest.NewRecorder() w = httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -1284,7 +963,7 @@ func TestSearchPage_EcosystemFilterWithSeededData(t *testing.T) {
} }
// Search with ecosystem filter for npm only // Search with ecosystem filter for npm only
req := httptest.NewRequest("GET", "/ui/search?q=eco-filter&ecosystem=npm", nil) req := httptest.NewRequest("GET", "/search?q=eco-filter&ecosystem=npm", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -1307,7 +986,7 @@ func TestHandlePackagesListPage(t *testing.T) {
seedTestPackage(t, ts.db, "list-test") seedTestPackage(t, ts.db, "list-test")
req := httptest.NewRequest("GET", "/ui/packages", nil) req := httptest.NewRequest("GET", "/packages", nil)
w := httptest.NewRecorder() w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req) ts.handler.ServeHTTP(w, req)
@ -1335,14 +1014,10 @@ func TestNewServer_StorageConnectivityCheck(t *testing.T) {
logger := slog.New(slog.NewTextHandler(io.Discard, nil)) logger := slog.New(slog.NewTextHandler(io.Discard, nil))
buildInfo := BuildInfo{Version: "test-version", Commit: "test-commit"} srv, err := New(cfg, logger)
srv, err := New(cfg, logger, buildInfo)
if err != nil { if err != nil {
t.Fatalf("New() failed: %v", err) t.Fatalf("New() failed: %v", err)
} }
if srv.buildInfo != buildInfo {
t.Errorf("build info = %#v, want %#v", srv.buildInfo, buildInfo)
}
// On Windows, OpenBucket normalises to file:///C:/path; on Unix the // On Windows, OpenBucket normalises to file:///C:/path; on Unix the
// absolute path already starts with /, so file:// + /path == file:///path. // absolute path already starts with /, so file:// + /path == file:///path.
@ -1356,27 +1031,6 @@ func TestNewServer_StorageConnectivityCheck(t *testing.T) {
_ = srv.db.Close() _ = srv.db.Close()
} }
func TestNewServer_InvalidAccessLogFailsBeforeDatabaseInit(t *testing.T) {
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
cfg := &config.Config{
Storage: config.StorageConfig{URL: "file://" + filepath.Join(tempDir, "artifacts")},
Database: config.DatabaseConfig{Path: dbPath},
AccessLog: config.AccessLogConfig{Path: filepath.Join(tempDir, "missing", "access.jsonl")},
}
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
if _, err := New(cfg, logger, BuildInfo{}); err == nil {
t.Fatal("New() succeeded with invalid access log path")
} else if !strings.Contains(err.Error(), "initializing access log") {
t.Fatalf("New() error = %v, want access log initialization error", err)
}
if _, err := os.Stat(dbPath); !os.IsNotExist(err) {
t.Errorf("database initialized before access log validation: %v", err)
}
}
func TestStatsEndpoint_StorageURL(t *testing.T) { func TestStatsEndpoint_StorageURL(t *testing.T) {
ts := newTestServer(t) ts := newTestServer(t)
defer ts.close() defer ts.close()

Some files were not shown because too many files have changed in this diff Show more