pkg-proxy/internal/handler/generic.go

package handler

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/git-pkgs/proxy/internal/config"
	"github.com/git-pkgs/proxy/internal/retention"
)

const (
	genericEcosystem = "generic"
	// genericAcceptAny is always sent for metadata so every client shares the
	// same cached representation, regardless of its Accept header.
	genericAcceptAny = "*/*"
	// githubReleaseAssetMatchCount is the full match plus owner, repository,
	// tag and asset filename.
	githubReleaseAssetMatchCount = 5
)

// githubReleaseAssetPattern matches the path of a GitHub release asset
// download, {owner}/{repo}/releases/download/{tag}/{asset}. A tag pins the
// asset to one release, so these downloads are cached in the artifact cache
// and served without revalidation once fetched.
var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/download/([^/]+)/([^/]+)$`)

func init() {
	retention.Default.Register(retention.Spec{Key: genericEcosystem, CanonicalPackage: retention.DefaultCanonical(genericEcosystem)})
}

// builtinGenericArtifacts maps well-known download hosts to patterns for
// their version-pinned paths. They apply to any generic upstream configured
// with that base URL, after the upstream's own generic_artifacts.
var builtinGenericArtifacts = map[string][]string{
	// dist/{project dirs}/{version}/[binaries/]{file}, e.g. maven/maven-3 or maven/mvnd.
	"https://archive.apache.org": {
		`dist/(?P<name>.+?)/(?P<version>[0-9][^/]*)/(?:binaries/)?(?P<file>[^/]+)`,
	},
	"https://nodejs.org": {
		`dist/(?P<version>v[0-9][^/]*)/(?P<file>[^/]+)`,
	},
	// The version is only in the filename: go1.23.2.linux-amd64.tar.gz.
	"https://dl.google.com/go": {
		`(?P<file>go(?P<version>[0-9][^/]*?)\.[a-z0-9]+-[a-z0-9]+\.(?:tar\.gz|zip)(?:\.sha256)?)`,
	},
}

// GenericHandler proxies plain HTTP downloads from configured upstream base
// URLs. Each configured upstream is mounted at /generic/{name}/ and the
// remaining request path (and query string) is appended to the upstream URL.
//
// Only configured upstreams are reachable, so the proxy is not an open HTTP
// proxy. The handler is the caching layer behind tools that download from
// fixed URL shapes, such as mise's aqua backend fetching GitHub release
// assets, and is pointed at by URL-rewriting settings on the client.
//
// Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}), and
// paths matching the upstream's built-in or configured artifact patterns, are
// version-pinned and cached in the shared artifact cache, so they keep being
// served when the upstream is unreachable. Every other path is served through
// the metadata cache: fresh within the metadata TTL, revalidated with the
// upstream's validators after that, and served stale when the upstream fails
// or refuses the request. That covers API responses such as
// api.github.com/repos/{owner}/{repo}/releases/tags/{tag}.
type GenericHandler struct {
	proxy            *Proxy
	repositories     map[string]string
	artifactPatterns map[string][]*regexp.Regexp
	// paths holds each upstream's upstream.generic_paths patterns; an
	// upstream without any serves every path.
	paths map[string][]string
}

// NewGenericHandler creates a generic HTTP download proxy handler.
// artifactPatterns maps upstream names to upstream.generic_artifacts patterns;
// a pattern that does not compile is logged and skipped.
func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPatterns ...map[string][]string) *GenericHandler {
	h := &GenericHandler{
		proxy:            proxy,
		repositories:     make(map[string]string, len(repositories)),
		artifactPatterns: make(map[string][]*regexp.Regexp),
	}
	for name, upstreamURL := range repositories {
		h.repositories[name] = strings.TrimSuffix(upstreamURL, "/")
	}
	// The first matching pattern wins, so configured patterns go first.
	if len(artifactPatterns) > 0 {
		for name, patterns := range artifactPatterns[0] {
			h.addArtifactPatterns(name, patterns)
		}
	}
	for name, upstreamURL := range h.repositories {
		h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL])
	}
	return h
}

// WithPaths limits upstreams to the paths their upstream.generic_paths
// patterns match. Any other path answers 404.
func (h *GenericHandler) WithPaths(paths map[string][]string) *GenericHandler {
	h.paths = paths
	return h
}

func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) {
	for _, pattern := range patterns {
		re, err := config.CompileGenericArtifactPattern(pattern)
		if err != nil {
			h.proxy.Logger.Warn("skipping generic artifact pattern",
				"repository", name, "pattern", pattern, "error", err)
			continue
		}
		h.artifactPatterns[name] = append(h.artifactPatterns[name], re)
	}
}

// Routes returns the HTTP handler for generic download requests.
// Mount this at /generic on your router.
func (h *GenericHandler) Routes() http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodGet && r.Method != http.MethodHead {
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
			return
		}

		path := strings.TrimPrefix(r.URL.Path, "/")

		if containsPathTraversal(path) {
			http.Error(w, "invalid path", http.StatusBadRequest)
			return
		}

		repository, rest, ok := strings.Cut(path, "/")
		if repository == HostedGenericRepository && h.proxy.HostedEnabled {
			h.handleHosted(w, r, rest)
			return
		}
		upstreamURL, found := h.repositories[repository]
		if !ok || rest == "" || !found || !config.GenericPathAllowed(h.paths[repository], rest) {
			http.NotFound(w, r)
			return
		}

		if artifact, ok := h.parseArtifact(repository, rest); ok {
			h.handleArtifact(w, r, repository, upstreamURL, rest, artifact)
			return
		}

		h.handleMetadata(w, r, repository, upstreamURL, rest)
	})
}

// HostedGenericRepository is the /generic/{name}/ path that serves pushed
// generic files instead of an upstream while hosted packages are enabled.
const HostedGenericRepository = config.HostedGenericRepository

// handleHosted serves a pushed file at {name}/{version}/{file}. Pushed files
// never come from an upstream, so anything not pushed is a 404.
func (h *GenericHandler) handleHosted(w http.ResponseWriter, r *http.Request, path string) {
	segments := strings.Split(path, "/")
	if len(segments) != hostedGenericPathSegments {
		http.NotFound(w, r)
		return
	}
	target, err := HostedTargetFor(genericEcosystem, segments[0], segments[1], segments[2])
	if err != nil {
		http.NotFound(w, r)
		return
	}
	result, err := h.proxy.GetLocalArtifact(r.Context(), genericEcosystem, target.Name, target.Version, target.Filename)
	if err != nil {
		h.proxy.serveArtifactError(w, err, "failed to read pushed file")
		return
	}
	if result == nil {
		http.NotFound(w, r)
		return
	}
	serveGenericArtifact(w, r, result)
}

// hostedGenericPathSegments is {name}/{version}/{file}.
const hostedGenericPathSegments = 3

// genericArtifact is the identity of a version-pinned download.
type genericArtifact struct {
	name     string
	version  string
	filename string
}

// parseArtifact returns the identity of a version-pinned download path: one
// matching the upstream's artifact patterns, else a GitHub release asset.
func (h *GenericHandler) parseArtifact(repository, path string) (genericArtifact, bool) {
	for _, pattern := range h.artifactPatterns[repository] {
		if artifact, ok := matchArtifactPattern(pattern, repository, path); ok {
			return artifact, true
		}
	}
	if asset, ok := parseGitHubReleaseAsset(path); ok {
		return genericArtifact{
			name:     asset.owner + "/" + asset.repo,
			version:  asset.tag,
			filename: asset.filename,
		}, true
	}
	return genericArtifact{}, false
}

// matchArtifactPattern extracts the artifact identity from a path matched by
// pattern. The version and file must be single, non-empty path segments; the
// name defaults to the upstream name.
func matchArtifactPattern(pattern *regexp.Regexp, repository, path string) (genericArtifact, bool) {
	matches := pattern.FindStringSubmatch(path)
	if matches == nil {
		return genericArtifact{}, false
	}
	artifact := genericArtifact{
		name:     repository,
		version:  matches[pattern.SubexpIndex("version")],
		filename: matches[pattern.SubexpIndex("file")],
	}
	if i := pattern.SubexpIndex("name"); i >= 0 && matches[i] != "" {
		artifact.name = matches[i]
	}
	if artifact.version == "" || artifact.filename == "" ||
		strings.Contains(artifact.version, "/") || strings.Contains(artifact.filename, "/") {
		return genericArtifact{}, false
	}
	return artifact, true
}

// githubReleaseAsset is the identity of a version-pinned release download.
type githubReleaseAsset struct {
	owner    string
	repo     string
	tag      string
	filename string
}

// parseGitHubReleaseAsset extracts the release identity from a path shaped
// like {owner}/{repo}/releases/download/{tag}/{asset}.
func parseGitHubReleaseAsset(path string) (githubReleaseAsset, bool) {
	matches := githubReleaseAssetPattern.FindStringSubmatch(path)
	if len(matches) != githubReleaseAssetMatchCount {
		return githubReleaseAsset{}, false
	}
	return githubReleaseAsset{
		owner:    matches[1],
		repo:     matches[2],
		tag:      matches[3],
		filename: matches[4],
	}, true
}

// handleArtifact fetches and caches a version-pinned download in the artifact
// cache. The upstream, full path and query identify the cached bytes, since
// distinct downloads can have the same captured name, version and filename.
func (h *GenericHandler) handleArtifact(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, artifact genericArtifact) {
	downloadURL := upstreamURL + "/" + path
	if r.URL.RawQuery != "" {
		downloadURL += "?" + r.URL.RawQuery
	}
	cacheFilename := repository + "/" + h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery) + "/" + artifact.filename

	h.proxy.Logger.Info("generic artifact download",
		"repository", repository, "name", artifact.name, "version", artifact.version, "filename", artifact.filename)

	result, err := h.legacyArtifact(r.Context(), repository, downloadURL, artifact)
	if err == nil && result == nil {
		result, err = h.proxy.GetOrFetchArtifactFromURL(
			r.Context(), genericEcosystem, artifact.name, artifact.version, cacheFilename, downloadURL)
	}
	if err != nil {
		h.proxy.serveArtifactError(w, err, "failed to fetch artifact")
		return
	}
	serveGenericArtifact(w, r, result)
}

// serveGenericArtifact serves a cached or pushed file, as a plain download
// when its media type is unknown.
func serveGenericArtifact(w http.ResponseWriter, r *http.Request, result *CacheResult) {
	if result.Artifact.MediaType == "" {
		result.Artifact.MediaType = "application/octet-stream"
	}
	ServeArtifactRequest(w, r, result)
}

// legacyArtifact returns the artifact cached under the unhashed
// {repository}/{filename} key, provided its recorded upstream URL is
// downloadURL, so a different path or query sharing that key is not served.
func (h *GenericHandler) legacyArtifact(ctx context.Context, repository, downloadURL string, artifact genericArtifact) (*CacheResult, error) {
	pkgPURL, versionPURL, err := packagePURLStrings(genericEcosystem, artifact.name, artifact.version)
	if err != nil {
		return nil, err
	}
	filename := repository + "/" + artifact.filename
	record, err := h.proxy.DB.GetArtifact(versionPURL, filename)
	if err != nil {
		return nil, fmt.Errorf("checking artifact cache: %w", err)
	}
	if record == nil || record.UpstreamURL != downloadURL {
		return nil, nil
	}
	return h.proxy.checkCache(ctx, pkgPURL, versionPURL, filename)
}

// handleMetadata serves any other path through the metadata cache. The query
// string is forwarded and is part of the cache identity. A fixed Accept header
// keeps all clients on one cached representation.
func (h *GenericHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) {
	target := upstreamURL + "/" + path
	if r.URL.RawQuery != "" {
		target += "?" + r.URL.RawQuery
	}

	h.proxy.ProxyCached(w, r, target, genericEcosystem,
		h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery), genericAcceptAny)
}

// cacheKey derives the cache key from the upstream name, its
// URL, the request path and query. Hashing the identity keeps distinct
// upstreams from sharing entries and drops cached entries when an upstream is
// repointed, mirroring APKHandler.metadataCacheKey.
func (h *GenericHandler) cacheKey(repository, upstreamURL, path, query string) string {
	identity := repository + "\x00" + upstreamURL + "\x00" + path + "\x00" + query
	digest := sha256.Sum256([]byte(identity))
	return hex.EncodeToString(digest[:])
}