package handler import ( "context" "crypto/sha256" "encoding/hex" "fmt" "net/http" "regexp" "strings" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/retention" ) const ( genericEcosystem = "generic" // genericAcceptAny is always sent for metadata so every client shares the // same cached representation, regardless of its Accept header. genericAcceptAny = "*/*" // githubReleaseAssetMatchCount is the full match plus owner, repository, // tag and asset filename. githubReleaseAssetMatchCount = 5 ) // githubReleaseAssetPattern matches the path of a GitHub release asset // download, {owner}/{repo}/releases/download/{tag}/{asset}. A tag pins the // asset to one release, so these downloads are cached in the artifact cache // and served without revalidation once fetched. var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/download/([^/]+)/([^/]+)$`) func init() { retention.Default.Register(retention.Spec{Key: genericEcosystem, CanonicalPackage: retention.DefaultCanonical(genericEcosystem)}) } // builtinGenericArtifacts maps well-known download hosts to patterns for // their version-pinned paths. They apply to any generic upstream configured // with that base URL, after the upstream's own generic_artifacts. var builtinGenericArtifacts = map[string][]string{ // dist/{project dirs}/{version}/[binaries/]{file}, e.g. maven/maven-3 or maven/mvnd. "https://archive.apache.org": { `dist/(?P.+?)/(?P[0-9][^/]*)/(?:binaries/)?(?P[^/]+)`, }, "https://nodejs.org": { `dist/(?Pv[0-9][^/]*)/(?P[^/]+)`, }, // The version is only in the filename: go1.23.2.linux-amd64.tar.gz. "https://dl.google.com/go": { `(?Pgo(?P[0-9][^/]*?)\.[a-z0-9]+-[a-z0-9]+\.(?:tar\.gz|zip)(?:\.sha256)?)`, }, } // GenericHandler proxies plain HTTP downloads from configured upstream base // URLs. Each configured upstream is mounted at /generic/{name}/ and the // remaining request path (and query string) is appended to the upstream URL. // // Only configured upstreams are reachable, so the proxy is not an open HTTP // proxy. The handler is the caching layer behind tools that download from // fixed URL shapes, such as mise's aqua backend fetching GitHub release // assets, and is pointed at by URL-rewriting settings on the client. // // Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}), and // paths matching the upstream's built-in or configured artifact patterns, are // version-pinned and cached in the shared artifact cache, so they keep being // served when the upstream is unreachable. Every other path is served through // the metadata cache: fresh within the metadata TTL, revalidated with the // upstream's validators after that, and served stale when the upstream fails // or refuses the request. That covers API responses such as // api.github.com/repos/{owner}/{repo}/releases/tags/{tag}. type GenericHandler struct { proxy *Proxy repositories map[string]string artifactPatterns map[string][]*regexp.Regexp // paths holds each upstream's upstream.generic_paths patterns; an // upstream without any serves every path. paths map[string][]string } // NewGenericHandler creates a generic HTTP download proxy handler. // artifactPatterns maps upstream names to upstream.generic_artifacts patterns; // a pattern that does not compile is logged and skipped. func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPatterns ...map[string][]string) *GenericHandler { h := &GenericHandler{ proxy: proxy, repositories: make(map[string]string, len(repositories)), artifactPatterns: make(map[string][]*regexp.Regexp), } for name, upstreamURL := range repositories { h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") } // The first matching pattern wins, so configured patterns go first. if len(artifactPatterns) > 0 { for name, patterns := range artifactPatterns[0] { h.addArtifactPatterns(name, patterns) } } for name, upstreamURL := range h.repositories { h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL]) } return h } // WithPaths limits upstreams to the paths their upstream.generic_paths // patterns match. Any other path answers 404. func (h *GenericHandler) WithPaths(paths map[string][]string) *GenericHandler { h.paths = paths return h } func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) { for _, pattern := range patterns { re, err := config.CompileGenericArtifactPattern(pattern) if err != nil { h.proxy.Logger.Warn("skipping generic artifact pattern", "repository", name, "pattern", pattern, "error", err) continue } h.artifactPatterns[name] = append(h.artifactPatterns[name], re) } } // Routes returns the HTTP handler for generic download requests. // Mount this at /generic on your router. func (h *GenericHandler) Routes() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet && r.Method != http.MethodHead { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } path := strings.TrimPrefix(r.URL.Path, "/") if containsPathTraversal(path) { http.Error(w, "invalid path", http.StatusBadRequest) return } repository, rest, ok := strings.Cut(path, "/") if repository == HostedGenericRepository && h.proxy.HostedEnabled { h.handleHosted(w, r, rest) return } upstreamURL, found := h.repositories[repository] if !ok || rest == "" || !found || !config.GenericPathAllowed(h.paths[repository], rest) { http.NotFound(w, r) return } if artifact, ok := h.parseArtifact(repository, rest); ok { h.handleArtifact(w, r, repository, upstreamURL, rest, artifact) return } h.handleMetadata(w, r, repository, upstreamURL, rest) }) } // HostedGenericRepository is the /generic/{name}/ path that serves pushed // generic files instead of an upstream while hosted packages are enabled. const HostedGenericRepository = config.HostedGenericRepository // handleHosted serves a pushed file at {name}/{version}/{file}. Pushed files // never come from an upstream, so anything not pushed is a 404. func (h *GenericHandler) handleHosted(w http.ResponseWriter, r *http.Request, path string) { segments := strings.Split(path, "/") if len(segments) != hostedGenericPathSegments { http.NotFound(w, r) return } target, err := HostedTargetFor(genericEcosystem, segments[0], segments[1], segments[2]) if err != nil { http.NotFound(w, r) return } result, err := h.proxy.GetLocalArtifact(r.Context(), genericEcosystem, target.Name, target.Version, target.Filename) if err != nil { h.proxy.serveArtifactError(w, err, "failed to read pushed file") return } if result == nil { http.NotFound(w, r) return } serveGenericArtifact(w, r, result) } // hostedGenericPathSegments is {name}/{version}/{file}. const hostedGenericPathSegments = 3 // genericArtifact is the identity of a version-pinned download. type genericArtifact struct { name string version string filename string } // parseArtifact returns the identity of a version-pinned download path: one // matching the upstream's artifact patterns, else a GitHub release asset. func (h *GenericHandler) parseArtifact(repository, path string) (genericArtifact, bool) { for _, pattern := range h.artifactPatterns[repository] { if artifact, ok := matchArtifactPattern(pattern, repository, path); ok { return artifact, true } } if asset, ok := parseGitHubReleaseAsset(path); ok { return genericArtifact{ name: asset.owner + "/" + asset.repo, version: asset.tag, filename: asset.filename, }, true } return genericArtifact{}, false } // matchArtifactPattern extracts the artifact identity from a path matched by // pattern. The version and file must be single, non-empty path segments; the // name defaults to the upstream name. func matchArtifactPattern(pattern *regexp.Regexp, repository, path string) (genericArtifact, bool) { matches := pattern.FindStringSubmatch(path) if matches == nil { return genericArtifact{}, false } artifact := genericArtifact{ name: repository, version: matches[pattern.SubexpIndex("version")], filename: matches[pattern.SubexpIndex("file")], } if i := pattern.SubexpIndex("name"); i >= 0 && matches[i] != "" { artifact.name = matches[i] } if artifact.version == "" || artifact.filename == "" || strings.Contains(artifact.version, "/") || strings.Contains(artifact.filename, "/") { return genericArtifact{}, false } return artifact, true } // githubReleaseAsset is the identity of a version-pinned release download. type githubReleaseAsset struct { owner string repo string tag string filename string } // parseGitHubReleaseAsset extracts the release identity from a path shaped // like {owner}/{repo}/releases/download/{tag}/{asset}. func parseGitHubReleaseAsset(path string) (githubReleaseAsset, bool) { matches := githubReleaseAssetPattern.FindStringSubmatch(path) if len(matches) != githubReleaseAssetMatchCount { return githubReleaseAsset{}, false } return githubReleaseAsset{ owner: matches[1], repo: matches[2], tag: matches[3], filename: matches[4], }, true } // handleArtifact fetches and caches a version-pinned download in the artifact // cache. The upstream, full path and query identify the cached bytes, since // distinct downloads can have the same captured name, version and filename. func (h *GenericHandler) handleArtifact(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, artifact genericArtifact) { downloadURL := upstreamURL + "/" + path if r.URL.RawQuery != "" { downloadURL += "?" + r.URL.RawQuery } cacheFilename := repository + "/" + h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery) + "/" + artifact.filename h.proxy.Logger.Info("generic artifact download", "repository", repository, "name", artifact.name, "version", artifact.version, "filename", artifact.filename) result, err := h.legacyArtifact(r.Context(), repository, downloadURL, artifact) if err == nil && result == nil { result, err = h.proxy.GetOrFetchArtifactFromURL( r.Context(), genericEcosystem, artifact.name, artifact.version, cacheFilename, downloadURL) } if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } serveGenericArtifact(w, r, result) } // serveGenericArtifact serves a cached or pushed file, as a plain download // when its media type is unknown. func serveGenericArtifact(w http.ResponseWriter, r *http.Request, result *CacheResult) { if result.Artifact.MediaType == "" { result.Artifact.MediaType = "application/octet-stream" } ServeArtifactRequest(w, r, result) } // legacyArtifact returns the artifact cached under the unhashed // {repository}/{filename} key, provided its recorded upstream URL is // downloadURL, so a different path or query sharing that key is not served. func (h *GenericHandler) legacyArtifact(ctx context.Context, repository, downloadURL string, artifact genericArtifact) (*CacheResult, error) { pkgPURL, versionPURL, err := packagePURLStrings(genericEcosystem, artifact.name, artifact.version) if err != nil { return nil, err } filename := repository + "/" + artifact.filename record, err := h.proxy.DB.GetArtifact(versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) } if record == nil || record.UpstreamURL != downloadURL { return nil, nil } return h.proxy.checkCache(ctx, pkgPURL, versionPURL, filename) } // handleMetadata serves any other path through the metadata cache. The query // string is forwarded and is part of the cache identity. A fixed Accept header // keeps all clients on one cached representation. func (h *GenericHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { target := upstreamURL + "/" + path if r.URL.RawQuery != "" { target += "?" + r.URL.RawQuery } h.proxy.ProxyCached(w, r, target, genericEcosystem, h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery), genericAcceptAny) } // cacheKey derives the cache key from the upstream name, its // URL, the request path and query. Hashing the identity keeps distinct // upstreams from sharing entries and drops cached entries when an upstream is // repointed, mirroring APKHandler.metadataCacheKey. func (h *GenericHandler) cacheKey(repository, upstreamURL, path, query string) string { identity := repository + "\x00" + upstreamURL + "\x00" + path + "\x00" + query digest := sha256.Sum256([]byte(identity)) return hex.EncodeToString(digest[:]) }