Merge pull request #188 from git-pkgs/http-timeout-config

Make upstream HTTP timeout configurable

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
           persist-credentials: false
 
       - name: Set up Go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
         with:
           go-version: ${{ matrix.go-version }}
 
@@ -40,7 +40,7 @@ jobs:
           persist-credentials: false
 
       - name: Set up Go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
         with:
           go-version: '1.25'
 

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
 
       - name: Set up Go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
         with:
           go-version-file: go.mod
           cache: false

.github/workflows/swagger.yml

@@ -17,7 +17,7 @@ jobs:
           persist-credentials: false
 
       - name: Set up Go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
         with:
           go-version: '1.25'
 

.github/workflows/zizmor.yml

@@ -26,4 +26,4 @@ jobs:
           persist-credentials: false
 
       - name: Run zizmor
-        uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
+        uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7

config.example.yaml

@@ -9,6 +9,11 @@ listen: ":8080"
 # so users know what to point their package manager at.
 base_url: "http://localhost:8080"
 
+# Timeout for individual upstream HTTP requests made by protocol handlers
+# (metadata fetches, pass-through file requests). Uses Go duration syntax.
+# Set to "0" to disable the timeout. Default: "30s".
+# http_timeout: "30s"
+
 # Public URL where the web UI is reached. Defaults to base_url when unset.
 # Set this separately when the UI is served on a different hostname than the
 # package endpoints — for example, the UI on a public domain behind auth while

docs/configuration.md

@@ -276,6 +276,18 @@ metadata_max_size: "100MB"   # default
 
 Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`.
 
+## Upstream HTTP timeout
+
+Protocol handlers use a shared HTTP client for upstream requests such as metadata fetches and pass-through file downloads. `http_timeout` sets that client's per-request timeout. Raise it if slow upstreams or large metadata responses cause `context deadline exceeded` errors.
+
+```yaml
+http_timeout: "30s"   # default
+```
+
+Or via environment variable: `PROXY_HTTP_TIMEOUT=2m`.
+
+Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout).
+
 ## Mirror API
 
 The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP:

go.mod

@@ -291,14 +291,14 @@ require (
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.51.0 // indirect
+	golang.org/x/crypto v0.53.0 // indirect
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
 	golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect
 	golang.org/x/mod v0.36.0 // indirect
-	golang.org/x/net v0.54.0 // indirect
+	golang.org/x/net v0.56.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
-	golang.org/x/text v0.37.0 // indirect
+	golang.org/x/text v0.38.0 // indirect
 	golang.org/x/tools v0.45.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.272.0 // indirect

go.sum

@@ -738,8 +738,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
+golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
-golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
+golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
 golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
@@ -771,8 +771,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
+golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
-golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
+golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -807,8 +807,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -823,8 +823,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
-golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
+golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

internal/config/config.go

@@ -110,6 +110,12 @@ type Config struct {
 	// size return ErrMetadataTooLarge. Default: "100MB".
 	MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
 
+	// HTTPTimeout is the timeout for individual upstream HTTP requests made
+	// by protocol handlers (metadata fetches, pass-through file requests).
+	// Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s".
+	// Set to "0" to disable the timeout entirely.
+	HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"`
+
 	// MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP.
 	// Disabled by default to prevent unauthenticated users from triggering downloads.
 	MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
@@ -460,6 +466,9 @@ func (c *Config) LoadFromEnv() {
 	if v := os.Getenv("PROXY_METADATA_MAX_SIZE"); v != "" {
 		c.MetadataMaxSize = v
 	}
+	if v := os.Getenv("PROXY_HTTP_TIMEOUT"); v != "" {
+		c.HTTPTimeout = v
+	}
 	if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY"); v != "" {
 		c.Gradle.BuildCache.ReadOnly = v == "true" || v == "1"
 	}
@@ -567,6 +576,10 @@ func (c *Config) Validate() error {
 		return err
 	}
 
+	if err := validateHTTPTimeout(c.HTTPTimeout); err != nil {
+		return err
+	}
+
 	if err := c.Health.Validate(); err != nil {
 		return err
 	}
@@ -636,6 +649,7 @@ func (g *GradleBuildCacheConfig) Validate() error {
 const (
 	defaultMetadataTTL                   = 5 * time.Minute  //nolint:mnd // sensible default
 	defaultDirectServeTTL                = 15 * time.Minute //nolint:mnd // sensible default
+	defaultHTTPTimeout                   = 30 * time.Second //nolint:mnd // sensible default
 	defaultMetadataMaxSize               = 100 << 20
 	defaultGradleBuildCacheMaxUploadSize = 100 << 20
 	defaultGradleBuildCacheSweepInterval = 10 * time.Minute
@@ -656,6 +670,20 @@ func (c *Config) ParseMaxSize() int64 {
 	return size
 }
 
+func validateHTTPTimeout(s string) error {
+	if s == "" || s == "0" {
+		return nil
+	}
+	d, err := time.ParseDuration(s)
+	if err != nil {
+		return fmt.Errorf("invalid http_timeout %q: %w", s, err)
+	}
+	if d < 0 {
+		return fmt.Errorf("invalid http_timeout %q: must be non-negative", s)
+	}
+	return nil
+}
+
 func validateMetadataMaxSize(s string) error {
 	if s == "" {
 		return nil
@@ -683,6 +711,22 @@ func (c *Config) ParseMetadataMaxSize() int64 {
 	return size
 }
 
+// ParseHTTPTimeout returns the upstream HTTP client timeout.
+// Returns 30s if unset, 0 (no timeout) if explicitly set to "0".
+func (c *Config) ParseHTTPTimeout() time.Duration {
+	if c.HTTPTimeout == "" {
+		return defaultHTTPTimeout
+	}
+	if c.HTTPTimeout == "0" {
+		return 0
+	}
+	d, err := time.ParseDuration(c.HTTPTimeout)
+	if err != nil || d < 0 {
+		return defaultHTTPTimeout
+	}
+	return d
+}
+
 // ParseMetadataTTL returns the metadata TTL duration.
 // Returns 5 minutes if unset, 0 if explicitly disabled.
 func (c *Config) ParseMetadataTTL() time.Duration {

internal/config/config_test.go

@@ -524,6 +524,69 @@ func TestValidateHealthStorageProbeInterval(t *testing.T) {
 	}
 }
 
+func TestParseHTTPTimeout(t *testing.T) {
+	tests := []struct {
+		name    string
+		timeout string
+		want    time.Duration
+	}{
+		{"empty defaults to 30s", "", 30 * time.Second},
+		{"explicit zero disables", "0", 0},
+		{"2 minutes", "2m", 2 * time.Minute},
+		{"90 seconds", "90s", 90 * time.Second},
+		{"invalid defaults to 30s", "not-a-duration", 30 * time.Second},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := Default()
+			cfg.HTTPTimeout = tt.timeout
+			got := cfg.ParseHTTPTimeout()
+			if got != tt.want {
+				t.Errorf("ParseHTTPTimeout() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestValidateHTTPTimeout(t *testing.T) {
+	cfg := Default()
+	cfg.HTTPTimeout = "not-a-duration"
+	if err := cfg.Validate(); err == nil {
+		t.Error("expected validation error for invalid http_timeout")
+	}
+
+	cfg.HTTPTimeout = "-5s"
+	if err := cfg.Validate(); err == nil {
+		t.Error("expected validation error for negative http_timeout")
+	}
+
+	cfg.HTTPTimeout = "2m"
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("unexpected error for valid http_timeout: %v", err)
+	}
+
+	cfg.HTTPTimeout = "0"
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("unexpected error for zero http_timeout: %v", err)
+	}
+
+	cfg.HTTPTimeout = ""
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("unexpected error for empty http_timeout: %v", err)
+	}
+}
+
+func TestLoadHTTPTimeoutFromEnv(t *testing.T) {
+	cfg := Default()
+	t.Setenv("PROXY_HTTP_TIMEOUT", "90s")
+	cfg.LoadFromEnv()
+
+	if cfg.HTTPTimeout != "90s" {
+		t.Errorf("HTTPTimeout = %q, want %q", cfg.HTTPTimeout, "90s")
+	}
+}
+
 func TestLoadMetadataTTLFromEnv(t *testing.T) {
 	cfg := Default()
 	t.Setenv("PROXY_METADATA_TTL", "10m")

internal/handler/composer.go

@@ -316,6 +316,10 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request)
 	// stable and dev versions resolve correctly.
 	metaURLs := h.metadataURLsForVersion(vendor, pkg, version)
 
+	h.proxy.Logger.Debug("resolving download URL",
+		"package", packageName, "version", version,
+		"metadata_urls", metaURLs)
+
 	var downloadURL string
 	for _, metaURL := range metaURLs {
 		url, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version)
@@ -331,10 +335,17 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request)
 	}
 
 	if downloadURL == "" {
+		h.proxy.Logger.Debug("version not found in any metadata source",
+			"package", packageName, "version", version,
+			"tried_urls", metaURLs)
 		http.Error(w, "version not found", http.StatusNotFound)
 		return
 	}
 
+	h.proxy.Logger.Debug("resolved download URL",
+		"package", packageName, "version", version,
+		"download_url", downloadURL)
+
 	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL)
 	if err != nil {
 		h.proxy.Logger.Error("failed to get artifact", "error", err)
@@ -372,6 +383,9 @@ func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []
 // An error is returned only on transport failure; a missing document (non-200)
 // or a missing version both yield an empty string so the caller can fall back.
 func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, error) {
+	h.proxy.Logger.Debug("fetching upstream metadata for download lookup",
+		"url", metaURL, "package", packageName, "version", version)
+
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil)
 	if err != nil {
 		return "", err
@@ -383,6 +397,9 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU
 	}
 	defer func() { _ = resp.Body.Close() }()
 
+	h.proxy.Logger.Debug("upstream metadata response",
+		"url", metaURL, "status", resp.StatusCode)
+
 	if resp.StatusCode != http.StatusOK {
 		return "", nil
 	}
@@ -392,7 +409,27 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU
 		return "", err
 	}
 
-	return h.findDownloadURL(metadata, packageName, version), nil
+	// Expand minified Composer v2 format so that inherited fields (including
+	// dist) are present on every version entry. Without this, versions that
+	// inherit dist from a previous entry will appear to have no download URL.
+	if metadata["minified"] == "composer/2.0" {
+		h.proxy.Logger.Debug("expanding minified metadata", "url", metaURL)
+		if packages, ok := metadata["packages"].(map[string]any); ok {
+			for pkgName, versions := range packages {
+				versionList, ok := versions.([]any)
+				if !ok {
+					continue
+				}
+				packages[pkgName] = expandMinifiedVersions(versionList)
+			}
+		}
+	}
+
+	url := h.findDownloadURL(metadata, packageName, version)
+	h.proxy.Logger.Debug("download URL lookup result",
+		"url", metaURL, "package", packageName, "version", version,
+		"download_url", url)
+	return url, nil
 }
 
 // findDownloadURL finds the dist URL for a specific version in metadata.

internal/server/server.go

@@ -166,6 +166,7 @@ func (s *Server) Start() error {
 		Packages:   s.cfg.Cooldown.Packages,
 	}
 	proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger)
+	proxy.HTTPClient.Timeout = s.cfg.ParseHTTPTimeout()
 	proxy.Cooldown = cd
 	proxy.CacheMetadata = s.cfg.CacheMetadata
 	proxy.MetadataTTL = s.cfg.ParseMetadataTTL()