package server import ( "fmt" "sort" "strconv" "time" "github.com/git-pkgs/proxy/internal/metrics" ) // RuntimeView holds the process-lifetime figures read out of the Prometheus // registry. Unlike the cache figures, which are computed from the database and // survive a restart, everything here starts at zero when the proxy starts. type RuntimeView struct { Available bool Requests string ActiveRequests string RequestMean string StatusClasses []LabelledCount // Artifact and metadata lookups are counted apart: they are separate // caches. Only artifacts feed the database figures above. ArtifactCache CacheLookups MetadataCache CacheLookups MetadataServed string UpstreamFetches string UpstreamFetchMean string UpstreamErrors []LabelledCount StorageOps []LabelledStat StorageErrors []LabelledCount IntegrityFailures []LabelledCount ProbeFailures []LabelledCount Evictions []LabelledCount MetadataEvictions []LabelledCount ChangeFeedExpired []LabelledCount Breakers []BreakerRow BreakerTrips []LabelledCount Scans []LabelledStat ScansBlocked []LabelledCount ScanErrors []LabelledCount ScanningOn bool ResponseBytes string Clients []ClientRow Sources []SourceRow SourceCount int TrustsForward bool // SourcesOn gates the caller table. The page is unauthenticated, so the // addresses behind it are published only when asked for. SourcesOn bool // FailureDetailsOn gates the recent failures listed under each failure // row, for the same reason as SourcesOn. FailureDetailsOn bool // HasFailures is set when any failure counter has fired, which is when the // note about turning failure details on is worth showing. HasFailures bool } // CacheLookups is one cache's hits and misses since the process started. type CacheLookups struct { Hits string Misses string HitRatio string HasTraffic bool } // cacheLookups reads one kind's share of the hit and miss counters. func cacheLookups(hits, misses map[string]float64, kind string) CacheLookups { h, m := hits[kind], misses[kind] c := CacheLookups{ Hits: formatCount(int64(h)), Misses: formatCount(int64(m)), HasTraffic: h+m > 0, } if c.HasTraffic { c.HitRatio = strconv.FormatFloat(h/(h+m)*100, 'f', 1, 64) //nolint:mnd // percent } return c } // ClientRow is one client tool's share of requests and bytes. type ClientRow struct { Client string Requests string Bytes string } // LabelledCount is a single counter series rendered as a row. type LabelledCount struct { Label string Count string // Bad marks a failure row, which the template tints. Bad bool // Recent lists the latest failures behind the count, newest first. Recent []FailureRow // series is the counter series the row was rendered from, for looking up // what was recorded against it. series map[string]string } // FailureRow is one recorded failure behind a failure counter. type FailureRow struct { Age string PURL string Filename string URL string Path string Error string } // LabelledStat is a histogram series rendered as a row: how many observations, // and their mean. type LabelledStat struct { Label string Count string Mean string } // BreakerRow is one upstream's circuit breaker state. type BreakerRow struct { Registry string State string Open bool } // runtimeView shapes a registry snapshot for the analytics page. With // failureDetails set, each failure row carries the latest failures behind it. func runtimeView(snap *metrics.Snapshot, failureDetails bool) RuntimeView { if snap == nil { return RuntimeView{} } v := RuntimeView{Available: true, FailureDetailsOn: failureDetails} failures := func(name string, labels ...string) []LabelledCount { return failureRows(snap, name, failureDetails, labels...) } requests := snap.Sum("proxy_requests_total") v.Requests = formatCount(int64(requests)) v.ActiveRequests = formatCount(int64(snap.Sum("proxy_active_requests"))) v.RequestMean = formatDuration(snap.Mean("proxy_request_duration_seconds")) v.StatusClasses = statusClasses(snap) hits := snap.SumBy("proxy_cache_hits_total", "kind") misses := snap.SumBy("proxy_cache_misses_total", "kind") v.ArtifactCache = cacheLookups(hits, misses, metrics.CacheKindArtifact) v.MetadataCache = cacheLookups(hits, misses, metrics.CacheKindMetadata) v.MetadataServed = formatSize(int64(snap.Sum("proxy_metadata_cache_served_bytes_total"))) v.UpstreamFetches = formatCount(int64(snap.Count("proxy_upstream_fetch_duration_seconds"))) v.UpstreamFetchMean = formatDuration(snap.Mean("proxy_upstream_fetch_duration_seconds")) v.UpstreamErrors = failures("proxy_upstream_errors_total", "ecosystem", "error_type") v.StorageOps = histogramRows(snap, "proxy_storage_operation_duration_seconds", "operation") v.StorageErrors = failures("proxy_storage_errors_total", "operation") v.IntegrityFailures = failures("proxy_integrity_failures_total", "ecosystem") v.ProbeFailures = failures("proxy_health_probe_failures_total", "step") v.Evictions = countRows(snap, "proxy_artifacts_evicted_total", "reason", "ecosystem") v.MetadataEvictions = countRows(snap, "proxy_metadata_evicted_total", "ecosystem") v.ChangeFeedExpired = countRows(snap, "proxy_metadata_expired_by_change_feed_total", "ecosystem") v.Breakers = breakerRows(snap) v.BreakerTrips = failures("proxy_circuit_breaker_trips_total", "registry") v.Scans = histogramRows(snap, "proxy_scan_duration_seconds", "ecosystem", "scanner") v.ScansBlocked = failures("proxy_scan_blocked_total", "ecosystem", "scanner") v.ScanErrors = failures("proxy_scan_errors_total", "ecosystem", "scanner", "error_type") v.ScanningOn = len(v.Scans) > 0 || len(v.ScansBlocked) > 0 || len(v.ScanErrors) > 0 v.HasFailures = len(v.UpstreamErrors) > 0 || len(v.StorageErrors) > 0 || len(v.IntegrityFailures) > 0 || len(v.ProbeFailures) > 0 || len(v.ScanErrors) > 0 v.ResponseBytes = formatSize(int64(snap.Sum("proxy_response_bytes_total"))) v.Clients = clientRows(snap) return v } // clientRows pairs each client tool's request count with the bytes it pulled, // busiest first. Both come from the same closed label set, so the two counters // line up row for row. func clientRows(snap *metrics.Snapshot) []ClientRow { bytesByClient := snap.SumBy("proxy_client_response_bytes_total", "client") type entry struct { client string requests float64 bytes float64 } entries := make([]entry, 0) for _, s := range snap.Samples("proxy_client_requests_total") { client := s.Label("client") entries = append(entries, entry{ client: client, requests: s.Value, bytes: bytesByClient[client], }) } sort.Slice(entries, func(i, j int) bool { a, b := entries[i], entries[j] if a.bytes != b.bytes { return a.bytes > b.bytes } return a.requests > b.requests }) rows := make([]ClientRow, 0, len(entries)) for _, e := range entries { rows = append(rows, ClientRow{ Client: e.client, Requests: formatCount(int64(e.requests)), Bytes: formatSize(int64(e.bytes)), }) } return rows } // statusClasses groups proxy_requests_total into 2xx/3xx/4xx/5xx buckets, which // is the breakdown worth showing; the per-code detail stays in Prometheus. func statusClasses(snap *metrics.Snapshot) []LabelledCount { byClass := make(map[string]float64) for _, s := range snap.Samples("proxy_requests_total") { status := s.Label("status") if len(status) == 0 { continue } byClass[string(status[0])+"xx"] += s.Value } classes := make([]string, 0, len(byClass)) for c := range byClass { classes = append(classes, c) } sort.Strings(classes) rows := make([]LabelledCount, 0, len(classes)) for _, c := range classes { rows = append(rows, LabelledCount{ Label: c, Count: formatCount(int64(byClass[c])), Bad: c == "5xx", }) } return rows } // failureRows renders every non-zero series of a failure counter, keyed by the // joined values of the given labels. Zero-valued series are dropped: a counter // that has never fired carries no information, and a wall of zeroes buries the // rows that matter. With withRecent set, each row carries the latest failures // recorded for its series. func failureRows(snap *metrics.Snapshot, name string, withRecent bool, labels ...string) []LabelledCount { rows := countRows(snap, name, labels...) for i := range rows { rows[i].Bad = true if withRecent { rows[i].Recent = recentFailureRows(metrics.RecentFailures(name, rows[i].series)) } } return rows } // countRows renders every non-zero series of a counter as a row. func countRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledCount { var rows []LabelledCount for _, s := range snap.Samples(name) { if s.Value == 0 { continue } rows = append(rows, LabelledCount{ Label: joinLabels(s, labels...), Count: formatCount(int64(s.Value)), series: s.Labels, }) } return rows } func recentFailureRows(failures []metrics.Failure) []FailureRow { if len(failures) == 0 { return nil } rows := make([]FailureRow, 0, len(failures)) for _, f := range failures { rows = append(rows, FailureRow{ Age: formatTimeAgo(f.At), PURL: f.PURL, Filename: f.Filename, URL: f.URL, Path: f.Path, Error: f.Error, }) } return rows } // histogramRows renders every observed series of a histogram with its count and mean. func histogramRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledStat { var rows []LabelledStat for _, s := range snap.Samples(name) { if s.Count == 0 { continue } rows = append(rows, LabelledStat{ Label: joinLabels(s, labels...), Count: formatCount(int64(s.Count)), Mean: formatDuration(s.Mean()), }) } return rows } func breakerRows(snap *metrics.Snapshot) []BreakerRow { var rows []BreakerRow for _, s := range snap.Samples("proxy_circuit_breaker_state") { state := "closed" switch s.Value { case 1: state = "half-open" case 2: //nolint:mnd // 2 = open, per the gauge's documented encoding state = "open" } rows = append(rows, BreakerRow{ Registry: s.Label("registry"), State: state, Open: s.Value > 0, }) } return rows } func joinLabels(s metrics.Sample, labels ...string) string { out := "" for _, l := range labels { v := s.Label(l) if v == "" { continue } if out != "" { out += " · " } out += v } if out == "" { return "-" } return out } // formatDuration renders a mean latency at a sensible unit. Storage operations // land in microseconds and upstream fetches in seconds, so a fixed unit would // read as either 0.000 or an unreadable pile of digits. func formatDuration(seconds float64) string { if seconds <= 0 { return "-" } d := time.Duration(seconds * float64(time.Second)) switch { case d < time.Microsecond: return fmt.Sprintf("%.0f ns", float64(d)) case d < time.Millisecond: return fmt.Sprintf("%.0f µs", float64(d)/float64(time.Microsecond)) case d < time.Second: return fmt.Sprintf("%.1f ms", float64(d)/float64(time.Millisecond)) default: return fmt.Sprintf("%.2f s", d.Seconds()) } }