pkg-proxy/internal/database/cache_eviction_test.go

package database

import (
	"database/sql"
	"strings"
	"testing"
	"time"
)

func upsertTestMetadata(t *testing.T, db *DB, name string, size int64) {
	t.Helper()
	if err := db.UpsertMetadataCache(&MetadataCacheEntry{
		Ecosystem:   testEcosystemNPM,
		Name:        name,
		StoragePath: "_metadata/npm/" + name + "/metadata",
		Size:        sql.NullInt64{Int64: size, Valid: true},
		FetchedAt:   sql.NullTime{Time: time.Now(), Valid: true},
	}); err != nil {
		t.Fatalf("UpsertMetadataCache: %v", err)
	}
}

func setTestMetadataAccess(t *testing.T, db *DB, name string, at time.Time) {
	t.Helper()
	query := db.Rebind(`UPDATE metadata_cache SET last_accessed_at = ? WHERE ecosystem = ? AND name = ?`)
	if _, err := db.Exec(query, at.UTC(), testEcosystemNPM, name); err != nil {
		t.Fatalf("setting last access: %v", err)
	}
}

func TestUpsertMetadataCacheMarksAccess(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		before := time.Now().Add(-time.Second)
		upsertTestMetadata(t, db, "lodash", 10)

		got, err := db.GetMetadataCache(testEcosystemNPM, "lodash")
		if err != nil || got == nil {
			t.Fatalf("GetMetadataCache: %v, %v", got, err)
		}
		if !got.LastAccessedAt.Valid || got.LastAccessedAt.Time.Before(before) {
			t.Errorf("last_accessed_at = %v, want the time of the write", got.LastAccessedAt)
		}
	})
}

func TestGetEvictableCacheSizeCountsMetadata(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		upsertTestMetadata(t, db, "a", 300)
		upsertTestMetadata(t, db, "b", 200)

		total, err := db.GetEvictableCacheSize()
		if err != nil {
			t.Fatalf("GetEvictableCacheSize: %v", err)
		}
		if total != 500 {
			t.Errorf("evictable size = %d, want 500", total)
		}
		// The size shown in stats stays the artifacts' total.
		artifacts, err := db.GetTotalCacheSize()
		if err != nil || artifacts != 0 {
			t.Errorf("GetTotalCacheSize = %d, %v, want 0", artifacts, err)
		}
	})
}

func TestGetLeastRecentlyUsedCacheEntriesOrdersMetadataAndArtifacts(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		now := time.Now()
		upsertTestMetadata(t, db, "old", 1)
		setTestMetadataAccess(t, db, "old", now.Add(-3*time.Hour))
		upsertTestMetadata(t, db, "new", 1)
		setTestMetadataAccess(t, db, "new", now)
		if err := db.UpsertArtifact(&Artifact{
			VersionPURL:    "pkg:npm/mid@1.0.0",
			Filename:       "mid-1.0.0.tgz",
			UpstreamURL:    "https://example.test/mid-1.0.0.tgz",
			StoragePath:    sql.NullString{String: "npm/mid/1.0.0/mid-1.0.0.tgz", Valid: true},
			Size:           sql.NullInt64{Int64: 1, Valid: true},
			LastAccessedAt: sql.NullTime{Time: now.Add(-time.Hour), Valid: true},
		}); err != nil {
			t.Fatalf("UpsertArtifact: %v", err)
		}

		entries, err := db.GetLeastRecentlyUsedCacheEntries(10)
		if err != nil {
			t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v", err)
		}
		var got []string
		for _, e := range entries {
			got = append(got, e.Kind+":"+e.Name)
		}
		want := []string{"metadata:old", "artifact:mid-1.0.0.tgz", "metadata:new"}
		if len(got) != len(want) {
			t.Fatalf("entries = %v, want %v", got, want)
		}
		for i := range want {
			if got[i] != want[i] {
				t.Fatalf("entries = %v, want %v", got, want)
			}
		}
	})
}

func TestEvictMetadataCacheKeepsRowUsedSinceRead(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		upsertTestMetadata(t, db, "lodash", 10)
		setTestMetadataAccess(t, db, "lodash", time.Now().Add(-time.Hour))
		read, _ := db.GetMetadataCache(testEcosystemNPM, "lodash")

		if err := db.RecordMetadataHit(testEcosystemNPM, "lodash"); err != nil {
			t.Fatalf("RecordMetadataHit: %v", err)
		}
		evicted, err := db.EvictMetadataCache(testEcosystemNPM, "lodash", read.StoragePath, read.LastAccessedAt)
		if err != nil || evicted {
			t.Fatalf("EvictMetadataCache = %v, %v, want the row kept", evicted, err)
		}

		current, _ := db.GetMetadataCache(testEcosystemNPM, "lodash")
		evicted, err = db.EvictMetadataCache(testEcosystemNPM, "lodash", current.StoragePath, current.LastAccessedAt)
		if err != nil || !evicted {
			t.Fatalf("EvictMetadataCache = %v, %v, want the row evicted", evicted, err)
		}
		if gone, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); gone != nil {
			t.Error("row still present after eviction")
		}
	})
}

func TestRecordMetadataHitBatched(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		upsertTestMetadata(t, db, "lodash", 10)
		old := time.Now().Add(-time.Hour)
		setTestMetadataAccess(t, db, "lodash", old)
		db.BatchHits(time.Hour, nil)

		if err := db.RecordMetadataHit(testEcosystemNPM, "lodash"); err != nil {
			t.Fatalf("RecordMetadataHit: %v", err)
		}
		if got, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); got.LastAccessedAt.Time.After(old.Add(time.Minute)) {
			t.Error("batched hit was written before the flush")
		}
		if err := db.flushHits(); err != nil {
			t.Fatalf("flushHits: %v", err)
		}
		if got, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); !got.LastAccessedAt.Time.After(old.Add(time.Minute)) {
			t.Errorf("last_accessed_at = %v after flush, want the hit's time", got.LastAccessedAt)
		}
	})
}

func TestPendingDeleteSkipsPathMetadataPointsAt(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		upsertTestMetadata(t, db, "lodash", 10)
		path := "_metadata/npm/lodash/metadata"
		if err := db.QueuePendingDelete(path); err != nil {
			t.Fatalf("QueuePendingDelete: %v", err)
		}

		due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10)
		if err != nil {
			t.Fatalf("GetDuePendingDeletes: %v", err)
		}
		if len(due) != 0 {
			t.Errorf("due = %v, want the path a metadata row uses skipped", due)
		}
	})
}

// TestGetLeastRecentlyUsedCacheEntriesCountsFetchAsUse checks that an
// artifact not served since its download ranks by when it was fetched, not
// ahead of every metadata document.
func TestGetLeastRecentlyUsedCacheEntriesCountsFetchAsUse(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		now := time.Now()
		upsertTestMetadata(t, db, "old", 1)
		setTestMetadataAccess(t, db, "old", now.Add(-time.Hour))
		if err := db.UpsertArtifact(&Artifact{
			VersionPURL: "pkg:npm/fresh@1.0.0",
			Filename:    "fresh-1.0.0.tgz",
			UpstreamURL: "https://example.test/fresh-1.0.0.tgz",
			StoragePath: sql.NullString{String: "npm/fresh/1.0.0/fresh-1.0.0.tgz", Valid: true},
			Size:        sql.NullInt64{Int64: 1, Valid: true},
			FetchedAt:   sql.NullTime{Time: now, Valid: true},
		}); err != nil {
			t.Fatalf("UpsertArtifact: %v", err)
		}

		entries, err := db.GetLeastRecentlyUsedCacheEntries(1)
		if err != nil || len(entries) != 1 {
			t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v, %v", entries, err)
		}
		if entries[0].Kind != CacheEntryMetadata {
			t.Errorf("least recently used = %+v, want the metadata used an hour ago", entries[0])
		}
	})
}

// TestLeastRecentlyUsedQueriesUseIndexes checks on SQLite that eviction reads
// both tables off an index rather than sorting them whole.
func TestLeastRecentlyUsedQueriesUseIndexes(t *testing.T) {
	db := setupMetadataCacheDB(t)
	for _, q := range []string{
		`SELECT version_purl FROM artifacts WHERE storage_path IS NOT NULL AND source <> 'local'
		 ORDER BY COALESCE(last_accessed_at, fetched_at) ASC NULLS FIRST LIMIT 50`,
		`SELECT name FROM metadata_cache ORDER BY COALESCE(last_accessed_at, updated_at) ASC NULLS FIRST LIMIT 50`,
	} {
		rows, err := db.Query("EXPLAIN QUERY PLAN " + q)
		if err != nil {
			t.Fatalf("explain: %v", err)
		}
		var plan []string
		for rows.Next() {
			var id, parent, unused int
			var detail string
			if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
				t.Fatalf("scan: %v", err)
			}
			plan = append(plan, detail)
		}
		_ = rows.Close()
		for _, step := range plan {
			if strings.Contains(step, "TEMP B-TREE") {
				t.Errorf("query sorts instead of using an index: %v\n%s", plan, q)
			}
		}
	}
}

// insertLegacyMetadata adds a row as the proxy wrote it before
// last_accessed_at existed: write times in local time, no access time.
func insertLegacyMetadata(t *testing.T, db *DB, name string, written time.Time) {
	t.Helper()
	if _, err := db.Exec(db.Rebind(`
		INSERT INTO metadata_cache (ecosystem, name, storage_path, size, created_at, updated_at)
		VALUES (?, ?, ?, ?, ?, ?)`), testEcosystemNPM, name, "_metadata/npm/"+name+"/metadata", 1, written, written); err != nil {
		t.Fatalf("inserting legacy row: %v", err)
	}
}

func TestMetadataLastAccessedMigrationLeavesBackfillToBackground(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		for _, q := range []string{
			"DROP INDEX idx_metadata_last_used",
			"DROP INDEX idx_metadata_storage_path",
			"DROP INDEX idx_artifacts_last_used",
			"ALTER TABLE metadata_cache DROP COLUMN last_accessed_at",
			"DELETE FROM migrations WHERE name = '014_add_metadata_last_accessed'",
		} {
			if _, err := db.Exec(q); err != nil {
				t.Fatalf("%s: %v", q, err)
			}
		}
		written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.Local)
		insertLegacyMetadata(t, db, "legacy", written)

		if err := db.MigrateSchema(); err != nil {
			t.Fatalf("MigrateSchema: %v", err)
		}
		if got, _ := db.GetMetadataCache(testEcosystemNPM, "legacy"); got == nil || got.LastAccessedAt.Valid {
			t.Fatalf("row after migration = %+v, want it kept with no access time", got)
		}
		if n, err := db.BackfillMetadataLastAccessed(10); err != nil || n != 1 {
			t.Fatalf("BackfillMetadataLastAccessed = %d, %v, want 1", n, err)
		}
		got, _ := db.GetMetadataCache(testEcosystemNPM, "legacy")
		if got == nil || !got.LastAccessedAt.Valid || !got.LastAccessedAt.Time.Equal(written) {
			t.Errorf("backfilled last_accessed_at = %+v, want %v", got, written.UTC())
		}
	})
}

// TestBackfillMetadataLastAccessedResumes runs the backfill in batches, as a
// restart after an interruption would, and leaves rows served meanwhile alone.
func TestBackfillMetadataLastAccessedResumes(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		written := time.Now().Add(-48 * time.Hour).Truncate(time.Second)
		for _, name := range []string{"a", "b", "c"} {
			insertLegacyMetadata(t, db, name, written)
		}
		if n, err := db.BackfillMetadataLastAccessed(2); err != nil || n != 2 {
			t.Fatalf("first batch = %d, %v, want 2", n, err)
		}
		var left string
		if err := db.Get(&left, `SELECT name FROM metadata_cache WHERE last_accessed_at IS NULL`); err != nil {
			t.Fatalf("finding the row left: %v", err)
		}
		if err := db.RecordMetadataHit(testEcosystemNPM, left); err != nil {
			t.Fatalf("RecordMetadataHit: %v", err)
		}
		if n, err := db.BackfillMetadataLastAccessed(2); err != nil || n != 0 {
			t.Fatalf("second batch = %d, %v, want 0", n, err)
		}
		got, _ := db.GetMetadataCache(testEcosystemNPM, left)
		if !got.LastAccessedAt.Time.After(written.Add(time.Hour)) {
			t.Errorf("served row last_accessed_at = %v, want the hit kept", got.LastAccessedAt)
		}
	})
}

// TestLeastRecentlyUsedOrdersUnbackfilledMetadataByWriteTime checks that rows
// still waiting for the backfill are not all evicted first.
func TestLeastRecentlyUsedOrdersUnbackfilledMetadataByWriteTime(t *testing.T) {
	runWithBothDatabases(t, func(t *testing.T, db *DB) {
		now := time.Now()
		insertLegacyMetadata(t, db, "recent", now)
		if err := db.UpsertArtifact(&Artifact{
			VersionPURL:    "pkg:npm/old@1.0.0",
			Filename:       "old-1.0.0.tgz",
			UpstreamURL:    "https://example.test/old-1.0.0.tgz",
			StoragePath:    sql.NullString{String: "npm/old/1.0.0/old-1.0.0.tgz", Valid: true},
			Size:           sql.NullInt64{Int64: 1, Valid: true},
			LastAccessedAt: sql.NullTime{Time: now.Add(-time.Hour), Valid: true},
		}); err != nil {
			t.Fatalf("UpsertArtifact: %v", err)
		}

		entries, err := db.GetLeastRecentlyUsedCacheEntries(1)
		if err != nil || len(entries) != 1 {
			t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v, %v", entries, err)
		}
		if entries[0].Kind != CacheEntryArtifact {
			t.Errorf("least recently used = %+v, want the artifact used an hour ago", entries[0])
		}
	})
}