pkg-proxy/internal/gitcache/mirror.go

package gitcache

import (
	"context"
	"errors"
	"fmt"
	"os/exec"
	"slices"
	"strings"
	"time"
)

// MirrorResult is what Mirror did.
type MirrorResult struct {
	// Cached reports that the repository, and the commit if a version was
	// given, were cached already.
	Cached bool
	// Bytes is how much the cached repository grew.
	Bytes int64
	// Commit is the commit the version names, empty without a version.
	Commit string
	// Submodules are the submodules Commit records, sorted by path.
	Submodules []Submodule
}

// Submodule is a submodule a commit records: its path, the URL .gitmodules
// gives for it, which may be relative to the repository's, and the commit
// the gitlink pins.
type Submodule struct {
	Path, URL, Commit string
}

// Mirror brings the repository cached under key up to date with repoURL,
// cloning it if needed, and waits for that to finish. version, if given, is
// a full commit ID or a tag. A commit is made sure to be cached: one on no
// branch or tag is fetched by ID when fetchWants allows it (see
// OpenOptions.FetchWants), and is kept as long as a commit a client fetched
// by ID. A tag is cached with the repository's tags.
func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) {
	if lower := strings.ToLower(version); isObjectID(lower) {
		version = lower
	}
	tag := version != "" && !isObjectID(version)
	if tag {
		if err := c.checkTagName(ctx, version); err != nil {
			return MirrorResult{}, err
		}
	}
	lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants, NoCheck: true})
	if err != nil {
		return MirrorResult{}, err
	}
	defer lease.Release()
	rp := lease.repo
	cached := lease.hit
	before := int64(0)
	var refreshErr error
	// Open cloned a repository that wasn't cached; one that was is
	// refreshed here, waiting for the fetch a client would not wait for.
	if cached {
		before = rp.size.Load()
		refresh := c.startRefresh(rp, repoURL)
		select {
		case <-refresh.done:
			refreshErr = refresh.err
		case <-ctx.Done():
			return MirrorResult{}, ctx.Err()
		}
	}

	// A refresh past max_repo_size gets the repository dropped once it is
	// released, commit and all.
	if errors.Is(refreshErr, ErrTooLarge) {
		return MirrorResult{}, refreshErr
	}
	commit := version
	switch {
	case tag:
		if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil {
			if refreshErr != nil {
				// The tag may be one the failed refresh would have brought.
				return MirrorResult{}, refreshErr
			}
			if errors.Is(err, ErrNotFound) && strings.Trim(strings.ToLower(version), "0123456789abcdef") == "" {
				// Likely an abbreviated commit ID.
				return MirrorResult{}, fmt.Errorf("%s is not a full commit ID or tag: %w", version, ErrNotFound)
			}
			return MirrorResult{}, err
		}
		if refreshErr != nil {
			// Tags can move; this is where it pointed when last fetched.
			c.logger.Warn("git cache: refresh failed, mirroring the cached tag", "repository", key, "tag", version, "commit", commit, "error", refreshErr)
		}
	case version != "":
		if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil {
			return MirrorResult{}, err
		}
	}
	// A refresh that failed leaves the commit cached all the same.
	if refreshErr != nil && version == "" {
		return MirrorResult{}, refreshErr
	}
	var submodules []Submodule
	if commit != "" {
		if submodules, err = c.submodules(ctx, key, rp.dir, commit); err != nil {
			return MirrorResult{}, err
		}
	}
	grown := max(rp.size.Load()-before, 0)
	return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil
}

// checkTagName refuses a version that is no valid tag name, such as one
// with .. or a revision suffix like ^{tree}.
func (c *Cache) checkTagName(ctx context.Context, name string) error {
	if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil {
		return fmt.Errorf("%q is neither a full commit ID nor a tag name", name)
	}
	return nil
}

// resolveTag returns the commit tag points at.
func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) {
	out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output()
	var exit *exec.ExitError
	if errors.As(err, &exit) && exit.ExitCode() == 1 {
		return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound)
	}
	if err != nil {
		return "", fmt.Errorf("resolving tag %s: %w", tag, err)
	}
	return strings.TrimSpace(string(out)), nil
}

// submodules lists the submodules commit records: its gitlinks that
// .gitmodules names a URL for.
func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Submodule, error) {
	err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", commit+":.gitmodules").Run()
	var exit *exec.ExitError
	if errors.As(err, &exit) && exit.ExitCode() == 1 && ctx.Err() == nil {
		return nil, nil
	}
	if err != nil {
		if ctx.Err() != nil {
			return nil, ctx.Err()
		}
		return nil, fmt.Errorf("looking for .gitmodules: %w", err)
	}
	// --list rather than --get-regexp, which fails when nothing matches.
	out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list")
	if err != nil {
		if ctx.Err() != nil {
			return nil, ctx.Err()
		}
		// The commit is cached; a .gitmodules git can't read only loses
		// its submodules.
		c.logger.Warn("git cache: cannot read .gitmodules, skipping submodules", "repository", key, "commit", commit, "error", err)
		return nil, nil
	}
	paths, urls := map[string]string{}, map[string]string{}
	for entry := range strings.SplitSeq(out, "\x00") {
		key, value, ok := strings.Cut(entry, "\n")
		section, ok2 := strings.CutPrefix(key, "submodule.")
		name, field, dotted := strings.CutLast(section, ".")
		if !ok || !ok2 || !dotted {
			continue
		}
		switch field {
		case "path":
			paths[value] = name
		case "url":
			urls[name] = value
		}
	}
	var pathspecs []string
	for path, name := range paths {
		if path != "" && urls[name] != "" {
			pathspecs = append(pathspecs, path)
		}
	}
	if len(pathspecs) == 0 {
		return nil, nil
	}
	args := append([]string{"--literal-pathspecs", "ls-tree", "-z", "--full-tree", commit, "--"}, pathspecs...)
	tree, err := c.run(ctx, dir, args...)
	if err != nil {
		return nil, err
	}
	var submodules []Submodule
	for entry := range strings.SplitSeq(tree, "\x00") {
		meta, path, ok := strings.Cut(entry, "\t")
		fields := strings.Fields(meta)
		if !ok || len(fields) != 3 || fields[0] != "160000" {
			continue
		}
		if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) {
			submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]})
		}
	}
	slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) })
	return submodules, nil
}

// ensureCommit makes sure the repository has commit, fetching it by ID when
// it doesn't and fetchWants allows it. A commit on no branch or tag that is
// cached already gets a new wanted ref, so it is kept for wantedMaxAge from
// this run on.
func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) error {
	missing, err := c.missingObjects(ctx, rp.dir, []string{commit})
	if err != nil {
		return err
	}
	if len(missing) == 0 {
		return c.renewWanted(ctx, rp, commit)
	}
	if !fetchWants {
		return fmt.Errorf("commit %s: %w", commit, ErrNotFound)
	}
	if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil {
		if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) {
			return err
		}
		// The upstream refuses a commit it doesn't have.
		return fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err)
	}
	// fetchMissingWants skips a commit that failed moments ago.
	if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil {
		return err
	}
	if len(missing) > 0 {
		return fmt.Errorf("commit %s: %w", commit, ErrNotFound)
	}
	return nil
}

// renewWanted writes a new wanted ref for commit when no branch or tag
// includes it.
func (c *Cache) renewWanted(ctx context.Context, rp *repo, commit string) error {
	unlock, err := rp.lockWrites(ctx)
	if err != nil {
		return err
	}
	defer unlock()
	unmerged, err := c.unmergedCommits(ctx, rp.dir, []string{commit})
	if err != nil || !unmerged[commit] {
		return err
	}
	cmd := c.command(ctx, nil, "-C", rp.dir, "update-ref", wantedRef(commit, time.Now()), commit)
	if out, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("renewing wanted ref: %w: %s", err, strings.TrimSpace(string(out)))
	}
	return nil
}