perf: speed up action downloads (#1209)

An action pinned to a full commit hash cloned the whole repository, and a cached one hit the network on every run. Now only the pinned commit is fetched at depth 1, a cached commit is reused with no network at all, and the action's `.git` directory no longer ships into the job container, matching GitHub. ``` repo cold ms warm ms cache KiB actions/checkout 5807 → 787 1064 → 12 11492 → 2349 actions/setup-node 15257 → 1013 1365 → 21 64303 → 9924 actions/cache 18782 → 760 1589 → 16 60890 → 12507 actions/setup-go 5513 → 764 589 → 20 16600 → 9104 docker/login-action 25809 → 1225 1288 → 10 81709 → 12503 ``` Reviewed-on: https://gitea.com/gitea/runner/pulls/1209 Reviewed-by: bircni <bircni@icloud.com> Co-authored-by: silverwind <me@silverwind.io>
2026-09-21 19:37:07 +02:00 · 2026-09-05 08:52:10 +00:00
parent 1745c7c841
commit c158ac5472
13 changed files with 255 additions and 67 deletions
@@ -391,7 +391,11 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
 			}
 		}

-		if !isOfflineMode {
+		// A just-cloned ref is as current as a fetch would make it, and a commit hash never moves.
+		_, _, present := refRevision(r, input.Ref)
+		refresh := !isOfflineMode && (!present || (reused && !plumbing.IsHash(input.Ref)))
+
+		if refresh {
 			err = r.FetchContext(ctx, &fetchOptions)
 			if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
 				return err
@@ -411,23 +415,7 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
 			}
 		}

-		// At this point we need to know if it's a tag or a branch
-		// And the easiest way to do it is duck typing
-		//
-		// If err is nil, it's a tag so let's proceed with that hash like we would if
-		// it was a sha
-		refType := "tag"
-		rev = plumbing.Revision(path.Join("refs", "tags", input.Ref))
-		if _, err := r.Tag(input.Ref); errors.Is(err, git.ErrTagNotFound) {
-			rName := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", input.Ref))
-			if _, err := r.Reference(rName, false); errors.Is(err, plumbing.ErrReferenceNotFound) {
-				refType = "sha"
-				rev = plumbing.Revision(input.Ref)
-			} else {
-				refType = "branch"
-				rev = plumbing.Revision(rName)
-			}
-		}
+		rev, refType, _ := refRevision(r, input.Ref)

 		if hash, err = r.ResolveRevision(rev); err != nil {
 			logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
@@ -459,7 +447,7 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
 		reusedMsg := ""

 		switch {
-		case !isOfflineMode && !shallow:
+		case refresh && !shallow:
 			// In shallow mode the depth-limited fetch above already advanced the ref.
 			if err = w.PullContext(ctx, &pullOptions); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
 				logger.Debugf("Unable to pull %s: %v", refName, err)
@@ -501,27 +489,37 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
 	}
 }

-// cloneAtDepth clones input.URL into input.Dir using opts.
-// With input.Depth > 0 it first tries a shallow, single-branch clone of input.Ref, falling back when error.
+// cloneAtDepth falls back to a full clone when the shallow attempt fails.
 func cloneAtDepth(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions, logger log.FieldLogger) (*git.Repository, error) {
 	if input.Depth > 0 {
-		for _, refName := range []plumbing.ReferenceName{
-			plumbing.NewBranchReferenceName(input.Ref),
-			plumbing.NewTagReferenceName(input.Ref),
-		} {
-			shallowOpts := opts
-			shallowOpts.Depth = input.Depth
-			shallowOpts.SingleBranch = true
-			shallowOpts.ReferenceName = refName
-			shallowOpts.Tags = git.NoTags
-
-			r, err := git.PlainCloneContext(ctx, input.Dir, false, &shallowOpts)
+		if plumbing.IsHash(input.Ref) {
+			r, err := fetchPinnedSHA(ctx, input, opts)
 			if err == nil {
 				return r, nil
 			}
-			logger.Debugf("Shallow clone of %s as %s failed: %v", input.URL, refName, err)
-			if rmErr := os.RemoveAll(input.Dir); rmErr != nil {
-				return nil, fmt.Errorf("remove partial clone %s: %w", input.Dir, rmErr)
+			logger.Debugf("Shallow fetch of %s at %s failed: %v", input.URL, input.Ref, err)
+			if err := removePartialClone(input.Dir); err != nil {
+				return nil, err
+			}
+		} else {
+			for _, refName := range []plumbing.ReferenceName{
+				plumbing.NewBranchReferenceName(input.Ref),
+				plumbing.NewTagReferenceName(input.Ref),
+			} {
+				shallowOpts := opts
+				shallowOpts.Depth = input.Depth
+				shallowOpts.SingleBranch = true
+				shallowOpts.ReferenceName = refName
+				shallowOpts.Tags = git.NoTags
+
+				r, err := git.PlainCloneContext(ctx, input.Dir, false, &shallowOpts)
+				if err == nil {
+					return r, nil
+				}
+				logger.Debugf("Shallow clone of %s as %s failed: %v", input.URL, refName, err)
+				if err := removePartialClone(input.Dir); err != nil {
+					return nil, err
+				}
 			}
 		}
 		logger.Debugf("Falling back to a full clone of %s for ref %q", input.URL, input.Ref)
@@ -530,14 +528,65 @@ func cloneAtDepth(ctx context.Context, input NewGitCloneExecutorInput, opts git.
 	return git.PlainCloneContext(ctx, input.Dir, false, &opts)
 }

+func removePartialClone(dir string) error {
+	if err := os.RemoveAll(dir); err != nil {
+		return fmt.Errorf("remove partial clone %s: %w", dir, err)
+	}
+	return nil
+}
+
+func fetchPinnedSHA(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions) (*git.Repository, error) {
+	r, err := git.PlainInit(input.Dir, false)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := r.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{input.URL}}); err != nil {
+		return nil, err
+	}
+	if err := r.FetchContext(ctx, &git.FetchOptions{
+		RefSpecs:        []config.RefSpec{pinnedRefSpec(input.Ref)},
+		Depth:           input.Depth,
+		Tags:            git.NoTags,
+		Auth:            opts.Auth,
+		Progress:        opts.Progress,
+		InsecureSkipTLS: opts.InsecureSkipTLS,
+	}); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// pinnedRefSpec keeps a hash-shaped name out of refs/heads and refs/tags.
+func pinnedRefSpec(sha string) config.RefSpec {
+	return config.RefSpec(fmt.Sprintf("+%s:refs/pinned/%s", sha, sha))
+}
+
+// refRevision picks the revision to check out and reports whether it resolves locally.
+func refRevision(r *git.Repository, ref string) (plumbing.Revision, string, bool) {
+	if plumbing.IsHash(ref) {
+		// git ignores a ref named as 40 hex digits, so a full hash always denotes the commit itself.
+		_, err := r.CommitObject(plumbing.NewHash(ref))
+		return plumbing.Revision(ref), "sha", err == nil
+	}
+	if _, err := r.Tag(ref); err == nil {
+		return plumbing.Revision(path.Join("refs", "tags", ref)), "tag", true
+	}
+	remoteRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", ref))
+	if _, err := r.Reference(remoteRef, false); err == nil {
+		return plumbing.Revision(remoteRef), "branch", true
+	}
+	rev := plumbing.Revision(ref)
+	_, err := r.ResolveRevision(rev)
+	return rev, "sha", err == nil
+}
+
 // isShallow reports whether the local repository was cloned with a limited depth.
 func isShallow(r *git.Repository) bool {
 	shallows, err := r.Storer.Shallow()
 	return err == nil && len(shallows) > 0
 }

-// shallowFetchRefSpec returns the single refspec that updates only input.Ref, keeping a shallow clone from re-downloading every branch's history.
-// ok is false when the ref is not present locally as a tag or remote-tracking branch, in which case the broad default refspec is used.
+// shallowFetchRefSpec limits a shallow update to the requested ref, falling back to the broad refspec when it is not present locally.
 func shallowFetchRefSpec(r *git.Repository, ref string) (config.RefSpec, bool) {
 	tagRef := plumbing.NewTagReferenceName(ref)
 	if _, err := r.Reference(tagRef, false); err == nil {
@@ -548,5 +597,9 @@ func shallowFetchRefSpec(r *git.Repository, ref string) (config.RefSpec, bool) {
 		branchRef := plumbing.NewBranchReferenceName(ref)
 		return config.RefSpec(fmt.Sprintf("+%s:%s", branchRef, remoteRef)), true
 	}
+	if plumbing.IsHash(ref) {
+		// The broad refspec carries only advertised tips, never a pinned commit.
+		return pinnedRefSpec(ref), true
+	}
 	return "", false
 }
@@ -16,6 +16,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"testing"
 	"time"
@@ -24,6 +25,9 @@ import (

 	gogit "github.com/go-git/go-git/v5"
 	gogitconfig "github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/plumbing/transport"
+	gogitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
+	gogitfile "github.com/go-git/go-git/v5/plumbing/transport/file"
 	log "github.com/sirupsen/logrus"
 	logrustest "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/assert"
@@ -435,7 +439,7 @@ func TestGitCloneExecutorShallow(t *testing.T) {
 		require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", m))
 	}
 	require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
-	sha := gitRevParse(t, workDir, "HEAD~1") // c2, a SHA that go-git cannot shallow-clone
+	unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
 	require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
 	require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v1"))

@@ -461,14 +465,43 @@ func TestGitCloneExecutorShallow(t *testing.T) {
 		assert.Equal(t, "c3", gitHeadSubject(t, dir))
 	})

-	t.Run("SHA falls back to a full clone", func(t *testing.T) {
+	unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
+		remote := filepath.Join(t.TempDir(), "remote.git")
+		require.NoError(t, gitCmd("clone", "--bare", remoteDir, remote))
+		require.NoError(t, gitCmd("-C", remote, "config", "uploadpack.allowAnySHA1InWant", strconv.FormatBool(allowUnadvertised)))
+		return remote
+	}
+
+	t.Run("commit hash falls back to a full clone when the remote refuses unadvertised objects", func(t *testing.T) {
 		dir := t.TempDir()
 		require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
-			URL: remoteDir, Ref: sha, Dir: dir, Depth: 1,
+			URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
 		})(t.Context()))
-		// go-git cannot shallow-clone a raw SHA, so it falls back to a full clone; the absence of a shallow marker proves the fallback happened.
-		assert.NoFileExists(t, shallowMarker(dir), "a SHA ref must not produce a shallow clone")
-		assert.Equal(t, sha, gitRevParse(t, dir, "HEAD"))
+		assert.NoFileExists(t, shallowMarker(dir))
+		assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
+	})
+
+	t.Run("commit hash is fetched shallowly, moved to another hash, then reused without reaching the remote", func(t *testing.T) {
+		remote := unadvertisedRemote(t, true)
+		dir := t.TempDir()
+		cloneAt := func(ref string) error {
+			return NewGitCloneExecutor(NewGitCloneExecutorInput{
+				URL: remote, Ref: ref, Dir: dir, Depth: 1,
+			})(t.Context())
+		}
+
+		require.NoError(t, cloneAt(unadvertisedSHA))
+		assert.FileExists(t, shallowMarker(dir))
+		assert.Equal(t, 1, gitRevCount(t, dir))
+		assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
+
+		olderSHA := gitRevParse(t, workDir, "HEAD~2")
+		require.NoError(t, cloneAt(olderSHA))
+		assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
+
+		require.NoError(t, os.RemoveAll(remote))
+		require.NoError(t, cloneAt(olderSHA))
+		assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
 	})

 	t.Run("moving branch updates while staying shallow", func(t *testing.T) {
@@ -491,6 +524,77 @@ func TestGitCloneExecutorShallow(t *testing.T) {
 	})
 }

+func TestGitCloneExecutorColdCloneSkipsRefresh(t *testing.T) {
+	remoteDir := t.TempDir()
+	require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
+	workDir := t.TempDir()
+	require.NoError(t, gitCmd("clone", remoteDir, workDir))
+	require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
+	require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c1"))
+	require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
+	require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
+	require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v1"))
+
+	for name, tt := range map[string]struct {
+		Ref   string
+		Depth int
+	}{
+		"shallow branch": {"main", 1},
+		"full clone tag": {"v1", 0},
+	} {
+		t.Run(name, func(t *testing.T) {
+			counter := installCountingTransport(t)
+			require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
+				URL: remoteDir, Ref: tt.Ref, Dir: t.TempDir(), Depth: tt.Depth,
+			})(t.Context()))
+			assert.Equal(t, int64(1), counter.sessions.Load())
+		})
+	}
+}
+
+func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
+	remoteDir := t.TempDir()
+	require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
+	workDir := t.TempDir()
+	require.NoError(t, gitCmd("clone", remoteDir, workDir))
+	require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
+	require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "pinned"))
+	pinned := gitRevParse(t, workDir, "HEAD")
+	require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
+	require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
+	require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
+	require.NoError(t, gitCmd("-C", workDir, "push", "origin", pinned))
+
+	for name, depth := range map[string]int{"shallow": 1, "full clone": 0} {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
+				URL: remoteDir, Ref: pinned, Dir: dir, Depth: depth,
+			})(t.Context()))
+			assert.Equal(t, pinned, gitRevParse(t, dir, "HEAD"))
+		})
+	}
+}
+
+type countingTransport struct {
+	transport.Transport
+	sessions atomic.Int64
+}
+
+func (c *countingTransport) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) {
+	c.sessions.Add(1)
+	return c.Transport.NewUploadPackSession(ep, auth)
+}
+
+func installCountingTransport(t *testing.T) *countingTransport {
+	t.Helper()
+	counter := &countingTransport{Transport: gogitfile.DefaultClient}
+	gogitclient.InstallProtocol("file", counter)
+	t.Cleanup(func() { gogitclient.InstallProtocol("file", gogitfile.DefaultClient) })
+
+	return counter
+}
+
 func gitRevParse(t *testing.T, dir, rev string) string {
 	t.Helper()
 	out, err := exec.Command("git", "-C", dir, "rev-parse", rev).Output()