perf: cut redundant work out of job setup and teardown (#1218)

Implement speedups to job start and shutdown.

- Create the job container while services are still becoming healthy, and poll their health at a flat one second instead of a 2s to 32s doubling backoff
- Pull each service image once instead of twice, and fetch a warm action cache once instead of twice
- Report the job result before reclaiming its volumes, and reap volumes stranded by a runner that died mid-job

| Step | Scenario | Before | After |
| --- | --- | --- | --- |
| Complete job | Large workspace volume | 4.2s | 0.4s |
| Set up job | One service, 2s health interval | 7.08s | 3.26s |
| Set up job | Two cached actions from github.com | 1.81s | 1.34s |
| Set up job | Two cached actions from gitea.com | 2.42s | 1.95s |
| Set up job | Two actions, cold action cache | 6.62s | unchanged |
| Set up job | Minimal job, no services or actions | 0.62s | unchanged |

Assisted-by: Claude Code:Opus 5
Reviewed-on: https://gitea.com/gitea/runner/pulls/1218
Reviewed-by: bircni <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>

internal/app/run/runner.go

@@ -146,6 +146,8 @@ func (r *Runner) Close() error {
 // removeOrphanNetworks is a variable so tests can substitute one that needs no Docker daemon.
 var removeOrphanNetworks = container.RemoveOrphanNetworks
 
+var removeOrphanJobVolumes = container.RemoveOrphanJobVolumes
+
 // OnIdle performs lightweight maintenance during polling idle windows.
 // It runs synchronously on the poller goroutine; shouldRunIdleCleanup
 // throttles invocations to runner.idle_cleanup_interval so the impact on
@@ -167,21 +169,20 @@ func (r *Runner) OnIdle(ctx context.Context) {
 	if hostRoot := filepath.FromSlash(r.cfg.Host.WorkdirParent); hostRoot != "" {
 		r.cleanupStaleDirs(ctx, hostRoot, isHostScratchDir)
 	}
-	r.cleanupOrphanNetworks(ctx)
+	r.cleanupOrphanDockerResources(ctx)
 }
 
-// cleanupOrphanNetworks reclaims the per-job networks of jobs this runner did not live to
-// tear down. A labelled network with no containers on it is finished with, and as for the
-// directories above, a task beginning during the pass is safe because the cutoff keeps a
-// network it has created but not yet attached a container to out of scope.
-func (r *Runner) cleanupOrphanNetworks(ctx context.Context) {
-	if r.uuid == "" || !r.requiresDocker() {
+func (r *Runner) cleanupOrphanDockerResources(ctx context.Context) {
+	if r.uuid == "" || (!r.requiresDocker() && !dockerReachable(ctx)) {
 		return
 	}
 	cutoff := r.now().Add(-r.cfg.Runner.WorkdirCleanupAge)
 	if err := removeOrphanNetworks(ctx, r.uuid, cutoff); err != nil {
 		log.Warnf("failed to clean up networks left behind by earlier jobs: %v", err)
 	}
+	if err := removeOrphanJobVolumes(ctx, r.uuid, cutoff); err != nil {
+		log.Warnf("failed to clean up volumes left behind by earlier jobs: %v", err)
+	}
 }
 
 func (r *Runner) shouldRunIdleCleanup() bool {
@@ -287,6 +288,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
 	defer r.runningTasks.Delete(task.Id)
 
 	r.runningCount.Add(1)
+	defer r.runningCount.Add(-1)
 
 	start := time.Now()
 
@@ -294,15 +296,25 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
 	defer cancel()
 	// A proxy URL may carry credentials, and every job is given it; keep them out of the log.
 	reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg, proxyPasswords()...)
+	var volumeCleanup []common.Executor
+	var volumeCleanupMu sync.Mutex
+	if r.cfg.Runner.PostTaskScript == "" {
+		ctx = runner.WithJobVolumeCleanup(ctx, func(cleanup common.Executor) {
+			volumeCleanupMu.Lock()
+			defer volumeCleanupMu.Unlock()
+			volumeCleanup = append(volumeCleanup, cleanup)
+		})
+	}
 	var runErr error
 	defer func() {
-		r.runningCount.Add(-1)
-
 		lastWords := ""
 		if runErr != nil {
 			lastWords = runErr.Error()
 		}
 		_ = reporter.Close(lastWords)
+		if err := cleanupJobVolumes(ctx, volumeCleanup); err != nil {
+			log.Warnf("task %d volume cleanup after reporting: %v", task.Id, err)
+		}
 
 		metrics.JobDuration.Observe(time.Since(start).Seconds())
 		metrics.JobsTotal.WithLabelValues(metrics.ResultToStatusLabel(reporter.Result())).Inc()
@@ -313,6 +325,16 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
 	return nil
 }
 
+func cleanupJobVolumes(ctx context.Context, cleanups []common.Executor) error {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
+	defer cancel()
+	var errs []error
+	for _, cleanup := range cleanups {
+		errs = append(errs, cleanup(ctx))
+	}
+	return errors.Join(errs...)
+}
+
 func (r *Runner) cloneEnvs() map[string]string {
 	// Reserve space for the per-task keys injected by run():
 	// ACTIONS_ID_TOKEN_REQUEST_URL, ACTIONS_ID_TOKEN_REQUEST_TOKEN, ACTIONS_RUNTIME_TOKEN,

internal/app/run/runner_idle_cleanup_test.go

@@ -311,6 +311,7 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 	}
 
 	var swept []string
+	var sweptVolumes []string
 	var sweptCutoff time.Time
 	origRemoveOrphanNetworks := removeOrphanNetworks
 	removeOrphanNetworks = func(_ context.Context, runnerUUID string, createdBefore time.Time) error {
@@ -319,15 +320,32 @@ func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
 		return nil
 	}
 	t.Cleanup(func() { removeOrphanNetworks = origRemoveOrphanNetworks })
+	origRemoveOrphanJobVolumes := removeOrphanJobVolumes
+	removeOrphanJobVolumes = func(_ context.Context, runnerUUID string, createdBefore time.Time) error {
+		sweptVolumes = append(sweptVolumes, runnerUUID)
+		assert.Equal(t, now.Add(-24*time.Hour), createdBefore)
+		return nil
+	}
+	t.Cleanup(func() { removeOrphanJobVolumes = origRemoveOrphanJobVolumes })
 
 	r := &Runner{uuid: "runner-1", cfg: cfg, now: func() time.Time { return now }}
 	r.OnIdle(context.Background())
 	assert.Equal(t, []string{"runner-1"}, swept)
+	assert.Equal(t, swept, sweptVolumes)
 	// a network of a job starting during the pass is younger than this and so out of scope
 	assert.Equal(t, now.Add(-24*time.Hour), sweptCutoff)
 
 	// a host-only runner has no daemon to sweep
+	origDockerReachable := dockerReachable
+	dockerReachable = func(context.Context) bool { return false }
+	t.Cleanup(func() { dockerReachable = origDockerReachable })
 	hostOnly := &Runner{uuid: "runner-2", cfg: &config.Config{Runner: cfg.Runner}, now: func() time.Time { return now }}
 	hostOnly.OnIdle(context.Background())
 	assert.Equal(t, []string{"runner-1"}, swept)
+	assert.Equal(t, swept, sweptVolumes)
+
+	dockerReachable = func(context.Context) bool { return true }
+	now = now.Add(time.Minute)
+	hostOnly.OnIdle(context.Background())
+	assert.Equal(t, []string{"runner-1", "runner-2"}, sweptVolumes)
 }

internal/app/run/runner_test.go

@@ -5,10 +5,18 @@ package run
 
 import (
 	"context"
+	"io"
 	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
 	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"gitea.com/gitea/runner/act/common"
 	"gitea.com/gitea/runner/act/runner"
 	clientmocks "gitea.com/gitea/runner/internal/pkg/client/mocks"
 	"gitea.com/gitea/runner/internal/pkg/config"
@@ -73,6 +81,113 @@ func TestRunnerRunningCountAndNullLogger(t *testing.T) {
 	require.NotNil(t, logger.Out)
 }
 
+func TestRunnerReclaimsVolumesAfterReporting(t *testing.T) {
+	for _, mode := range []string{"deferred", "post-task script"} {
+		t.Run(mode, func(t *testing.T) {
+			cfg := &config.Config{
+				Cache:     config.Cache{Enabled: new(false)},
+				Runner:    config.Runner{Timeout: time.Minute, LogReportInterval: time.Minute, StateReportInterval: time.Minute},
+				Container: config.Container{Network: "host", DockerHost: "-"},
+			}
+			if mode == "post-task script" {
+				if runtime.GOOS == "windows" {
+					t.Skip("uses a POSIX script")
+				}
+				cfg.Runner.PostTaskScript = filepath.Join(t.TempDir(), "post-task.sh")
+				require.NoError(t, os.WriteFile(cfg.Runner.PostTaskScript, []byte("#!/bin/sh\n: > \"$0.done\"\n"), 0o700))
+			}
+			cli := clientmocks.NewClient(t)
+			cli.AddressValue = "https://gitea.example/"
+			r := NewRunner(cfg, &config.Registration{UUID: "runner-1", Labels: []string{"ubuntu:docker://node:20"}}, cli)
+			var reported atomic.Bool
+			var removed atomic.Int64
+			cli.On("UpdateLog", mock.Anything, mock.Anything).Return(func(_ context.Context, req *connect.Request[runnerv1.UpdateLogRequest]) (*connect.Response[runnerv1.UpdateLogResponse], error) {
+				assert.False(t, reported.Load())
+				return connect.NewResponse(&runnerv1.UpdateLogResponse{AckIndex: req.Msg.Index + int64(len(req.Msg.Rows))}), nil
+			})
+			cli.On("UpdateTask", mock.Anything, mock.Anything).Return(func(_ context.Context, req *connect.Request[runnerv1.UpdateTaskRequest]) (*connect.Response[runnerv1.UpdateTaskResponse], error) {
+				if req.Msg.State.Result != runnerv1.Result_RESULT_UNSPECIFIED {
+					assert.Equal(t, runnerv1.Result_RESULT_SUCCESS, req.Msg.State.Result)
+					assert.Equal(t, int64(1), r.RunningCount())
+					if mode == "post-task script" {
+						assert.FileExists(t, cfg.Runner.PostTaskScript+".done")
+					}
+					reported.Store(true)
+				}
+				return connect.NewResponse(&runnerv1.UpdateTaskResponse{State: req.Msg.State}), nil
+			})
+			var volumesCreated atomic.Bool
+			daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+				writer.Header().Set("Content-Type", "application/json")
+				writer.Header().Set("API-Version", "1.47")
+				path := strings.TrimPrefix(request.URL.Path, "/v1.47")
+				if response, ok := map[string]string{
+					"/_ping":                  "OK",
+					"/info":                   `{"Architecture":"amd64","OSType":"linux"}`,
+					"/containers/json":        "[]",
+					"/networks":               "[]",
+					"/volumes":                `{"Volumes":[]}`,
+					"/containers/create":      `{"Id":"job-id"}`,
+					"/containers/job-id/json": `{"Id":"job-id","Config":{},"State":{"Status":"running"}}`,
+				}[path]; ok {
+					_, _ = io.WriteString(writer, response)
+					return
+				}
+				switch {
+				case strings.HasPrefix(path, "/images/"):
+					_, _ = io.WriteString(writer, `{"Id":"image-id","Config":{},"Os":"linux","Architecture":"amd64"}`)
+				case path == "/volumes/create":
+					volumesCreated.Store(true)
+					_, _ = io.WriteString(writer, "{}")
+				case strings.HasSuffix(path, "/exec"):
+					_, _ = io.WriteString(writer, `{"Id":"exec-id"}`)
+				case strings.HasPrefix(path, "/exec/"):
+					_, _ = io.WriteString(writer, `{"Running":false,"ExitCode":0}`)
+				case request.Method == http.MethodDelete || strings.HasSuffix(path, "/start") || strings.HasSuffix(path, "/kill") || strings.HasSuffix(path, "/archive"):
+					if request.Method == http.MethodDelete && strings.HasPrefix(path, "/volumes/") && volumesCreated.Load() {
+						assert.Equal(t, mode != "post-task script", reported.Load())
+						if mode == "post-task script" {
+							assert.NoFileExists(t, cfg.Runner.PostTaskScript+".done")
+						}
+						assert.Equal(t, int64(1), r.RunningCount())
+						removed.Add(1)
+					}
+					writer.WriteHeader(http.StatusNoContent)
+				default:
+					t.Errorf("unexpected Docker request: %s %s", request.Method, request.URL)
+					http.NotFound(writer, request)
+				}
+			}))
+			t.Cleanup(daemon.Close)
+			t.Setenv("DOCKER_HOST", daemon.URL)
+			require.NoError(t, r.Run(t.Context(), &runnerv1.Task{
+				Context:         &structpb.Struct{},
+				WorkflowPayload: []byte("jobs:\n  job:\n    runs-on: ubuntu\n    steps:\n      - run: exit 0\n        if: false\n"),
+			}))
+			assert.True(t, reported.Load())
+			assert.Equal(t, int64(2), removed.Load())
+			assert.Zero(t, r.RunningCount())
+		})
+	}
+}
+
+func TestCleanupJobVolumesJoinsErrorsAfterCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	err := cleanupJobVolumes(ctx, []common.Executor{
+		func(ctx context.Context) error {
+			require.NoError(t, ctx.Err())
+			deadline, ok := ctx.Deadline()
+			assert.True(t, ok)
+			assert.InDelta(t, time.Minute.Seconds(), time.Until(deadline).Seconds(), 1)
+			return io.EOF
+		},
+		func(context.Context) error { return io.ErrClosedPipe },
+	})
+	require.ErrorIs(t, err, io.EOF)
+	require.ErrorIs(t, err, io.ErrClosedPipe)
+}
+
 func TestNewRunnerInitializesLabelsAndEnvironment(t *testing.T) {
 	cacheEnabled := false
 	cfg := &config.Config{}

internal/pkg/config/config.example.yaml

@@ -53,11 +53,12 @@ runner:
   # scratch directories (left behind when a host cleanup delete stalls) older than
   # this duration. Setting either workdir_cleanup_age or idle_cleanup_interval to 0
   # (or any non-positive value) disables stale-directory cleanup entirely, along with
-  # the docker network cleanup below.
+  # the docker network and volume cleanup below.
   #workdir_cleanup_age: 24h
   # Cadence for the idle cleanup pass. Besides the directories above, on runners that use
-  # docker it removes the per-job networks of jobs this runner did not live to tear down,
-  # which would otherwise hold a subnet of the daemon address pool until the host is rebuilt.
+  # docker it removes the per-job networks and volumes of jobs this runner did not live to
+  # tear down, which would otherwise hold a subnet of the daemon address pool and the job's
+  # workspace contents until the host is rebuilt.
   #idle_cleanup_interval: 10m
   # The base interval for periodic log flush to the Gitea instance.
   # Logs may be sent earlier if the buffer reaches log_report_batch_size

internal/pkg/config/config.go

@@ -63,8 +63,8 @@ type Runner struct {
 	FetchTimeout          time.Duration     `yaml:"fetch_timeout"`            // FetchTimeout specifies the timeout duration for fetching resources.
 	FetchInterval         time.Duration     `yaml:"fetch_interval"`           // FetchInterval specifies the interval duration for fetching resources.
 	FetchIntervalMax      time.Duration     `yaml:"fetch_interval_max"`       // FetchIntervalMax specifies the maximum backoff interval when idle.
-	WorkdirCleanupAge     time.Duration     `yaml:"workdir_cleanup_age"`      // WorkdirCleanupAge removes stale bind-workdir task directories and orphaned host-mode scratch dirs older than this duration during idle cleanup.
-	IdleCleanupInterval   time.Duration     `yaml:"idle_cleanup_interval"`    // IdleCleanupInterval runs the idle cleanup (stale directories and orphaned docker networks) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
+	WorkdirCleanupAge     time.Duration     `yaml:"workdir_cleanup_age"`      // WorkdirCleanupAge removes stale bind-workdir task directories, orphaned host-mode scratch dirs and orphaned docker job resources older than this duration during idle cleanup.
+	IdleCleanupInterval   time.Duration     `yaml:"idle_cleanup_interval"`    // IdleCleanupInterval runs the idle cleanup (stale directories and orphaned docker networks and volumes) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
 	LogReportInterval     time.Duration     `yaml:"log_report_interval"`      // LogReportInterval specifies the base interval for periodic log flush.
 	LogReportMaxLatency   time.Duration     `yaml:"log_report_max_latency"`   // LogReportMaxLatency specifies the max time a log row can wait before being sent.
 	LogReportBatchSize    int               `yaml:"log_report_batch_size"`    // LogReportBatchSize triggers immediate log flush when buffer reaches this size.