package runner
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
)
// Per-task temp directories (#317). A task killed before `go test`
// removed its go-build directories left them in the host's /tmp, a
// tmpfs that filled until NATS refused streams. Each task now gets a
// directory of its own under the runner's TempRoot, handed to it as
// TMPDIR and GOTMPDIR, and the runner removes that directory when the
// task ends, after reclaim has ended whatever the task left running.
//
// The path is short whatever WorkDir is (#385). A task's programs make
// Unix sockets under TMPDIR, and a socket's path has room for 108 bytes
// on Linux (104 on macOS): Chrome puts SingletonSocket in a directory of
// its own beneath it, and under /srv/ci-pikchr-studio/runner's
// quarry-runner-tmp/quarry-task-<pid>-<random> that came to 117 and
// every browser test aborted. The default root is /var/tmp/qr-<hash>,
// /var/tmp because it is disk-backed where /tmp is often a tmpfs (the
// #317 exhaustion), and named for WorkDir so it is stable across
// restarts and distinct per runner; a task dir is t<pid>-<random>.
// tempBase is where default roots go: disk-backed by the FHS and kept
// across reboots, unlike a tmpfs /tmp. A variable so the package's
// tests keep their roots out of the real /var/tmp.
var tempBase = "/var/tmp"
const (
// legacyRootName is the root's name under WorkDir before #385; a
// sweep still clears what dead runners left there.
legacyRootName = "quarry-runner-tmp"
// taskPrefix starts every task dir's name, followed by the runner's
// pid, so a sweep knows its own dirs and which runner owns each.
taskPrefix = "t"
// legacyTaskPrefix is the task dir prefix before #385.
legacyTaskPrefix = "quarry-task-"
)
// defaultTempRoot is the short root for a runner working in workDir.
func defaultTempRoot(workDir string) string {
abs, err := filepath.Abs(workDir)
if err != nil {
abs = workDir
}
sum := sha256.Sum256([]byte(abs))
return filepath.Join(tempBase, "qr-"+hex.EncodeToString(sum[:4]))
}
// sweepTemp makes the runner's temp root and removes the task dirs a
// previous life of this runner left: those named taskPrefix<pid>-*
// whose pid is this process's (a restart in a container is pid 1 again)
// or no longer runs, so another runner sharing the work dir keeps its
// live ones. Every other entry is left alone, and a root that is a
// symlink or not a directory is refused, never followed.
func (r *Runner) sweepTemp() error {
root := r.cfg.TempRoot
if err := os.MkdirAll(filepath.Dir(root), 0o755); err != nil {
return err
}
if err := os.Mkdir(root, 0o700); err != nil && !os.IsExist(err) {
return err
}
info, err := os.Lstat(root)
if err != nil {
return err
}
if !info.IsDir() {
return fmt.Errorf("runner: temp root %s is a %s, not a directory", root, info.Mode().Type())
}
// The default root sits in a world-writable directory, where
// another account could make it first and wait for a task's files.
if uid, _, ok := ownerOf(info); ok && uid != os.Getuid() {
return fmt.Errorf("runner: temp root %s belongs to uid %d, not this runner", root, uid)
}
if r.users != nil {
// A job user must be able to reach its own directory under it.
if err := os.Chmod(root, 0o711); err != nil {
return err
}
}
r.sweepDir(root, taskPrefix)
// Task dirs a runner from before #385 left under WorkDir.
if legacy := filepath.Join(r.cfg.WorkDir, legacyRootName); legacy != root {
if li, err := os.Lstat(legacy); err == nil && li.IsDir() {
r.sweepDir(legacy, legacyTaskPrefix)
}
}
return nil
}
// sweepDir removes the task dirs named prefix<pid>-* in dir whose
// runner is this process or no longer runs.
func (r *Runner) sweepDir(dir, prefix string) {
es, err := os.ReadDir(dir)
if err != nil {
return
}
for _, e := range es {
pid, ok := taskPid(e.Name(), prefix)
if !ok || !e.IsDir() || (pid != os.Getpid() && pidAlive(pid)) {
continue
}
if r.users != nil {
r.users.handBack(filepath.Join(dir, e.Name()))
}
if err := removeAll(filepath.Join(dir, e.Name())); err != nil {
r.cfg.Logf("stale temp dir %s was not removed: %v", e.Name(), err)
}
}
}
// taskPid reads the pid out of a task dir's name.
func taskPid(name, prefix string) (int, bool) {
rest, ok := strings.CutPrefix(name, prefix)
if !ok {
return 0, false
}
n, _, _ := strings.Cut(rest, "-")
pid, err := strconv.Atoi(n)
return pid, err == nil && pid > 0
}
// taskTemp makes one task's temp directory.
func (r *Runner) taskTemp() (string, error) {
if err := os.MkdirAll(r.cfg.TempRoot, 0o700); err != nil {
return "", err
}
return os.MkdirTemp(r.cfg.TempRoot, taskPrefix+strconv.Itoa(os.Getpid())+"-")
}
// withTemp points every temp variable a command reads at tmp. It comes
// after whatever the task set, so the per-task directory overrides a
// TMPDIR in a job's or step's env: a step cannot send its scratch space
// back to the host's /tmp.
func withTemp(next starter, tmp string) starter {
temp := map[string]string{"TMPDIR": tmp, "GOTMPDIR": tmp, "TMP": tmp, "TEMP": tmp}
return func(ctx context.Context, dir string, argv []string, env ...map[string]string) *exec.Cmd {
return next(ctx, dir, argv, append(env, temp)...)
}
}
// removeAll removes dir, making read-only directories writable first
// where the plain removal is refused.
func removeAll(dir string) error {
err := os.RemoveAll(dir)
if err == nil {
return nil
}
filepath.WalkDir(dir, func(p string, d os.DirEntry, _ error) error {
if d != nil && d.IsDir() {
os.Chmod(p, 0o700)
}
return nil
})
return os.RemoveAll(dir)
}
// handBackWorkspaces has the job users hand back every workspace, so a
// job that a stopped runner killed does not leave one the runner cannot
// check out into (#419). Only real directories under ws are touched.
func (r *Runner) handBackWorkspaces() {
if r.users == nil {
return
}
ws := filepath.Join(r.cfg.WorkDir, "ws")
es, err := os.ReadDir(ws)
if err != nil {
return
}
for _, e := range es {
if e.Type().IsDir() {
r.users.handBack(filepath.Join(ws, e.Name()))
}
}
}