package runner
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"os/exec"
"os/user"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
)
// Running jobs as other users (RFC 0019 §isolation).
//
// A runner that may receive secrets must not run a job as itself: any
// process of the runner's user can read the runner's environment, and
// with it the worker token (/proc/<pid>/environ), and any job can read
// a concurrent job's environment, and with it that job's secrets. So a
// runner given --job-users runs every task as one of those users, a
// pool of unprivileged accounts it may only become through sudo:
//
// ciworker ALL=(qjob1,qjob2,qjob3) NOPASSWD: ALL
//
// Nothing runs as root. Each pool user has a group of its own, the
// runner's user is a member of each, and a job's checkout is shared
// with that group alone, so concurrent jobs can neither read nor touch
// each other's trees. The environment, secrets included, is handed to
// the job over stdin to `quarry job-env`, never on a command line,
// which any user on the machine can read.
// slots is the pool of job users a runner hands out, one per running
// task, so no two tasks ever share a user.
type slots struct {
free chan string
self string // this program, whose job-env subcommand starts each command
}
func newSlots(users []string, self string) (*slots, error) {
if len(users) == 0 {
return nil, nil
}
if runtime.GOOS == "windows" {
return nil, errors.New("runner: --job-users needs Unix accounts and sudo")
}
if self == "" {
return nil, errors.New("runner: job users need this program's path, for job-env")
}
s := &slots{free: make(chan string, len(users)), self: self}
seen := map[string]bool{}
for _, u := range users {
if u == "" || seen[u] {
return nil, fmt.Errorf("runner: job user %q is empty or repeated", u)
}
seen[u] = true
// The probe is a real reclaim: it proves the runner can end
// what a job leaves behind before any job runs, and ends what
// a runner that died left running.
if err := s.endAll(u); err != nil {
return nil, fmt.Errorf("runner: --job-users: %v", err)
}
s.free <- u
}
return s, nil
}
// share gives a slot's group the checkout at dir: group-owned, group
// read-write, setgid directories, nothing for anyone else.
func share(dir, name string) error {
g, err := user.LookupGroup(name)
if err != nil {
return fmt.Errorf("job user %s has no group of its own: %v", name, err)
}
gid, err := strconv.Atoi(g.Gid)
if err != nil {
return err
}
return filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if err := os.Lchown(p, -1, gid); err != nil {
return err
}
if d.Type()&fs.ModeSymlink != 0 {
return nil
}
info, err := d.Info()
if err != nil {
return err
}
mode := info.Mode().Perm()&0o770 | 0o060
if d.IsDir() {
mode |= 0o010 | fs.ModeSetgid
}
return os.Chmod(p, mode)
})
}
// jobEnv is the whole environment a job starts with: nothing of the
// runner's own but PATH and LANG, so neither its token nor anything
// else of its reaches the job.
func jobEnv(dir string, extra ...map[string]string) []string {
env := map[string]string{
"PATH": os.Getenv("PATH"),
"HOME": dir,
"TMPDIR": filepath.Join(dir, ".quarry-tmp"),
"CI": "true",
}
if l := os.Getenv("LANG"); l != "" {
env["LANG"] = l
}
for _, m := range extra {
for k, v := range m {
env[k] = v
}
}
keys := make([]string, 0, len(env))
for k := range env {
keys = append(keys, k)
}
sort.Strings(keys)
out := make([]string, 0, len(keys))
for _, k := range keys {
out = append(out, k+"="+env[k])
}
return out
}
// encodeEnv is job-env's stdin: NUL-separated KEY=VALUE, the one
// encoding a value with newlines in it (a private key) survives.
func encodeEnv(env []string) []byte {
var b bytes.Buffer
for _, kv := range env {
b.WriteString(kv)
b.WriteByte(0)
}
return b.Bytes()
}
// JobEnvMain is `quarry job-env -- COMMAND [ARG...]`: read the
// environment from stdin (NUL-separated KEY=VALUE), then become COMMAND
// with exactly that environment and a umask that keeps what it writes
// from everyone outside its group. It runs as the job user, started by
// the runner through sudo.
func JobEnvMain(args []string) error {
if len(args) < 2 || args[0] != "--" {
return errors.New("usage: job-env -- COMMAND [ARG...] (the environment on stdin, NUL-separated)")
}
raw, err := io.ReadAll(io.LimitReader(os.Stdin, 8<<20))
if err != nil {
return err
}
var env []string
for _, kv := range bytes.Split(raw, []byte{0}) {
if len(kv) == 0 {
continue
}
if !bytes.ContainsRune(kv, '=') {
return errors.New("job-env: an environment entry has no '='")
}
env = append(env, string(kv))
}
path, err := lookPath(args[1], env)
if err != nil {
return err
}
if dir := envValue(env, "TMPDIR"); dir != "" {
os.MkdirAll(dir, 0o770)
}
// stdin has been read to its end, so the command finds it empty.
return become(path, args[1:], env)
}
func envValue(env []string, key string) string {
for _, kv := range env {
if k, v, _ := strings.Cut(kv, "="); k == key {
return v
}
}
return ""
}
// lookPath finds a command on the PATH the job will have, not the one
// sudo gave job-env.
func lookPath(name string, env []string) (string, error) {
if strings.Contains(name, "/") {
return name, nil
}
for _, d := range filepath.SplitList(envValue(env, "PATH")) {
p := filepath.Join(d, name)
if st, err := os.Stat(p); err == nil && !st.IsDir() && st.Mode()&0o111 != 0 {
return p, nil
}
}
return "", fmt.Errorf("job-env: %s is not on the job's PATH", name)
}
// masker replaces every secret value with *** in what passes through it,
// a line at a time, so a value split across two writes is still caught.
// A multi-line value is masked line by line as well. It is a courtesy:
// a job can always print a secret encoded, and the protection is that
// unreviewed code never receives one.
type masker struct {
mu sync.Mutex
w io.Writer
values []string
buf []byte
}
func newMasker(w io.Writer, secrets map[string]string) io.WriteCloser {
var vals []string
for _, v := range secrets {
for _, part := range append([]string{v}, strings.Split(v, "\n")...) {
if part = strings.TrimSpace(part); len(part) >= 4 {
vals = append(vals, part)
}
}
}
sort.Slice(vals, func(i, j int) bool { return len(vals[i]) > len(vals[j]) })
return &masker{w: w, values: vals}
}
func (m *masker) Write(p []byte) (int, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.buf = append(m.buf, p...)
if i := bytes.LastIndexByte(m.buf, '\n'); i >= 0 {
if err := m.emit(m.buf[:i+1]); err != nil {
return 0, err
}
m.buf = append(m.buf[:0], m.buf[i+1:]...)
} else if len(m.buf) > 64<<10 {
if err := m.emit(m.buf); err != nil {
return 0, err
}
m.buf = m.buf[:0]
}
return len(p), nil
}
func (m *masker) emit(b []byte) error {
s := string(b)
for _, v := range m.values {
s = strings.ReplaceAll(s, v, "***")
}
_, err := io.WriteString(m.w, s)
return err
}
// Close emits what is left after the last newline.
func (m *masker) Close() error {
m.mu.Lock()
defer m.mu.Unlock()
if len(m.buf) == 0 {
return nil
}
err := m.emit(m.buf)
m.buf = nil
return err
}
// starter makes the command for one argv in dir, with env layered in
// order (later maps win).
type starter func(ctx context.Context, dir string, argv []string, env ...map[string]string) *exec.Cmd
// plainStarter runs as the runner itself, with its environment, which is
// how a runner without job users has always run tasks. CI=true is added
// as jobEnv adds it, so a test that needs a tool CI provides can tell CI
// from a developer's machine whichever way the runner was started; the
// task's own env comes after it and wins (exec keeps the last value).
func plainStarter(ctx context.Context, dir string, argv []string, env ...map[string]string) *exec.Cmd {
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
cmd.Dir = dir
cmd.Env = append(append(os.Environ(), "CI=true"), envList(env...)...)
stopTogether(cmd)
return cmd
}
// starter runs argv as the slot's user, through sudo and job-env, with
// only the environment jobEnv builds; the values go over stdin.
func (s *slots) starter(slot, root string) starter {
return func(ctx context.Context, dir string, argv []string, env ...map[string]string) *exec.Cmd {
args := append([]string{"-n", "-u", slot, "--", s.self, "job-env", "--"}, argv...)
cmd := exec.CommandContext(ctx, "sudo", args...)
cmd.Dir = dir
cmd.Env = []string{"PATH=" + os.Getenv("PATH")}
cmd.Stdin = bytes.NewReader(encodeEnv(jobEnv(root, env...)))
// The job runs as the slot's user, whom the runner may not
// signal; reclaim ends its processes after the task either way.
stopTogether(cmd)
return cmd
}
}
// reclaim ends whatever the job left running (it may have started a
// server and walked away, or outlived a timeout that killed only sudo)
// and makes everything it wrote group-writable, so the runner can clear
// the tree for the next check-in. Best-effort: a failure here shows up
// as the next checkout's error, naming the tree.
//
// The kill is this program's own job-reclaim, run as the slot, so it
// needs nothing installed (slim images have no pkill, jva #1620). An
// error means the slot's processes may still be running: the caller
// keeps the slot out of the pool rather than start a job beside them.
func (s *slots) reclaim(dir, slot string) error {
if err := s.endAll(slot); err != nil {
return err
}
if dir != "" {
exec.Command("sudo", "-n", "-u", slot, "--", "chmod", "-R", "g+rwX", dir).Run()
}
return nil
}
// endAll ends every process the slot's user is running.
func (s *slots) endAll(slot string) error {
out, err := exec.Command("sudo", "-n", "-u", slot, "--", s.self, "job-reclaim", slot).CombinedOutput()
if err != nil {
return fmt.Errorf("ending job user %s's processes: %v: %s", slot, err, strings.TrimSpace(string(out)))
}
return nil
}
// JobReclaimMain is `job-reclaim SLOT`: run as SLOT through sudo, it
// kills every process SLOT's user runs but itself. It refuses to run as
// root, where the same call would end the machine, and as anyone but
// the user it names, so a mistyped sudo cannot aim it elsewhere.
func JobReclaimMain(args []string) error {
if len(args) != 1 || args[0] == "" {
return errors.New("usage: job-reclaim SLOT (run as SLOT)")
}
if os.Geteuid() == 0 {
return errors.New("job-reclaim refuses to run as root")
}
u, err := user.Current()
if err != nil {
return err
}
if u.Username != args[0] {
return fmt.Errorf("job-reclaim runs as %s, not %s", u.Username, args[0])
}
return endOwn()
}
// fetchSecrets asks the forge for the values a job names (RFC 0019
// §secrets). The forge checks this runner holds the task and decides
// from its own records; a refusal comes back in words and fails the
// task before anything runs.
func (r *Runner) fetchSecrets(ctx context.Context, taskID, environment string, names []string) (map[string]string, error) {
body, _ := json.Marshal(map[string]any{"environment": environment, "names": names})
resp, err := r.post(ctx, "/v1/tasks/"+url.PathEscape(taskID)+"/secrets", body)
if err != nil {
return nil, fmt.Errorf("the forge gave no secrets: %w", err)
}
defer resp.Body.Close()
var out struct {
Secrets map[string]string `json:"secrets"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&out); err != nil {
return nil, fmt.Errorf("the forge's secrets could not be read: %v", err)
}
for _, n := range names {
if _, ok := out.Secrets[n]; !ok {
return nil, fmt.Errorf("the forge did not send %s", n)
}
}
return out.Secrets, nil
}