Skip to content
quarry Artifact [8920b2172b]
Login

Artifact [8920b2172b]

Artifact 8920b2172b44c5600b378770ee11475ff638997672960bfe52864868556a8a1a:


package runner

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// JobKey is the step metadata key a compiled workflow job travels under
// (quarry RFC 0019; dagnats#728, v0.0.21, whose jobPayload is the wire
// shape Job and JobStep read). The engine copies a step's metadata to
// the task it dispatches, so a job's commands reach the runner with the
// task and no ci/runner.json is read for it.
const JobKey = "ci.job"

// Job is one workflow job: its steps run in order, in the check-in's
// tree, and the first that fails ends it.
type Job struct {
	Name             string            `json:"name,omitempty"`
	Env              map[string]string `json:"env,omitempty"`
	WorkingDirectory string            `json:"working-directory,omitempty"`
	Environment      string            `json:"environment,omitempty"`
	// Secrets names the secrets every step gets, never their values,
	// each exported under its own name; delivering them is RFC 0019
	// rung 5.
	Secrets []string  `json:"secrets,omitempty"`
	Steps   []JobStep `json:"steps"`
}

// JobStep is one `run:` of a job.
type JobStep struct {
	Name             string            `json:"name,omitempty"`
	Run              string            `json:"run"`
	Env              map[string]string `json:"env,omitempty"`
	WorkingDirectory string            `json:"working-directory,omitempty"`
	// Secrets names the secrets this step alone gets, as in Actions.
	Secrets []string `json:"secrets,omitempty"`
}

// secrets lists every secret the job asks for, the job's and its steps'.
func (j Job) secrets() []string {
	seen := map[string]bool{}
	var out []string
	for _, n := range j.Secrets {
		if !seen[n] {
			seen[n] = true
			out = append(out, n)
		}
	}
	for _, s := range j.Steps {
		for _, n := range s.Secrets {
			if !seen[n] {
				seen[n] = true
				out = append(out, n)
			}
		}
	}
	return out
}

// ParseJob reads a job from its metadata value, refusing one that could
// not run as written: no steps, a step with nothing to run, or a working
// directory that leaves the tree.
func ParseJob(raw string) (Job, error) {
	var j Job
	if err := json.Unmarshal([]byte(raw), &j); err != nil {
		return Job{}, fmt.Errorf("the task's job is not valid JSON: %v", err)
	}
	if len(j.Steps) == 0 {
		return Job{}, errors.New("the task's job has no steps")
	}
	if err := localDir(j.WorkingDirectory); err != nil {
		return Job{}, fmt.Errorf("the job's %v", err)
	}
	for i, s := range j.Steps {
		if strings.TrimSpace(s.Run) == "" {
			return Job{}, fmt.Errorf("step %d (%s) has nothing to run", i+1, s.label(i))
		}
		if err := localDir(s.WorkingDirectory); err != nil {
			return Job{}, fmt.Errorf("step %d (%s): %v", i+1, s.label(i), err)
		}
	}
	return j, nil
}

func localDir(d string) error {
	if d != "" && !filepath.IsLocal(filepath.FromSlash(d)) {
		return fmt.Errorf("working-directory %q leaves the tree", d)
	}
	return nil
}

// label names a step in the log: its name, or the first line of its run.
func (s JobStep) label(i int) string {
	if s.Name != "" {
		return s.Name
	}
	first, _, _ := strings.Cut(strings.TrimSpace(s.Run), "\n")
	if first == "" {
		return fmt.Sprintf("step %d", i+1)
	}
	return first
}

// runJob runs a job's steps in the tree at root. Each step is a shell
// script run with `sh -e`, as Actions runs `run:` with `-e`: the first
// failing command ends the step, and the first failing step ends the job.
// run is what the run itself tells every step (QUARRY_TAG); the job's
// and the step's env come after it.
func runJob(ctx context.Context, j Job, root string, run, secrets map[string]string, start starter, logs, errs io.Writer) error {
	printEnvironment(ctx, j, root, run, secrets, start, logs, errs)
	for i, s := range j.Steps {
		fmt.Fprintf(logs, "$ %s\n", s.label(i))
		// A step's working-directory replaces the job's, as in Actions;
		// both are relative to the tree's root.
		dir := filepath.Join(root, filepath.FromSlash(j.WorkingDirectory))
		if s.WorkingDirectory != "" {
			dir = filepath.Join(root, filepath.FromSlash(s.WorkingDirectory))
		}
		// Secrets as Actions scopes them: the job's to every step, a
		// step's to that step alone, each under its own name.
		cmd := start(ctx, dir, []string{"sh", "-e", "-c", s.Run}, run, j.Env, s.Env, pick(secrets, j.Secrets), pick(secrets, s.Secrets))
		cmd.Stdout, cmd.Stderr = logs, errs
		if err := cmd.Run(); err != nil {
			return fmt.Errorf("step %d (%s): %w", i+1, s.label(i), err)
		}
	}
	return nil
}

func pick(values map[string]string, names []string) map[string]string {
	out := make(map[string]string, len(names))
	for _, n := range names {
		if v, ok := values[n]; ok {
			out[n] = v
		}
	}
	return out
}

// envList layers env maps (later wins), sorted so a command's
// environment is built the same way every run.
func envList(maps ...map[string]string) []string {
	merged := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			merged[k] = v
		}
	}
	keys := make([]string, 0, len(merged))
	for k := range merged {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	out := make([]string, 0, len(keys))
	for _, k := range keys {
		out = append(out, k+"="+merged[k])
	}
	return out
}

// EnvBegin and EnvEnd delimit the environment block at the top of every
// job's log, so a reader or the API can skip it.
const (
	EnvBegin = "::quarry-job-environment::"
	EnvEnd   = "::end-quarry-job-environment::"
)

// printEnvironment writes the job's environment to the log before its
// first step, by running a small script through the same starter a step
// uses, so it reports what the steps really see: user, umask, directory,
// PATH, the names (never the values) of the variables, and tool
// versions. It never fails the job: a script error is only logged.
func printEnvironment(ctx context.Context, j Job, root string, run, secrets map[string]string, start starter, logs, errs io.Writer) {
	dir := filepath.Join(root, filepath.FromSlash(j.WorkingDirectory))
	if st, err := os.Stat(dir); err != nil || !st.IsDir() {
		dir = root
	}
	masked := []string{"QUARRY_TOKEN"}
	masked = append(masked, j.secrets()...)
	cmd := start(ctx, dir, []string{"sh", "-c", envScript(masked)}, run, j.Env, nil, pick(secrets, j.Secrets), nil)
	cmd.Stdout, cmd.Stderr = logs, errs
	if err := cmd.Run(); err != nil {
		fmt.Fprintf(logs, "the environment report failed: %v\n%s\n", err, EnvEnd)
	}
}

// envScript is the report as a shell script. maskedNames are the secret
// names; only names made of word characters are embedded, so none can
// break out of the case pattern.
func envScript(maskedNames []string) string {
	var pat []string
	for _, n := range maskedNames {
		if n != "" && strings.Trim(n, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_") == "" {
			pat = append(pat, n)
		}
	}
	masked := "NONE_"
	if len(pat) > 0 {
		masked = strings.Join(pat, "|")
	}
	return `echo '` + EnvBegin + `'
echo "user: $(id -un 2>/dev/null) (uid $(id -u 2>/dev/null))"
echo "umask: $(umask)"
echo "directory: $(pwd)"
echo "PATH: $PATH"
echo "limits: open files $(ulimit -n 2>/dev/null) (hard $(ulimit -Hn 2>/dev/null)), processes $(ulimit -u 2>/dev/null), stack $(ulimit -s 2>/dev/null) KB, core $(ulimit -c 2>/dev/null)"
echo "variables (names only):"
env | grep -E '^[A-Za-z_][A-Za-z0-9_]*=' | cut -d= -f1 | sort | while read -r n; do
  case "$n" in
    ` + masked + `) echo "  $n (masked)";;
    *) echo "  $n";;
  esac
done
probe() {
  name=$1; shift
  if ! command -v "$1" >/dev/null 2>&1; then echo "  $name: absent"; return 0; fi
  if command -v timeout >/dev/null 2>&1; then v=$(timeout 2 "$@" 2>&1 </dev/null | head -n 1); else v=$("$@" 2>&1 </dev/null | head -n 1); fi
  echo "  $name: ${v:-present}"
}
echo "tools:"
probe go go version
probe zig zig version
probe gcc gcc --version
probe cc cc --version
probe make make --version
probe fossil fossil version
probe zit zit version
probe git git --version
probe node node --version
probe python3 python3 --version
echo '` + EnvEnd + `'
true
`
}