#!/usr/bin/env python3
"""Rank the tests of a run made with ZIT_TEST_TIMES=DIR.
The test runner (src/test_runner.zig) writes DIR/<pid>, one file per test
process, one line per test:
NANOSECONDS <TAB> TEARDOWN_NANOSECONDS <TAB> STATUS <TAB> NAME
Usage:
scripts/test-times.py DIR... [--top N] [--wall SECONDS]
scripts/test-times.py DIR... --weights [--min SECONDS] > src/testing/test_weights.txt
The report gives each process's total (one process is one shard of one test
binary, so an unbalanced shard shows here), the N slowest tests, the total
per source file, and the share of the time the slowest 5% take. --wall, the
step's wall time, adds how much of it ran in parallel.
--weights prints the tests slower than --min seconds (default 2) as
`NAME <TAB> SECONDS`, sorted by name: the file src/testing/test_shard.zig packs
shards by. Give it the directories of several runs (e2e and the unit step,
taken at different loads); a test seen in several keeps its slowest time.
"""
import argparse
import os
import sys
def read(directories):
rows = []
many = len(directories) > 1
for directory, pid in sorted((d, p) for d in directories for p in os.listdir(d)):
with open(os.path.join(directory, pid), errors="replace") as f:
for line in f:
parts = line.rstrip("\n").split("\t", 3)
if len(parts) != 4:
continue
process = f"{os.path.basename(os.path.normpath(directory))}/{pid}" if many else pid
rows.append((int(parts[0]) / 1e9, int(parts[1]) / 1e9, parts[2], parts[3], process))
return rows
def source_of(name):
# A test is named after its module path: `e2e.pull.test.cli pull: ...`.
at = name.find(".test.")
return name[:at] if at >= 0 else name.split(".")[0]
def report(rows, top, wall):
total = sum(r[0] for r in rows)
teardown = sum(r[1] for r in rows)
line = f"{len(rows)} tests, {total:.1f} s summed, {teardown:.2f} s of it teardown"
if wall:
line += f", {wall:.1f} s wall, {total / wall:.2f} running at once"
print(line)
print("\nper process")
for pid in sorted({r[4] for r in rows}):
mine = [r for r in rows if r[4] == pid]
print(f" {pid:>8} {len(mine):5} tests {sum(r[0] for r in mine):8.1f} s slowest {max(r[0] for r in mine):7.1f} s")
ranked = sorted(rows, key=lambda r: -r[0])
print(f"\n{top} slowest")
for r in ranked[:top]:
status = "" if r[2] == "pass" else f" [{r[2]}]"
print(f" {r[0]:8.2f} s {r[3]}{status}")
per_file = {}
for r in rows:
entry = per_file.setdefault(source_of(r[3]), [0, 0.0])
entry[0] += 1
entry[1] += r[0]
print("\nper file")
for name, (count, seconds) in sorted(per_file.items(), key=lambda kv: -kv[1][1]):
print(f" {seconds:8.1f} s {count:5} tests {name}")
slowest = ranked[: max(1, len(rows) // 20)]
share = sum(r[0] for r in slowest) / total * 100 if total else 0
print(f"\nslowest 5% ({len(slowest)} tests) take {share:.0f}% of the summed time")
for limit in (5, 2, 1, 0.5):
over = [r[0] for r in rows if r[0] > limit]
print(f" over {limit:>3} s: {len(over):5} tests, {sum(over) / total * 100 if total else 0:3.0f}%")
def weights(rows, minimum):
slowest = {}
for r in rows:
slowest[r[3]] = max(slowest.get(r[3], 0.0), r[0])
print("# The tests slower than %g s, as NAME<TAB>SECONDS; src/testing/test_shard.zig" % minimum)
print("# spreads them across shards by weight. Regenerate from fresh runs with")
print("# scripts/test-times.py DIR... --weights; a stale line is harmless.")
for name in sorted(n for n, s in slowest.items() if s > minimum):
if "\t" in name or "\n" in name:
continue
print(f"{name}\t{slowest[name]:.1f}")
def main():
parser = argparse.ArgumentParser(description="Rank the tests of a ZIT_TEST_TIMES run.")
parser.add_argument("dirs", nargs="+", metavar="DIR")
parser.add_argument("--top", type=int, default=50)
parser.add_argument("--wall", type=float)
parser.add_argument("--weights", action="store_true")
parser.add_argument("--min", type=float, default=2.0)
args = parser.parse_args()
rows = read(args.dirs)
if not rows:
sys.exit(f"no test times in {' '.join(args.dirs)}")
if args.weights:
weights(rows, args.min)
else:
report(rows, args.top, args.wall)
if __name__ == "__main__":
main()