Skip to content
zit Artifact [675cc6aa66]
Login

Artifact [675cc6aa66]

Artifact 675cc6aa663e6ad6c9e59b426491242e6244186f91e310f4175fed9454f13823:


#!/usr/bin/env python3
"""Rank the tests of a run made with ZIT_TEST_TIMES=DIR.

The test runner (src/test_runner.zig) writes DIR/<pid>, one file per test
process, one line per test:

    NANOSECONDS <TAB> TEARDOWN_NANOSECONDS <TAB> STATUS <TAB> NAME

Usage:
    scripts/test-times.py DIR [--top N] [--wall SECONDS]
    scripts/test-times.py DIR --weights [--min SECONDS]

The report gives each process's total (one process is one shard of one test
binary, so an unbalanced shard shows here), the N slowest tests, the total
per source file, and the share of the time the slowest 5% take. --wall, the
step's wall time, adds how much of it ran in parallel.

--weights prints the tests slower than --min seconds (default 2) as
`NAME <TAB> SECONDS`, sorted by name, which is the format the runner's
weighted sharding reads. A test seen in several runs keeps its slowest time.
"""

import argparse
import os
import sys


def read(directory):
    rows = []
    for pid in sorted(os.listdir(directory)):
        with open(os.path.join(directory, pid), errors="replace") as f:
            for line in f:
                parts = line.rstrip("\n").split("\t", 3)
                if len(parts) != 4:
                    continue
                rows.append((int(parts[0]) / 1e9, int(parts[1]) / 1e9, parts[2], parts[3], pid))
    return rows


def source_of(name):
    # A test is named after its module path: `e2e.pull.test.cli pull: ...`.
    at = name.find(".test.")
    return name[:at] if at >= 0 else name.split(".")[0]


def report(rows, top, wall):
    total = sum(r[0] for r in rows)
    teardown = sum(r[1] for r in rows)
    line = f"{len(rows)} tests, {total:.1f} s summed, {teardown:.2f} s of it teardown"
    if wall:
        line += f", {wall:.1f} s wall, {total / wall:.2f} running at once"
    print(line)

    print("\nper process")
    for pid in sorted({r[4] for r in rows}):
        mine = [r for r in rows if r[4] == pid]
        print(f"  {pid:>8}  {len(mine):5} tests  {sum(r[0] for r in mine):8.1f} s  slowest {max(r[0] for r in mine):7.1f} s")

    ranked = sorted(rows, key=lambda r: -r[0])
    print(f"\n{top} slowest")
    for r in ranked[:top]:
        status = "" if r[2] == "pass" else f" [{r[2]}]"
        print(f"  {r[0]:8.2f} s  {r[3]}{status}")

    per_file = {}
    for r in rows:
        entry = per_file.setdefault(source_of(r[3]), [0, 0.0])
        entry[0] += 1
        entry[1] += r[0]
    print("\nper file")
    for name, (count, seconds) in sorted(per_file.items(), key=lambda kv: -kv[1][1]):
        print(f"  {seconds:8.1f} s  {count:5} tests  {name}")

    slowest = ranked[: max(1, len(rows) // 20)]
    share = sum(r[0] for r in slowest) / total * 100 if total else 0
    print(f"\nslowest 5% ({len(slowest)} tests) take {share:.0f}% of the summed time")
    for limit in (5, 2, 1, 0.5):
        over = [r[0] for r in rows if r[0] > limit]
        print(f"  over {limit:>3} s: {len(over):5} tests, {sum(over) / total * 100 if total else 0:3.0f}%")


def weights(rows, minimum):
    slowest = {}
    for r in rows:
        slowest[r[3]] = max(slowest.get(r[3], 0.0), r[0])
    for name in sorted(n for n, s in slowest.items() if s > minimum):
        if "\t" in name or "\n" in name:
            continue
        print(f"{name}\t{slowest[name]:.1f}")


def main():
    parser = argparse.ArgumentParser(description="Rank the tests of a ZIT_TEST_TIMES run.")
    parser.add_argument("dir")
    parser.add_argument("--top", type=int, default=50)
    parser.add_argument("--wall", type=float)
    parser.add_argument("--weights", action="store_true")
    parser.add_argument("--min", type=float, default=2.0)
    args = parser.parse_args()
    rows = read(args.dir)
    if not rows:
        sys.exit(f"no test times in {args.dir}")
    if args.weights:
        weights(rows, args.min)
    else:
        report(rows, args.top, args.wall)


if __name__ == "__main__":
    main()