34 lines
1.4 KiB
Python
34 lines
1.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Re-audit every bench run log by its SCRIPT-LEVEL step lines (ground truth:
|
|
# "STEP <label> rx=<got>/<want> OK|QUIET"). probe_classify verdicts are advisory
|
|
# only; a real freeze shows a short/QUIET step here.
|
|
# ASCII output only.
|
|
import glob, os, re
|
|
|
|
def audit(path):
|
|
steps = []
|
|
for line in open(path, encoding="utf-8-sig", errors="replace"):
|
|
m = re.match(r"STEP (\S+) rx=(\d+)/(\d+) (OK|QUIET)", line.strip())
|
|
if m:
|
|
steps.append((m.group(1), int(m.group(2)), int(m.group(3)), m.group(4)))
|
|
bad = [s for s in steps if s[3] != "OK" or s[1] != s[2]]
|
|
return steps, bad
|
|
|
|
groups = {}
|
|
for path in sorted(glob.glob("MDK-ARM/ab_*.log")) + sorted(glob.glob("MDK-ARM/reg_*.log")):
|
|
name = os.path.basename(path)
|
|
key = re.sub(r"_run\d+\.log$", "", name)
|
|
steps, bad = audit(path)
|
|
groups.setdefault(key, []).append((name, len(steps), bad))
|
|
|
|
for key in sorted(groups):
|
|
runs = groups[key]
|
|
total = len(runs)
|
|
clean = sum(1 for _, n, bad in runs if n > 0 and not bad)
|
|
frozen = [name for name, n, bad in runs if n > 0 and bad]
|
|
nostep = [name for name, n, bad in runs if n == 0]
|
|
print("%-16s runs=%2d script-OK=%2d script-FAIL=%2d %s" %
|
|
(key, total, clean, len(frozen), ("FAIL:" + ",".join(x.replace(key + "_", "") for x in frozen)) if frozen else ""))
|
|
if nostep:
|
|
print(" (no STEP lines: %s)" % ",".join(nostep))
|