91 lines
4.0 KiB
Python
91 lines
4.0 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Authoritative bench verdict (2026-09-23), replacing probe_classify mode A/C
|
|
# verdicts which were proven WRONG (they reported STALL on runs whose script-level
|
|
# STEP lines show all bytes received, and ignored in-firmware readers that consume
|
|
# peer data without echoing it).
|
|
#
|
|
# Ground truth = the run log's own STEP lines printed by probe_bench_lib:
|
|
# STEP <label> rx=<got>/<want> OK|QUIET
|
|
# plus the firmware's eaten-byte log on the serial line:
|
|
# probe: recv64 n=<N> (test probe)
|
|
# NOTE: Phase 9/11 style in-firmware readers consume peer bytes WITHOUT logging.
|
|
# For such builds a shortfall is EXPECTED, so this tool reports the shortfall
|
|
# and whether it can be explained by logged eaten bytes:
|
|
# PASS - every step got its full byte count
|
|
# EXPECTED-SHORT - shortfall <= logged eaten bytes (test-reader artifact)
|
|
# SUSPECT-FREEZE - shortfall > logged eaten bytes, or a step got 0 bytes
|
|
# while no logged reader could explain it
|
|
# Only a build with NO non-echoing reader (e.g. Phase 6 only) can yield a
|
|
# meaningful freeze verdict.
|
|
# ASCII output only.
|
|
import sys, re
|
|
|
|
def clean(s):
|
|
return re.sub(r"\x1b\[[0-9;]*m", "", s)
|
|
|
|
def parse_run(path):
|
|
steps = []
|
|
ts = None
|
|
for line in open(path, encoding="utf-8-sig", errors="replace"):
|
|
c = clean(line)
|
|
m = re.match(r"TS (\d+)", c.strip())
|
|
if m and ts is None:
|
|
ts = int(m.group(1))
|
|
m = re.match(r"STEP (.+?) rx=(\d+)/(\d+) (OK|QUIET)", c.strip())
|
|
if m:
|
|
steps.append((m.group(1), int(m.group(2)), int(m.group(3)), m.group(4)))
|
|
return ts, steps
|
|
|
|
def parse_eaten(serial_path, ts, window_s=20):
|
|
"""sum of probe eaten bytes inside the run window [ts, ts+window)"""
|
|
eaten = 0
|
|
n = 0
|
|
for line in open(serial_path, encoding="utf-8-sig", errors="replace"):
|
|
m = re.search(r"\[(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\].*probe: recv64 n=(-?\d+)", line)
|
|
if not m:
|
|
continue
|
|
# device UTC stamp; run TS is UTC epoch -> convert both to epoch
|
|
import calendar
|
|
ep = calendar.timegm((int(m.group(1)), int(m.group(2)), int(m.group(3)),
|
|
int(m.group(4)), int(m.group(5)), int(m.group(6)), 0, 0, 0))
|
|
if ts is not None and ts - 4 <= ep < ts + window_s:
|
|
v = int(m.group(7))
|
|
if v > 0:
|
|
eaten += v
|
|
n += 1
|
|
return eaten, n
|
|
|
|
def main():
|
|
run_log, serial_log = sys.argv[1], sys.argv[2]
|
|
silent_reader = (len(sys.argv) > 3 and sys.argv[3] == "1")
|
|
ts, steps = parse_run(run_log)
|
|
if not steps:
|
|
print("VERDICT UNKNOWN: no STEP lines in run log (not a probe-script run)")
|
|
return 2
|
|
eaten, nev = parse_eaten(serial_log, ts)
|
|
total_got = sum(s[1] for s in steps)
|
|
total_want = sum(s[2] for s in steps)
|
|
shortfall = total_want - total_got
|
|
zero_steps = [s[0] for s in steps if s[1] == 0]
|
|
print("run TS=%s steps=%d got=%d want=%d shortfall=%d logged_eaten=%d(%d reads)" %
|
|
(ts, len(steps), total_got, total_want, shortfall, eaten, nev))
|
|
for lab, got, want, st in steps:
|
|
if got != want:
|
|
print(" short: %-14s %d/%d %s" % (lab, got, want, st))
|
|
if shortfall == 0:
|
|
print("VERDICT PASS")
|
|
elif shortfall <= eaten:
|
|
print("VERDICT EXPECTED-SHORT (test probe ate %d bytes; not a freeze)" % eaten)
|
|
elif silent_reader:
|
|
print("VERDICT INCONCLUSIVE (shortfall %d; build contains a non-echoing in-firmware "
|
|
"reader (Phase 9/11) which eats peer bytes silently - need device-side evidence)" % shortfall)
|
|
elif zero_steps:
|
|
print("VERDICT SUSPECT-FREEZE (zero-byte step: %s; unexplained shortfall %d)" %
|
|
",".join(zero_steps), shortfall - eaten)
|
|
else:
|
|
print("VERDICT SUSPECT-FREEZE (shortfall %d exceeds logged eaten %d; note Phase 9/11 "
|
|
"readers eat bytes silently)" % (shortfall, eaten))
|
|
return 0
|
|
|
|
sys.exit(main())
|