From cdee9acfae0bbb674c6ce811ed79bfb059032a85 Mon Sep 17 00:00:00 2001 From: Anachronaut Date: Sat, 5 Sep 2026 09:40:10 -0400 Subject: [PATCH] break.sh believes a suite's exit status, not the shape of its output It decided whether a suite noticed by grepping its output for a "N passed, M failed" summary line. That is right for the suites that print one, since they print it only when something failed - and impossible for the three that never print one at all. docs, terminal and voyager report in their own words, so a break any of them caught loudly was reported as "NOTHING CAUGHT THE BREAK". That is the one wrong answer the tool exists never to give, and it was turning up in a third place: the header already tells the story of the first two. It made the whole docs suite unverifiable by the harness the project uses to decide whether a check is worth having. Every suite already exits nonzero when it fails, so that is the signal now. The summary line is still printed where a suite keeps a count, and the suite's own last line stands in where it does not. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01E2JrLzFvuFX9fgi1LDRjrW --- Tests/break.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Tests/break.sh b/Tests/break.sh index 8a4e58c..fd7ec1f 100755 --- a/Tests/break.sh +++ b/Tests/break.sh @@ -104,12 +104,24 @@ for suite in "$@"; do die "no such suite: $suite" fi printf -- '---- %s ----\n' "$name" - if "$run" 2>&1 | tee "$KEEP.out" | grep -E '^\s*\[FAIL\]' | sed 's/^ *//'; then - : - fi - if grep -qE '^[0-9]+ passed, [0-9]+ failed' "$KEEP.out"; then + "$run" > "$KEEP.out" 2>&1 + status=$? + grep -E '^\s*\[FAIL\]' "$KEEP.out" | sed 's/^ *//' + + # THE SUITE'S EXIT STATUS IS THE SIGNAL, not the shape of its last line. This used to + # look for a "N passed, M failed" summary, which is right for the suites that print one + # - they print it only when something failed - and impossible for the three that never + # do. docs, terminal and voyager report in their own words and say so with their exit + # status, so a break they caught loudly was answered with "NOTHING CAUGHT THE BREAK": + # the one wrong answer this tool exists never to give, turning up in a third place. + if [ "$status" -ne 0 ]; then NOTICED=1 - grep -E '^[0-9]+ passed, [0-9]+ failed' "$KEEP.out" | tail -1 + # The count where a suite keeps one, and its own last word where it does not. + if grep -qE '^[0-9]+ passed, [0-9]+ failed' "$KEEP.out"; then + grep -E '^[0-9]+ passed, [0-9]+ failed' "$KEEP.out" | tail -1 + else + tail -1 "$KEEP.out" + fi else printf '%s%s did not notice%s\n' "$RED" "$name" "$RESET" fi