SplitBit-Emulator/Tests/break.sh

#!/usr/bin/env bash
# Breaks something on purpose and checks that a suite notices.
#
# ---- Why this is a script and not a thing to type each time ----
#
# A check that passes proves nothing until it has been seen to fail. Typing that by hand went
# wrong twice in two days, and BOTH TIMES IT LOOKED LIKE A RESULT: the suite ran, went green,
# and read exactly like "this check does not catch that".
#
#   Once the edit produced code that would not compile. make failed, the exit status was not
#   looked at, and the PREVIOUS binary ran the suite.
#   Once the anchor was right but the filename was wrong, so nothing was edited at all.
#
# Neither had anything to do with the build system - headers have always rebuilt what depends
# on them. What was missing was a harness that refuses to report a result it did not earn. So
# every step below is checked, and anything unexpected is a hard error rather than a green
# run: NOT FINDING THE BREAK IS THE ANSWER THIS TOOL EXISTS TO GIVE, and it is worthless if it
# can also be the answer when the break never happened.
#
# Usage:
#   ./Tests/break.sh <file> <anchor> <replacement> <suite> [suite ...]
#
# The anchor must appear EXACTLY ONCE in the file. The file is restored afterwards whatever
# happens, including on an interrupt.
#
# Written by Anachronaut

set -u

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
GREEN=$'\033[32m'; RED=$'\033[31m'; RESET=$'\033[0m'
[ -t 1 ] || { GREEN=""; RED=""; RESET=""; }

die() { printf '%sbreak.sh: %s%s\n' "$RED" "$1" "$RESET" >&2; exit 2; }

[ $# -ge 4 ] || die "usage: break.sh <file> <anchor> <replacement> <suite> [suite ...]"

FILE="$1"; ANCHOR="$2"; REPLACEMENT="$3"; shift 3
[ -f "$FILE" ] || die "no such file: $FILE"

# ---- The file comes back whatever happens ----
#
# Including a Ctrl-C in the middle of a suite, which is a thing that happens: a half broken
# working tree that looks fine is worse than any failing test.
KEEP="$(mktemp)"
cp "$FILE" "$KEEP"
restore() {
    cp "$KEEP" "$FILE"; rm -f "$KEEP"
    (cd "$ROOT" && make >/dev/null 2>&1 && ./Tests/makedisks.sh Tests/build >/dev/null 2>&1)
}
trap restore EXIT INT TERM

# ---- The edit, and proof it happened ----
python3 - "$FILE" "$ANCHOR" "$REPLACEMENT" <<'PY' || die "the edit did not apply"
import sys
path, anchor, replacement = sys.argv[1], sys.argv[2], sys.argv[3]
text = open(path).read()
seen = text.count(anchor)
if seen == 0:
    sys.stderr.write("anchor not found in %s\n" % path)
    raise SystemExit(1)
if seen > 1:
    sys.stderr.write("anchor appears %d times in %s; it must be unique\n" % (seen, path))
    raise SystemExit(1)
open(path, "w").write(text.replace(anchor, replacement))
PY

# ---- The build, and proof of that too ----
#
# And the disk images after it, because half the things worth breaking are SplitBit assembly
# rather than C, and those live on the fixture disks rather than in the binaries. Rebuilding
# only the emulator meant an edit to a .asm file changed nothing the suite could see and the
# tool reported that nothing caught the break - which is the exact lie it exists to prevent,
# turning up in a new place.
if ! (cd "$ROOT" && make) > "$KEEP.build" 2>&1; then
    printf '%sbreak.sh: the broken version does not build, so nothing was tested%s\n' \
        "$RED" "$RESET" >&2
    tail -5 "$KEEP.build" >&2
    rm -f "$KEEP.build"
    exit 2
fi
rm -f "$KEEP.build"

if ! (cd "$ROOT" && ./Tests/makedisks.sh Tests/build) > "$KEEP.disks" 2>&1; then
    printf '%sbreak.sh: the disks did not build, so nothing was tested%s\n' "$RED" "$RESET" >&2
    tail -5 "$KEEP.disks" >&2
    rm -f "$KEEP.disks"
    exit 2
fi
rm -f "$KEEP.disks"

# ---- And then the suites ----
#
# A suite that fails is the GOOD outcome here, so the exit status is inverted: this reports
# success when the break was noticed.
NOTICED=0
for suite in "$@"; do
    name="$(basename "$suite" .sh)"
    if [ -x "$ROOT/Tests/$name.sh" ]; then
        run="$ROOT/Tests/$name.sh"
    elif [ -x "$suite" ]; then
        run="$suite"
    else
        die "no such suite: $suite"
    fi
    printf -- '---- %s ----\n' "$name"
    "$run" > "$KEEP.out" 2>&1
    status=$?
    grep -E '^\s*\[FAIL\]' "$KEEP.out" | sed 's/^ *//'

    # THE SUITE'S EXIT STATUS IS THE SIGNAL, not the shape of its last line. This used to
    # look for a "N passed, M failed" summary, which is right for the suites that print one
    # - they print it only when something failed - and impossible for the three that never
    # do. docs, terminal and voyager report in their own words and say so with their exit
    # status, so a break they caught loudly was answered with "NOTHING CAUGHT THE BREAK":
    # the one wrong answer this tool exists never to give, turning up in a third place.
    if [ "$status" -ne 0 ]; then
        NOTICED=1
        # The count where a suite keeps one, and its own last word where it does not.
        if grep -qE '^[0-9]+ passed, [0-9]+ failed' "$KEEP.out"; then
            grep -E '^[0-9]+ passed, [0-9]+ failed' "$KEEP.out" | tail -1
        else
            tail -1 "$KEEP.out"
        fi
    else
        printf '%s%s did not notice%s\n' "$RED" "$name" "$RESET"
    fi
    rm -f "$KEEP.out"
done

if [ "$NOTICED" = "1" ]; then
    printf '%sThe break was caught.%s\n' "$GREEN" "$RESET"
    exit 0
fi
printf '%sNOTHING CAUGHT THE BREAK.%s\n' "$RED" "$RESET"
exit 1