diff --git a/SplitBit Test Manual.md b/SplitBit Test Manual.md index 92c7ebb..ab56170 100644 --- a/SplitBit Test Manual.md +++ b/SplitBit Test Manual.md @@ -70,6 +70,28 @@ Individual scripts can be run on their own, from anywhere: `Tests/makedisks.sh` is not in that list because it checks nothing. It builds the disk images the other scripts read, and `run.sh` calls it. +`Tests/break.sh` is not in it either, for the reason turned round: it checks that a check +works. Run it by hand when a check is written, not as part of `make test`. + +``` +./Tests/break.sh [suite ...] +``` + +It replaces the anchor - which must appear exactly once - rebuilds, runs the suites, restores +the file, and reports whether anything failed. **A suite failing is the good outcome.** + +A check that passes proves nothing until it has been seen to fail, and doing that by hand +went wrong twice in two days. Both times it looked like a result: the suite ran, went green, +and read exactly like "this check does not catch that". Once the edit did not compile, `make` +failed, the exit status was not looked at, and the previous binary ran the suite. Once the +anchor was right and the filename was wrong, so nothing was edited at all. + +Neither had anything to do with the build system; headers have always rebuilt what depends on +them. What was missing was a harness that refuses to report a result it did not earn, so +every step is checked and anything unexpected is a hard error rather than a green run. **Not +finding the break is the answer this tool exists to give, and it is worthless if it can also +be the answer when the break never happened.** + Everything is built into `Tests/build`, which is removed and remade at the start of every run. **The suite never writes into `Programs/`.** A binary sitting next to its source came from `make`, not from here. diff --git a/Tests/break.sh b/Tests/break.sh new file mode 100755 index 0000000..fbc6f1a --- /dev/null +++ b/Tests/break.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Breaks something on purpose and checks that a suite notices. +# +# ---- Why this is a script and not a thing to type each time ---- +# +# A check that passes proves nothing until it has been seen to fail. Typing that by hand went +# wrong twice in two days, and BOTH TIMES IT LOOKED LIKE A RESULT: the suite ran, went green, +# and read exactly like "this check does not catch that". +# +# Once the edit produced code that would not compile. make failed, the exit status was not +# looked at, and the PREVIOUS binary ran the suite. +# Once the anchor was right but the filename was wrong, so nothing was edited at all. +# +# Neither had anything to do with the build system - headers have always rebuilt what depends +# on them. What was missing was a harness that refuses to report a result it did not earn. So +# every step below is checked, and anything unexpected is a hard error rather than a green +# run: NOT FINDING THE BREAK IS THE ANSWER THIS TOOL EXISTS TO GIVE, and it is worthless if it +# can also be the answer when the break never happened. +# +# Usage: +# ./Tests/break.sh [suite ...] +# +# The anchor must appear EXACTLY ONCE in the file. The file is restored afterwards whatever +# happens, including on an interrupt. +# +# Written by Anachronaut + +set -u + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +GREEN=$'\033[32m'; RED=$'\033[31m'; RESET=$'\033[0m' +[ -t 1 ] || { GREEN=""; RED=""; RESET=""; } + +die() { printf '%sbreak.sh: %s%s\n' "$RED" "$1" "$RESET" >&2; exit 2; } + +[ $# -ge 4 ] || die "usage: break.sh [suite ...]" + +FILE="$1"; ANCHOR="$2"; REPLACEMENT="$3"; shift 3 +[ -f "$FILE" ] || die "no such file: $FILE" + +# ---- The file comes back whatever happens ---- +# +# Including a Ctrl-C in the middle of a suite, which is a thing that happens: a half broken +# working tree that looks fine is worse than any failing test. +KEEP="$(mktemp)" +cp "$FILE" "$KEEP" +restore() { cp "$KEEP" "$FILE"; rm -f "$KEEP"; (cd "$ROOT" && make >/dev/null 2>&1); } +trap restore EXIT INT TERM + +# ---- The edit, and proof it happened ---- +python3 - "$FILE" "$ANCHOR" "$REPLACEMENT" <<'PY' || die "the edit did not apply" +import sys +path, anchor, replacement = sys.argv[1], sys.argv[2], sys.argv[3] +text = open(path).read() +seen = text.count(anchor) +if seen == 0: + sys.stderr.write("anchor not found in %s\n" % path) + raise SystemExit(1) +if seen > 1: + sys.stderr.write("anchor appears %d times in %s; it must be unique\n" % (seen, path)) + raise SystemExit(1) +open(path, "w").write(text.replace(anchor, replacement)) +PY + +# ---- The build, and proof of that too ---- +if ! (cd "$ROOT" && make) > "$KEEP.build" 2>&1; then + printf '%sbreak.sh: the broken version does not build, so nothing was tested%s\n' \ + "$RED" "$RESET" >&2 + tail -5 "$KEEP.build" >&2 + rm -f "$KEEP.build" + exit 2 +fi +rm -f "$KEEP.build" + +# ---- And then the suites ---- +# +# A suite that fails is the GOOD outcome here, so the exit status is inverted: this reports +# success when the break was noticed. +NOTICED=0 +for suite in "$@"; do + name="$(basename "$suite" .sh)" + if [ -x "$ROOT/Tests/$name.sh" ]; then + run="$ROOT/Tests/$name.sh" + elif [ -x "$suite" ]; then + run="$suite" + else + die "no such suite: $suite" + fi + printf -- '---- %s ----\n' "$name" + if "$run" 2>&1 | tee "$KEEP.out" | grep -E '^\s*\[FAIL\]' | sed 's/^ *//'; then + : + fi + if grep -qE '^[0-9]+ passed, [0-9]+ failed' "$KEEP.out"; then + NOTICED=1 + grep -E '^[0-9]+ passed, [0-9]+ failed' "$KEEP.out" | tail -1 + else + printf '%s%s did not notice%s\n' "$RED" "$name" "$RESET" + fi + rm -f "$KEEP.out" +done + +if [ "$NOTICED" = "1" ]; then + printf '%sThe break was caught.%s\n' "$GREEN" "$RESET" + exit 0 +fi +printf '%sNOTHING CAUGHT THE BREAK.%s\n' "$RED" "$RESET" +exit 1 diff --git a/Tests/docs.sh b/Tests/docs.sh index d968493..364aea6 100755 --- a/Tests/docs.sh +++ b/Tests/docs.sh @@ -643,7 +643,11 @@ else: # file. # # The bullets now live in the Test Manual rather than the README, so that is what is read. -# makedisks.sh is not counted, because it builds the images rather than checking anything; +# makedisks.sh is not counted, because it builds the images rather than checking anything, +# and break.sh is not counted for the same reason turned round: it checks that a check works, +# is run by hand at the moment a check is written, and is not part of what "make test" means. +# Both are still described in the manual - what they are excluded from is the COUNT of the +# suite, not from being documented, and the check below enforces that. # run.sh is counted, because the manual describes it alongside the rest. rootReadme = open("README.md").read() manual = open("SplitBit Test Manual.md").read() @@ -651,8 +655,9 @@ manual = open("SplitBit Test Manual.md").read() # lines. Every pattern below runs against a copy with its whitespace flattened. flat = re.sub(r"\s+", " ", manual) +notSuite = ("makedisks.sh", "break.sh") scripts = sorted(os.path.basename(p) for p in glob.glob("Tests/*.sh") - if os.path.basename(p) != "makedisks.sh") + if os.path.basename(p) not in notSuite) # Spelled out, because that is how the documents say them. Kept a few ahead of the count so # that adding a script fails on the number being wrong rather than on the word being unknown, # which is a much less helpful thing to be told. @@ -669,6 +674,17 @@ for name in scripts: problems.append("Tests/%s runs in the suite and the Test Manual does not say what" " it is for" % name) +# ---- And the two that are not in the suite are still described ---- +# +# Being left out of the COUNT is not the same as being left out of the manual, and the gap +# between those two is exactly where a script goes undocumented for months. A tool nobody has +# written down is a tool nobody uses, which for break.sh would be a particular waste: it +# exists because the technique it automates was got wrong by hand twice. +for name in notSuite: + if ("`Tests/%s`" % name) not in manual: + problems.append("Tests/%s is a tool the suite does not count, and the Test Manual" + " does not say what it is for" % name) + # ---- The shape of the manifest, which the manual states outright ---- # # Five numbers in one sentence, all of them countable from the file they describe. This is diff --git a/Tests/video.sh b/Tests/video.sh index 23d604a..38a78d2 100755 --- a/Tests/video.sh +++ b/Tests/video.sh @@ -23,6 +23,21 @@ for tool in "$ASM" "$EMU"; do [ -x "$tool" ] || { echo "$(basename "$tool") is not built."; exit 1; } done +# ---- The fixture disks, which this suite reads and does not build ---- +# +# Everything that boots CosmOS below runs off Tests/build/disks/cosmos.img, and that is made +# by run.sh rather than here. When it is absent - after make sanitize, which clears the build +# directory - the emulator has no disk, seven checks find no picture, and the run reports +# SEVEN PRODUCT FAILURES for a missing fixture. That is the worst kind of red: it looks +# exactly like something broke. +# +# So the disks are built if they are not there, and this says so rather than limping on. +if [ ! -f "$ROOT/Tests/build/disks/cosmos.img" ]; then + echo "The fixture disks are not built; building them." + "$ROOT/Tests/makedisks.sh" "$ROOT/Tests/build" > /dev/null \ + || { echo "Couldn't build the test disks."; exit 1; } +fi + rm -rf "$BUILD"; mkdir -p "$BUILD" PASS=0