#!/usr/bin/env bash # Checks the manuals against the code, and the repository against its own rules. # # Documentation goes stale quietly. An instruction added without a table row, or a count # in a heading that nobody updated, is wrong in a way nothing notices until somebody # trusts it. Everything here is a claim the manuals make that can be settled by looking # at the source, so it is settled every time the tests run. # # Written by Anachronaut set -u ROOT="$(cd "$(dirname "$0")/.." && pwd)" cd "$ROOT" || exit 1 python3 - <<'PY' import glob import re import sys problems = [] def read(path): return open(path).read() pm = read("SplitBit Programming Manual.md") am = read("SplitBit Assembler Manual.md") # The third manual. CosmOS is a system that runs ON SplitBit rather than part of it, so # what it offers a program is documented with it and checked here alongside the other two. cr = read("Programs/CosmOS/README.md") asmc = read("Source/Assembler/assembly.c") util = read("Source/Assembler/Assm-util.c") # ---- Every tracked file is plain ASCII ---- # # A standing rule of this repository, and nothing enforced it, so it drifted: 39 em dashes # and an ellipsis had collected in the two manuals, all of them typed by something that # helpfully substituted a nicer character. # # GIT LS-FILES IS READ NUL SEPARATED, and that is not fussiness. The obvious shell version # of this check - looping over $(git ls-files) - splits on whitespace, so it looked for a # file called "SplitBit" and reported the repository clean while both manuals had drifted. # A check that cannot see the files with spaces in their names is worse than no check. import subprocess tracked = subprocess.run(["git", "ls-files", "-z"], capture_output=True).stdout for name in tracked.split(b"\0"): if not name: continue path = name.decode() try: text = open(path, encoding="utf-8").read() except (UnicodeDecodeError, OSError): continue for number, line in enumerate(text.split("\n"), 1): odd = sorted({c for c in line if ord(c) > 127}) if odd: problems.append("%s line %d is not plain ASCII: %s" % (path, number, ", ".join("%r (U+%04X)" % (c, ord(c)) for c in odd))) break # ---- Every link in the documents goes somewhere ---- # # A link that does not resolve is the same kind of wrong as a stale claim: it looks like # information and is not, and nobody notices until a stranger clicks it. The manuals have # SPACES IN THEIR NAMES, so a link to one carries %20 and has to be unquoted before it can # be looked for - which is the sort of thing that would otherwise be got wrong once and # then reported as fine. import os import urllib.parse for name in tracked.split(b"\0"): if not name or not name.endswith(b".md"): continue path = name.decode() here = os.path.dirname(path) for match in re.finditer(r"\[[^\]]*\]\(([^)]+)\)", read(path)): target = match.group(1) if target.startswith(("http://", "https://", "#", "mailto:")): continue # A raw space ends the link early in most renderers, so the file existing is not # enough - the manuals have spaces in their names and must carry %20. if " " in target: problems.append("%s links to \"%s\", which has a space in it: most renderers" " stop at the space. Write it as %%20." % (path, target)) continue wanted = urllib.parse.unquote(target.split("#")[0]) if not os.path.exists(os.path.normpath(os.path.join(here, wanted))): problems.append("%s links to %s, and there is nothing there" % (path, target)) # ---- Every instruction has a row, and every row is an instruction ---- # # A mnemonic begins with a letter, which is what keeps the offset and size columns of the # other tables in these manuals out of it. documented = {(int(m.group(1), 16), m.group(2)) for m in re.finditer(r'^\|\s*([0-9A-F]{2})\s*\|\s*([A-Z][A-Z0-9]*)\s*\|', pm, re.M)} implemented = {(int(m.group(1), 16), m.group(2)) for m in re.finditer(r'\{0x([0-9A-Fa-f]{2}),\s*"([A-Z0-9]+)"\}', asmc)} for opcode, name in sorted(implemented - documented): problems.append("%s (0x%02X) is implemented and not in the manual" % (name, opcode)) for opcode, name in sorted(documented - implemented): problems.append("%s (0x%02X) is in the manual and not implemented" % (name, opcode)) # ---- The counts in the group headings ---- body = asmc[asmc.index("Instruction instruction_set[]"):asmc.index("int num_instructions")] actual = {} group = None for line in body.split("\n"): heading = re.match(r'\s*// (.+?) Operations:', line) if heading: group = heading.group(1) actual.setdefault(group, 0) if re.search(r'\{0x[0-9A-Fa-f]{2},', line) and group: actual[group] += 1 for m in re.finditer(r'^### (.+?) Operations: (\d+) Instructions?$', pm, re.M): name, claimed = m.group(1), int(m.group(2)) # The manual's headings are wordier than the source's comments, so match on the start. match = [v for k, v in actual.items() if name.startswith(k)] if not match: problems.append("the manual has a group called \"%s\" that the source does not" % name) elif match[0] != claimed: problems.append("the manual says %s has %d instructions, and it has %d" % (name, claimed, match[0])) # ---- How many instructions carry a Data Pointer selector ---- # # The manual says this as a word rather than a figure, and it is the sort of number that # goes stale quietly: adding an instruction that takes a selector leaves the sentence # looking perfectly reasonable and wrong. dataPointerOperands is the list, so it is the # one to believe. # Past twenty the number is two words, the way this manual writes every other one, so the # pattern has to allow a second - and the count going past twenty is exactly the sort of # thing that would otherwise turn "the manual is wrong" into "the manual has stopped # saying it", which reads as a different kind of problem. words = {12: "Twelve", 13: "Thirteen", 14: "Fourteen", 15: "Fifteen", 16: "Sixteen", 17: "Seventeen", 18: "Eighteen", 19: "Nineteen", 20: "Twenty", 21: "Twenty one", 22: "Twenty two", 23: "Twenty three", 24: "Twenty four", 25: "Twenty five", 26: "Twenty six"} selectors = asmc[asmc.index("int dataPointerOperands"):asmc.index("int getOpcode")] taking = len(re.findall(r'^\s*case 0x[0-9A-Fa-f]{2}:', selectors, re.M)) said = re.search(r'^([A-Z][a-z]+(?: [a-z]+)?) instructions work through a Data Pointer\.', pm, re.M) if not said: problems.append("the manual no longer says how many instructions take a Data Pointer") elif said.group(1) != words.get(taking): problems.append("the manual says %s instructions work through a Data Pointer, and %d do" % (said.group(1).lower(), taking)) # ---- Every device class in the header has a row in the Devices table ---- # # The table says which ports a device answers on and what class it reports. Adding a # device, or widening one from a single port to a block, leaves the table looking perfectly # reasonable and describing a machine that no longer exists. The classes are the part that # can be checked against the source without teaching this script how ports are laid out: # every class the header defines except DEVICE_NONE is something a program can find on the # bus, so every one of them has to be findable in the manual too. ioh = read("Source/Emulator/io.h") classes = {name: int(value, 16) for name, value in re.findall(r'^#define (DEVICE_[A-Z_]+)\s+(0x[0-9A-Fa-f]{2})$', ioh, re.M) if name not in ("DEVICE_NONE",)} if "## Devices:" not in pm: problems.append("the Programming Manual has lost its Devices table") else: table = pm.split("## Devices:")[1].split("\n## ")[0] listed = {int(m, 16) for m in re.findall(r'\|\s*(0x[0-9A-Fa-f]{2})\s*\|\s*$', table, re.M)} for name, value in sorted(classes.items(), key=lambda pair: pair[1]): if value not in listed: problems.append("%s (0x%02X) is a device class and has no row in the Devices" " table" % (name, value)) # ---- Every option the assembler takes is written down in both places ---- # # An option added to getopt is an option nobody knows about until it is said twice: in the # manual's table, and in the usage the assembler prints when it is asked for help or refuses # a command line. -S arrived with neither, and the damage was worse than an undocumented # switch. The usage listed a bare "-S " with no long name and no statement that the # file is one it WRITES, so it read as "dump the symbols of " - which hands the source # to -S, leaves nothing positional behind it, and gets answered with "No source file # specified" on a command line that plainly names one. # # The option table is the source of truth because it is what getopt_long is actually given. assembler = read("Source/Assembler/Assembler.c") options = re.findall(r'^\s*\{"([a-z]+)",\s*\w+,\s*0,\s*\'(\w)\'\s*\},', assembler, re.M) if not options: problems.append("could not find the assembler's option table") elif "void printUsage" not in assembler: problems.append("the assembler has lost its printUsage") elif "## Running the Assembler:" not in am: problems.append("the Assembler Manual has lost its options table") else: usage = assembler.split("void printUsage")[1].split("\n}")[0] section = am.split("## Running the Assembler:")[1].split("\n## ")[0] # Only the table rows count, so a switch merely mentioned in the prose below it does not # pass for a documented one. rows = re.findall(r'^\| (-\w, --[a-z]+)', section, re.M) for longName, shortName in options: spelled = "-%s, --%s" % (shortName, longName) if spelled not in rows: problems.append("the assembler takes %s and the Assembler Manual's option table" " has no row for it" % spelled) if spelled not in usage: problems.append("the assembler takes %s and its own usage message does not name" " it" % spelled) # ---- The vector ranges the manuals quote are the ones the assembler uses ---- # # Both manuals print the boundary between numbers a program may pin and numbers the # assembler hands out. Those are two constants in one header, and moving them without # touching the manuals would leave every programmer reading a range that no longer exists # and being refused a number the manual said was theirs. header = read("Source/Assembler/assembly.h") ranges = {name: int(value) for name, value in re.findall(r'^#define (VECTOR_FIRST_[A-Z]+)\s+(\d+)$', header, re.M)} if set(ranges) != {"VECTOR_FIRST_PINNED", "VECTOR_FIRST_AUTO"}: problems.append("the vector range constants are not the two this check knows about: %s" % ", ".join(sorted(ranges)) if ranges else "none found") else: pinnedFrom = ranges["VECTOR_FIRST_PINNED"] autoFrom = ranges["VECTOR_FIRST_AUTO"] said = "%d to %d" % (pinnedFrom, autoFrom - 1) for manual, text in [("Programming Manual", pm), ("Assembler Manual", am)]: if said not in text: problems.append("the %s does not say the pinned vectors are %s" % (manual, said)) if "%d and up" % autoFrom not in pm: problems.append("the Programming Manual does not say the automatic vectors start" " at %d" % autoFrom) if "from vector %d upwards" % autoFrom not in am: problems.append("the Assembler Manual does not say the automatic vectors start" " at %d" % autoFrom) # ---- The loadable header table matches the offsets the assembler writes ---- # # The Programming Manual prints the header field by field, which is the description two # implementations work from. sbex.h is where the offsets actually are, so a field moved # there and not here would leave the manual describing a format nobody writes. sbex = read("Source/Assembler/sbex.h") offsets = {name: int(value) for name, value in re.findall(r'^#define (SBEX_[A-Z_]+_AT)\s+(\d+)$', sbex, re.M)} if "## Loading A Program From A Disk:" not in am: problems.append("the Assembler Manual has lost its loadable program section") else: loading = am.split("## Loading A Program From A Disk:")[1].split("\n## ")[0] listed = [int(m) for m in re.findall(r'^\| (\d+) \| \d* \|', loading, re.M)] for name, offset in sorted(offsets.items(), key=lambda pair: pair[1]): if offset not in listed: problems.append("%s is at offset %d and the header table has no row for it" % (name, offset)) # Spelled as a word, the way these manuals write small numbers in prose. asWord = {1: "one", 2: "two", 3: "three", 4: "four", 5: "five"} for version in ("SBEX_VERSION", "SBEX_VERSION_VECTORS"): number = re.search(r'^#define %s\s+(\d+)$' % version, sbex, re.M) if not number: problems.append("%s is gone from sbex.h" % version) continue said = asWord.get(int(number.group(1))) if said is None or said not in loading.lower(): problems.append("the loadable program section does not mention version %s (%s)" % (number.group(1), said)) # ---- Every console status bit is described ---- # # The status port is read by writing a mask and testing it, so a program can only use a bit # it has been told the number of. Adding one and forgetting to write it down leaves a bit # that works and that nobody can discover. The section names them as "bit N", so that is # what is looked for. status = {name: int(value, 16) for name, value in re.findall(r'^#define (CONSOLE_STATUS_[A-Z]+)\s+(0x[0-9A-Fa-f]{2})$', ioh, re.M)} if "## The Console:" not in pm: problems.append("the Programming Manual has lost its \"The Console\" section") else: console = pm.split("## The Console:")[1].split("\n## ")[0] for name, value in sorted(status.items(), key=lambda pair: pair[1]): bit = value.bit_length() - 1 if "bit %d" % bit not in console: problems.append("%s is bit %d of the console status port and The Console does" " not mention it" % (name, bit)) # ---- Every service the system offers has a row ---- # # services.asm is the one place the numbers are written, and both the system and every # program include it. A service added there and not here is one nothing can find out about # except by reading the source of the operating system. # DECLARING A SERVICE AND IMPLEMENTING ONE ARE DIFFERENT THINGS, and the manual should # describe the second. services.asm names them and fixes their numbers, which is what lets a # number be pinned before anything answers to it; cosmos.asm is where a name gets a handler. # A row for a service nothing implements would be describing a call that faults, and a # missing row for one that works is a service nobody can find out about. services = read("Programs/CosmOS/Source/services.asm") named = set(re.findall(r'^\s{2}(os[A-Za-z]+)\s+0d\d+', services, re.M)) system = read("Programs/CosmOS/Source/cosmos.asm") vectors = system.split("#Vectors")[-1] if "#Vectors" in system else "" implemented = {name for name in re.findall(r'^\s{2}(os[A-Za-z]+)\s+[a-zA-Z]', vectors, re.M) if name in named} if not named: problems.append("no services could be found in services.asm") elif "## What A Program May Ask The System For:" not in cr: problems.append("the CosmOS README has lost its services section") else: section = cr.split("## What A Program May Ask The System For:")[1].split("\n## ")[0] documented = set(re.findall(r'^\| (os[A-Za-z]+) \|', section, re.M)) for name in sorted(implemented - documented): problems.append("%s is a service the system implements and has no row in the" " services table" % name) for name in sorted(documented - implemented): problems.append("the services table describes %s, which nothing implements: calling" " it would dispatch through an empty vector and fault" % name) # ---- Every program the manual describes is really there ---- # # The table names what the shell can load. A program renamed or removed leaves a row # describing something nobody can run, which is the same kind of quiet wrongness as a # routine that no longer exists. The other direction is deliberately not checked: the ported # programs are covered in the prose rather than given a row each. import os if "## Included Applications:" not in cr: problems.append("the CosmOS README has lost its list of applications") else: listed = cr.split("## Included Applications:")[1].split("\n### ")[0] # After the separator, so the table's own heading row is not mistaken for a program. listed = listed.split("| --- |")[-1] for name in re.findall(r'^\| ([A-Z][A-Za-z0-9-]*) \|', listed, re.M): if not os.path.exists("Programs/CosmOS/Apps/%s.asm" % name): problems.append("the CosmOS README describes an application called %s, and" " there is no Programs/CosmOS/Apps/%s.asm" % (name, name)) # ---- The monitor's instruction table is the assembler's ---- # # The monitor disassembles, so it needs the same 64 instructions with the same names and the # same lengths. A disassembler that disagreed about a length would not print one line wrong, # it would lose its place and print everything after it wrong, which is the worst way for a # tool like that to fail: confidently. So the table is generated from assembly.c by # Tests/instructiontable.py, and what is in the monitor is checked against it here. import subprocess generated = subprocess.run([sys.executable, "Tests/instructiontable.py"], capture_output=True, text=True) if generated.returncode != 0: problems.append("the instruction table generator would not run") else: wanted = [line.rstrip() for line in generated.stdout.splitlines() if line.strip()] # TWO copies now, and both are checked. The monitor has one and the assembler that # runs on the machine has another, because they are separate programs and there is no # linker to let them share: the monitor's lives in the system's data at an address # that moves every rebuild. Duplication is the cost of having no libraries, and a # check on every copy is what keeps the cost to bytes rather than to correctness. copies = [("the system", "Programs/CosmOS/Source/cosmos.asm", "\nInstructions:\n"), ("the native assembler", "Programs/CosmOS/Assembler/table.asm", "\nAsmInstructions:\n")] for who, path, marker in copies: text = read(path) if marker not in text: problems.append("%s has lost its instruction table" % who) continue block = text.split(marker)[1] have = [] for line in block.splitlines(): if not line.strip() or not line.startswith(" 0x"): break have.append(line.rstrip()) if have != wanted: problems.append("%s's instruction table is not what the assembler's" " instruction set generates: %d entries against %d, first" " difference at %s" % (who, len(have), len(wanted), next((a or b for a, b in zip(have + [None] * len(wanted), wanted + [None] * len(have)) if a != b), "the end"))) # ---- And the lengths that table implies are the ones the manual prints ---- # # The generator works out how long each instruction is from rules written in it; the manual # says so in a column somebody typed. They are independent accounts of the same fact, which # is exactly the pair worth checking against each other. sys.path.insert(0, "Tests") import instructiontable lengthOf = {0: 1, 1: 3, 2: 2, 3: 2, 4: 3, 5: 4, 6: 3} printed = {} for m in re.finditer(r'^\|\s*[0-9A-F]{2}\s*\|\s*([A-Z][A-Z0-9]*)\s*\|\s*(\d+)\s*\|', pm, re.M): printed[m.group(1)] = int(m.group(2)) for opcode, name in instructiontable.table(): implied = lengthOf[instructiontable.shapeOf(opcode)] if name in printed and printed[name] != implied: problems.append("the manual says %s is %d bytes and the disassembler will read it" " as %d" % (name, printed[name], implied)) # ---- Every directive the assembler knows is written down ---- for directive in sorted(set(re.findall(r'"(#[A-Za-z]+)"', util))): if directive not in am: problems.append("%s is a directive and is not in the Assembler Manual" % directive) # ---- Every routine the manual promises exists ---- # # The first column of the table in each of these sections names something the library has # to define. A routine renamed in the source and not in the manual is caught here, which # is what keeps the tables a description rather than a memory. for heading, library in [("## The Filesystem Library:", "Programs/CosmOS/Source/sbfs.asm"), ("## The Console Library:", "Programs/CosmOS/Source/console.asm")]: if heading not in cr: problems.append("the CosmOS README has lost its \"%s\" section" % heading.strip("# :")) continue section = cr.split(heading)[1].split("\n## ")[0] defined = set(re.findall(r'^([a-zA-Z][A-Za-z0-9]*):', read(library), re.M)) for name in re.findall(r'^\| ([a-z][A-Za-z0-9]*) \|', section, re.M): if name not in defined: problems.append("the manual lists %s, which %s does not define" % (name, library)) # ---- The worked example still assembles to the bytes the manual prints ---- # # The hello world program and the hex dump beside it are two claims about the same thing, # and nothing but this keeps them agreeing. import os import subprocess import tempfile # BOTH ANCHORS ARE CHECKED BEFORE THEY ARE USED. Every other heading this script splits on # says what it could not find; these two were the exception, and a rename here produced an # IndexError and a traceback instead of a sentence. That is a worse answer than a stale # manual, because whoever reads it learns nothing about which heading moved. # THE TWO HALVES ARE IN DIFFERENT MANUALS NOW, and that makes this a better check than it # was. The program belongs with the machine, where it arrives just after the instruction # list; the hex dump belongs with the boot image format it is an example of, which is the # assembler's business. So this settles three things against each other at once: what the # Programming Manual prints, what the Assembler Manual prints, and what the assembler does. exampleAnchor = "### Example Program: Hello World" dumpAnchor = "assembled and dumped as hex:" if exampleAnchor not in pm: problems.append("the Programming Manual has lost its \"Example Program: Hello World\"" " heading, so the worked example cannot be found") elif dumpAnchor not in am: problems.append("the Assembler Manual no longer says \"%s\" before the hex dump, so" " there is nothing to compare the worked example against" % dumpAnchor) else: source = pm.split(exampleAnchor)[1].split("```")[1] claimed = am.split(dumpAnchor)[1].split("```")[1].split() with tempfile.TemporaryDirectory() as work: asm = os.path.join(work, "hello.asm") binary = os.path.join(work, "hello.bin") open(asm, "w").write(source) built = subprocess.run(["./Assembler", asm, "-o", binary], capture_output=True) if built.returncode != 0: problems.append("the hello world program in the manual no longer assembles") else: actual = ["%02x" % b for b in open(binary, "rb").read()] if [c.lower() for c in claimed] != actual: problems.append("the hex dump in the manual is not what that program" " assembles to now: it prints %d bytes and the assembler" " makes %d" % (len(claimed), len(actual))) # ---- The Assembler Manual's worked programs still assemble ---- for heading in ["## An Example SplitBit Assembly Program:", "## An Example Using More Than One Data Pointer:", "## An Example Using Interrupts:"]: if heading not in am: problems.append("the Assembler Manual has lost its \"%s\" section" % heading.strip("# :")) continue example = am.split(heading)[1].split("```")[1] with tempfile.TemporaryDirectory() as work: asm = os.path.join(work, "example.asm") open(asm, "w").write(example) built = subprocess.run(["./Assembler", "-I", "Programs/Libraries", "-I", "Programs/CosmOS/Source", asm, "-o", os.path.join(work, "example.bin")], capture_output=True) if built.returncode != 0: problems.append("the example under \"%s\" no longer assembles" % heading.strip("# :")) # ---- No manual still says the filesystem is flat ---- # # It was flat, and every manual said so in passing, and those sentences went on being true # for as long as nobody looked. SBFS grew directories over five separate pieces of work and # the last thing left claiming otherwise was one line in the Assembler Manual explaining # why an include had nowhere else to look. # # A phrase rather than a section, because that is the shape this kind of staleness takes: # not a chapter that is wrong, one clause inside a paragraph that is otherwise right. for name, text in (("the Programming Manual", pm), ("the Assembler Manual", am), ("the README", open("README.md").read()), ("the CosmOS README", open("Programs/CosmOS/README.md").read())): for claim in ("SBFS is flat", "the filesystem is flat", "flat filesystem", "SBFS has no directories"): if claim.lower() in text.lower(): problems.append("%s still says \"%s\", and it has not been true since" " directories arrived" % (name, claim)) # ---- The shell's words are a table as well as a chain of comparisons ---- # # The dispatch is a run of "is the line this name" tests, which is fine to execute and # impossible to WALK - so completing a half typed command needs the names as data too, and # they are: fifteen strings packed end to end from ShellNames, each ending in the zero that # says where the next begins. # # THE TWO CAN DISAGREE AND THE WAY THEY DO IS QUIET. A command added to the dispatch and not # to the run simply never completes, which nobody would think to test by hand; something put # BETWEEN the strings ends the walk early and takes the rest of the commands with it. So this # reads both and compares them, and reads the count as well, because a run of strings does # not say where it stops. cosmosSource = open("Programs/CosmOS/Source/cosmos.asm").read() # The dispatch, up to the point where the monitor's single letters begin - those are one # character each and there is nothing to complete about them. dispatchEnd = cosmosSource.find("SETD.0 Mode") dispatched = re.findall(r"SETD\.1 (\w+Name)\b", cosmosSource[:dispatchEnd]) # The run, which ends at the first thing that is not a label and a string. runAt = cosmosSource.find("ShellNames:") packed = [] if runAt < 0: problems.append("cosmos.asm has no ShellNames run for the shell's own words") else: lines = cosmosSource[runAt:].split("\n")[1:] while len(lines) >= 2 and re.fullmatch(r"(\w+Name):", lines[0]) \ and re.fullmatch(r'"[^"]*"', lines[1]): packed.append(lines[0][:-1]) lines = lines[2:] stated = re.search(r"ShellNameCount:\s*\n\s*0d(\d+)", cosmosSource) if not stated: problems.append("cosmos.asm no longer says how many shell names there are") elif int(stated.group(1)) != len(packed): problems.append("cosmos.asm says there are %s shell names and the run holds %d" % (stated.group(1), len(packed))) if runAt >= 0 and sorted(packed) != sorted(dispatched): missing = sorted(set(dispatched) - set(packed)) extra = sorted(set(packed) - set(dispatched)) problems.append("the shell's dispatch and its packed names disagree:%s%s" % ("".join(" %s is dispatched and not in the run;" % n for n in missing), "".join(" %s is in the run and not dispatched;" % n for n in extra))) # ---- CosmOS fits in the half of the machine it says it does ---- # # The memory map in the CosmOS README is a CONVENTION. Nothing in the assembler, the # loader or the machine enforces it: an application says where it goes with #Base, and # CosmOS puts it there. So when CosmOS grew past the address applications are based at, # nothing said so - the next program loaded simply landed on top of the shell's own code, # and what broke was whichever part of the shell that program happened to cover, at # whatever later moment somebody used it. # # That is why this reads the numbers out of the table rather than being told them: the # table is the specification, and a table nothing checks is the thing that goes stale. readme = open("Programs/CosmOS/README.md").read() row = re.compile(r"\|\s*(Program|Data) Memory\s*\|\s*`0x0000` through `0x([0-9A-Fa-f]{4})`" r"\s*\|\s*`0x([0-9A-Fa-f]{4})` and above\s*\|") table = {kind: (int(top, 16) + 1, int(base, 16)) for kind, top, base in row.findall(readme)} limits = {kind: room for kind, (room, base) in table.items()} bases = {kind: base for kind, (room, base) in table.items()} if len(table) != 2: problems.append("the CosmOS README no longer states a memory map this can check") else: built = subprocess.run(["./Assembler", "-I", "Programs/CosmOS/Source", "Programs/CosmOS/Source/cosmos.asm", "-o", os.devnull], capture_output=True, text=True) sizes = dict(re.findall(r"(Program|Data) Segment size: (\d+)", built.stdout)) if len(sizes) != 2: problems.append("could not measure the CosmOS segments") else: for kind in ("Program", "Data"): used, room = int(sizes[kind]), limits[kind] if used > room: problems.append( "CosmOS's %s Segment is %d bytes and the map gives it %d - it now" " reaches into the %d bytes an application is loaded at, and loading" " one will overwrite it" % (kind, used, room, used - room)) # ---- And the table against itself ---- # # The check above reads the CosmOS column and never the one beside it, so it passed a # Data row that gave the system 0x0000-0x3FFF and an application 0x2000 and above - # two halves of one sentence contradicting each other in print. A number checked # against the code and not against the number next to it is still unchecked. for kind in ("Program", "Data"): room, base = table[kind] if base != room: problems.append( "the CosmOS README gives the system %s Memory up to 0x%04X and puts an" " application at 0x%04X - the two columns of that row disagree" % (kind, room - 1, base)) # ---- And the example under it ---- # # The minimal application in the same section is what somebody copies, so it is the # part of the map most worth being right. It went stale across the doubling while the # table above it was corrected. # ---- Found by its section, not by being first ---- # # This took the first asm block in the file, which was the minimal application right up # until somebody documented a program with an assembly example above it - and then this # said the minimal application had no #Base, about a block that was never claiming to be # one. The example lives under System Services; that is what identifies it. services = readme.split("### System Services:", 1) example = (re.search(r"```asm\n(.*?)```", services[1], re.S) if len(services) > 1 else None) if not example: problems.append("the CosmOS README no longer shows a minimal application") else: shown = dict(zip(("Program", "Data"), re.findall(r"#Base 0x([0-9A-Fa-f]{4})", example.group(1)))) for kind in ("Program", "Data"): if kind not in shown: problems.append("the minimal CosmOS application shows no %s #Base" % kind) elif int(shown[kind], 16) != bases[kind]: problems.append( "the minimal CosmOS application is based at 0x%s in %s Memory and the" " map above it says 0x%04X" % (shown[kind].upper(), kind, bases[kind])) # ---- And the copy in the source ---- # # cosmos.asm opens with the same map in its own words, because somebody reading the # system reads that before they read the README. Three copies of one fact, so all # three are compared. header = open("Programs/CosmOS/Source/cosmos.asm").read()[:4096] stated = dict(re.findall( r"(Program|Data) Memory\s+0x0000 - 0x([0-9A-Fa-f]{4})\s+the system", header)) for kind in ("Program", "Data"): if kind not in stated: problems.append("cosmos.asm no longer opens with a %s Memory map" % kind) elif int(stated[kind], 16) + 1 != limits[kind]: problems.append( "cosmos.asm says the system keeps below 0x%s in %s Memory and the CosmOS" " README says 0x%04X" % (stated[kind].upper(), kind, limits[kind] - 1)) # ---- The assembler's scratch map sits above what it says it sits above ---- # # scratch.asm is a MAP rather than a set of declarations: the buffers are not reserved, # they are addresses written in a comment, because reserving them would put 22K of zeroes # in the file and the assembler could not load itself. Nothing enforces a word of it. # # So it carries two claims about the machine around it, and both have gone stale once. It # said the system keeps below 0x1000 for a while after the system's half of Data Memory # was doubled - twelve lines above the paragraph explaining the doubling. And the map # starts at 0x4000 on the grounds that the assembler's own data ends well before there, # which was measured on the day and is not measured again by anything. scratch = open("Programs/CosmOS/Assembler/scratch.asm").read() floor = re.search(r"the system keeps\s*;?\s*below 0x([0-9A-Fa-f]{4})", scratch) if not floor: problems.append("scratch.asm no longer says what the system keeps below") elif "Data" in limits and int(floor.group(1), 16) + 1 != limits["Data"]: problems.append( "the assembler's scratch map says the system keeps below 0x%s and the CosmOS" " README says 0x%04X" % (floor.group(1).upper(), limits["Data"] - 1)) first = re.search(r"^;\s+0x([0-9A-Fa-f]{4})\s+\d+\s+the label index", scratch, re.M) if not first: problems.append("scratch.asm no longer states where its buffers begin") else: built = subprocess.run(["./Assembler", "-I", "Programs/CosmOS/Source", "-I", "Programs/CosmOS/Assembler", "Programs/CosmOS/Assembler/Asm.asm", "-o", os.devnull], capture_output=True, text=True) # A loadable program reports "Data: N bytes at 0xAAAA"; a boot image says it another # way. The assembler is the former, and the address matters as much as the size. said = re.search(r"Data:\s*(\d+) bytes at 0x([0-9A-Fa-f]{4})", built.stdout) if not said: problems.append("could not measure the native assembler's data") else: ends = int(said.group(2), 16) + int(said.group(1)) if ends > int(first.group(1), 16): problems.append( "the native assembler's data reaches 0x%04X and its scratch map begins at" " 0x%s - the buffers are on top of the variables" % (ends - 1, first.group(1).upper())) # ---- The Test Manual against the suite it describes ---- # # The suite documents itself, and a document about the suite goes stale the same way every # other one does. Two claims in the README went stale at once before this check existed and # neither was noticed: it said FIVE more scripts run alongside run.sh when there were six, # and "all three tools" when there were four. Both are the kind of number that is written # once, is true for months, and is then quietly wrong - which is the entire subject of this # file. # # The bullets now live in the Test Manual rather than the README, so that is what is read. # makedisks.sh is not counted, because it builds the images rather than checking anything, # and break.sh is not counted for the same reason turned round: it checks that a check works, # is run by hand at the moment a check is written, and is not part of what "make test" means. # Both are still described in the manual - what they are excluded from is the COUNT of the # suite, not from being documented, and the check below enforces that. # run.sh is counted, because the manual describes it alongside the rest. rootReadme = open("README.md").read() manual = open("SplitBit Test Manual.md").read() # THE MANUAL IS WRAPPED, so a number and the noun it counts are regularly on two different # lines. Every pattern below runs against a copy with its whitespace flattened. flat = re.sub(r"\s+", " ", manual) notSuite = ("makedisks.sh", "break.sh") scripts = sorted(os.path.basename(p) for p in glob.glob("Tests/*.sh") if os.path.basename(p) not in notSuite) # Spelled out, because that is how the documents say them. Kept a few ahead of the count so # that adding a script fails on the number being wrong rather than on the word being unknown, # which is a much less helpful thing to be told. words = {"three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, "eleven": 11, "twelve": 12, "thirteen": 13, "fourteen": 14} said = re.search(r"It is ([a-z]+) scripts making", flat) if not said: problems.append("the Test Manual no longer says how many scripts the suite is") elif words.get(said.group(1)) != len(scripts): problems.append("the Test Manual says the suite is %s scripts, and there are %d: %s" % (said.group(1), len(scripts), ", ".join(scripts))) for name in scripts: if ("`Tests/%s`" % name) not in manual: problems.append("Tests/%s runs in the suite and the Test Manual does not say what" " it is for" % name) # ---- And the two that are not in the suite are still described ---- # # Being left out of the COUNT is not the same as being left out of the manual, and the gap # between those two is exactly where a script goes undocumented for months. A tool nobody has # written down is a tool nobody uses, which for break.sh would be a particular waste: it # exists because the technique it automates was got wrong by hand twice. for name in notSuite: if ("`Tests/%s`" % name) not in manual: problems.append("Tests/%s is a tool the suite does not count, and the Test Manual" " does not say what it is for" % name) # ---- The shape of the manifest, which the manual states outright ---- # # Five numbers in one sentence, all of them countable from the file they describe. This is # the most quotable thing in the manual and the least likely to be recounted by hand. modes = {} total = 0 for line in open("Tests/manifest"): line = line.strip() if not line or line.startswith("#"): continue total += 1 fields = line.split("|") if len(fields) > 2: modes[fields[2].strip()] = modes.get(fields[2].strip(), 0) + 1 said = re.search(r"(\d+) tests, of which (\d+) run, (\d+) only assemble," r" (\d+) are expected to fail to assemble, and (\d+) boot from ROM", flat) if not said: problems.append("the Test Manual no longer states the shape of the manifest") else: for index, (what, count) in enumerate((("tests", total), ("run tests", modes.get("run", 0)), ("assemble-only tests", modes.get("assemble", 0)), ("xfail tests", modes.get("xfail", 0)), ("rom tests", modes.get("rom", 0)))): if int(said.group(index + 1)) != count: problems.append("the Test Manual says there are %s %s, and there are %d" % (said.group(index + 1), what, count)) said = re.search(r"The (\d+) `xfail` tests", flat) if said and int(said.group(1)) != modes.get("xfail", 0): problems.append("the Test Manual says %s xfail tests in one place and %d in another" % (said.group(1), modes.get("xfail", 0))) # ---- And the fixtures and the baseline ---- disks = len(re.findall(r'format "\$DISKS/', open("Tests/makedisks.sh").read())) said = re.search(r"builds (\d+) images with SplitDisk", flat) if not said: problems.append("the Test Manual no longer says how many fixture disks are built") elif int(said.group(1)) != disks: problems.append("the Test Manual says %s fixture disks are built, and makedisks.sh" " builds %d" % (said.group(1), disks)) pairs = sum(1 for line in open("Tests/lint-baseline.txt") if line.strip()) said = re.search(r"(\d+) file-and-rule pairs", flat) if not said: problems.append("the Test Manual no longer says how large the lint baseline is") elif int(said.group(1)) != pairs: problems.append("the Test Manual says the lint baseline holds %s file and rule pairs," " and it holds %d" % (said.group(1), pairs)) # ---- And the tool count is the number of things the makefile builds ---- # # Claimed in both documents, so both are read. makefile = open("makefile").read() # TOOLS rather than the all target, which now depends on whether Raylib is installed. # What "the tools" means should be a fact in one place, not read off a conditional. built = re.search(r"^TOOLS = (.*)$", makefile, re.M) if not built: problems.append("the makefile no longer has a TOOLS list this can count") else: tools = len(built.group(1).split()) for where, text in (("README", rootReadme), ("Test Manual", manual)): for said in re.findall(r"(?:build|rebuild)s? (?:all )?(?:the )?([a-z]+) tools", text, re.I): if words.get(said.lower()) != tools: problems.append("the %s says the %s tools and the makefile builds %d" % (where, said, tools)) # ---- The sizes the CosmOS README quotes for its own programs ---- # # It said Files was 645 bytes in two places and Edit 1,983. They were 665 and 1,996: the # programs kept being improved and the sentences about how small they are did not move. # These are the most quotable numbers in the document and the least likely to be rechecked # by hand, so they are measured. # # A claim is " ... in N bytes" on one line. Only lines that name # an app are looked at, so ordinary prose about bytes is left alone. apps = {} for source in glob.glob("Programs/CosmOS/Apps/*.asm"): apps[os.path.basename(source)[:-4]] = source for line in readme.split("\n"): said = re.search(r"\b(?:in|to) ([\d,]+) bytes", line) if not said: continue named = [name for name in apps if re.search(r"(?:`|\| )%s(?:\.asm)?\b" % re.escape(name), line)] if len(named) != 1: continue built = subprocess.run(["./Assembler", "-I", "Programs/CosmOS/Source", apps[named[0]], "-o", os.devnull], capture_output=True, text=True) real = re.search(r"Total size: (\d+) bytes", built.stdout) if not real: problems.append("could not measure %s, which the CosmOS README quotes a size for" % named[0]) elif int(said.group(1).replace(",", "")) != int(real.group(1)): problems.append("the CosmOS README says %s is %s bytes and it is %s" % (named[0], said.group(1), real.group(1))) # ---- The symbol file has the fields the manual says it has ---- # # The manual describes -S field by field, and a program that reads a symbol file is written # against that description rather than against the assembler. So a field reordered, renamed # or added would leave every such program cutting the wrong column, and the manual would go # on describing a format nobody writes. # # The values are not checked, because an example naming real addresses in real files goes # stale the moment either changes and would turn every edit to a program into an edit to the # manual. The SHAPE is the contract: five fields, tab separated, in the documented order. import subprocess import tempfile if "## Running the Assembler:" not in am: pass # Already reported above. else: section = am.split("## Running the Assembler:")[1].split("\n## ")[0] said = [row.split(" |")[0] for row in re.findall(r'^\| (\w+ \|.*)$', section, re.M) if row.split(" |")[0] in ("Kind", "Number", "Address", "Name", "File", "Line")] if said != ["Kind", "Number", "Address", "Name", "File", "Line"]: problems.append("the Assembler Manual does not describe the symbol file's six" " fields in order; it lists %s" % (said or "none")) else: # Two programs, because no one of them covers everything the format claims. # # Keys is a small LOADABLE program, where the two segments are based apart, and # the smallest thing that produces all four kinds at once: labels in both # memories, vectors it implements, vectors it only declares, and a device. # # cosmos is a BOOT IMAGE, where both segments start at zero, which is the case # the kind field exists for - and it is the one that holds strings with newlines # inside them, which is what the line numbers below go wrong on. # # Both assemble in a few thousandths of a second, so there is nothing to save by # checking only one. for program in ["Programs/CosmOS/Apps/Keys.asm", "Programs/CosmOS/Source/cosmos.asm"]: with tempfile.TemporaryDirectory() as scratch: dump = os.path.join(scratch, "symbols") built = subprocess.run(["./Assembler", "-I", "Programs/Libraries", "-I", "Programs/CosmOS/Source", "-S", dump, "-o", os.path.join(scratch, "out"), program], capture_output=True, text=True) if built.returncode != 0: problems.append("could not assemble %s to check the symbol file" % program) else: kinds = set() for number, line in enumerate(open(dump).read().splitlines(), 1): fields = line.split("\t") if len(fields) != 6: problems.append("line %d of a symbol file has %d fields and the" " Assembler Manual describes six" % (number, len(fields))) break kind, index, address, name, source, where = fields kinds.add(kind) if kind not in ("Program", "Data", "Vector", "Device"): problems.append("a symbol file calls something a %r, and the" " Assembler Manual allows Program, Data, Vector" " and Device" % kind) break if not re.fullmatch(r'[0-9A-F]{4}', address): problems.append("a symbol file gives %r as an address, and the" " Assembler Manual says four hexadecimal digits" % address) break if not name or not source or not where.isdigit(): problems.append("a symbol file line is missing a name, a file or a" " line number: %r" % line) break # A label has no number and says so with a dash; a vector has one, and # the address it sits at has to be the slot that number works out to, # which is the claim the manual makes about both fields at once. if kind in ("Program", "Data"): if index != "-": problems.append("a %s label was given the number %r, and the" " Assembler Manual says a label has none" % (kind, index)) break elif not index.isdigit(): problems.append("a %s row has %r where its number should be" % (kind, index)) break else: base = 0xFC00 if kind == "Vector" else 0xFE00 if int(address, 16) != base + int(index) * 2: problems.append("%s is number %s and the Assembler Manual puts" " that at 0x%04X, but the symbol file says %s" % (name, index, base + int(index) * 2, address)) break else: # Only meaningful if every line was well formed, which is what the else # on the loop says. for wanted in ("Program", "Data", "Vector", "Device"): if wanted not in kinds: problems.append("a program with labels in both memories and a" " Vector Segment produced a symbol file with no" " %s row" % wanted) # ---- And the line really is the line ---- # # "Which line of that file, counting from one" is a claim, and it was # wrong for years without anything noticing: a string literal with a # newline inside it was read as one token and the newline was never # counted, so every line number after one was short by one. cosmos.asm # has thirteen, and its last label was reported thirteen lines early - # which had been sending every error message after that point at # somebody else's code, not only the symbol file. # # A name has to actually appear on the line its row names. That is a # weak test of a right answer and a very strong test of a wrong one: # drift lands on a line that has nothing to do with the name. sources = {} for row in open(dump).read().splitlines(): kind, index, address, name, source, where = row.split("\t") if source not in sources: try: sources[source] = open(source).read().splitlines() except OSError: sources[source] = None lines = sources[source] if lines is None: problems.append("a symbol file names %r, which cannot be read" % source) break at = int(where) if at < 1 or at > len(lines): problems.append("%s is said to be on line %d of %s, which has" " %d lines" % (name, at, source, len(lines))) break if not re.search(r'\b%s\b' % re.escape(name), lines[at - 1]): problems.append("%s is said to be on line %d of %s, and that" " line is %r" % (name, at, source, lines[at - 1].strip())) break # ---- Every directive TuneC takes has a row in the Tune Manual ---- # # The same check the assembler's options get, for the same reason: a directive added to the # compiler and not written down is one nobody can use, and one written down and removed is # worse - it is an instruction that fails. # # The manual also carries the limits, which are #defines and exactly the sort of number that # goes stale silently: raising one leaves a sentence looking perfectly reasonable and wrong. tunec = read("Source/Tune/TuneC.c") tm = read("SplitBit Tune Manual.md") directives = sorted(set(re.findall(r'strcmp\(t->text, "(#\w+)"\)', tunec)) | set(re.findall(r'strcmp\(t->text, "(#Use)"\)', tunec))) if not directives: problems.append("could not find the directives TuneC understands") else: for directive in directives: if not re.search(r'^\| %s[ \\]' % re.escape(directive), tm, re.M): problems.append("TuneC takes %s and the Tune Manual has no row for it" % directive) limits = {name: int(value) for name, value in re.findall(r'^#define (MAX_\w+|VOICES)\s+(\d+)', tunec, re.M)} said = { "VOICES": r'^\| Voices \| (\d+) \|', "MAX_PATCHES": r'^\| Patches \| (\d+) \|', "MAX_SEQUENCES": r'^\| Sequences \| (\d+)\.', "MAX_EVENTS": r'^\| Events in a sequence \| (\d+) ', "MAX_ORDER": r'^\| Entries in an order list \| (\d+) \|', } for name, pattern in said.items(): if name not in limits: problems.append("TuneC no longer defines %s, which the Tune Manual quotes" % name) continue m = re.search(pattern, tm, re.M) if not m: problems.append("the Tune Manual has lost its %s limit" % name) elif int(m.group(1)) != limits[name]: problems.append("the Tune Manual says %s is %s and TuneC says %d" % (name, m.group(1), limits[name])) if problems: print("The manuals and the code disagree:") for p in problems: print(" " + p) sys.exit(1) print("The manuals agree with the code.") PY