mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
351 lines
13 KiB
Python
Executable file
351 lines
13 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""
|
|
Run an array of tiny GCL boot surfaces under QEMU.
|
|
|
|
The harness builds the boot sector once, launches N independent QEMU instances,
|
|
captures each serial log, and validates the recovery pulse. It is designed to
|
|
expose concurrency, boot drift, malformed pulse, and timeout bugs early.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
BUILD_DIR = ROOT / "build"
|
|
BOOT_BIN = BUILD_DIR / "gcl_boot.bin"
|
|
ARRAY_DIR = BUILD_DIR / "array"
|
|
|
|
REQUIRED_LINES = (
|
|
"GCLBOOT v0.1",
|
|
"surface=node kind=qemu.boot op=recover",
|
|
"ot1 op=0x0D source=0x002A route=0x0001 seq=0x0001",
|
|
"gcl admission=admitted invariant=recovery_subset",
|
|
"BOOT_OK",
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class NodeRun:
|
|
index: int
|
|
attempt: int
|
|
log_path: Path
|
|
process: subprocess.Popen[bytes]
|
|
started_at: float
|
|
boot_ok_at: float | None = None
|
|
error: str | None = None
|
|
|
|
|
|
def run(cmd: list[str]) -> None:
|
|
subprocess.run(cmd, cwd=ROOT, check=True)
|
|
|
|
|
|
def verify_boot_image() -> None:
|
|
if not BOOT_BIN.exists():
|
|
raise RuntimeError(f"missing boot image: {BOOT_BIN}")
|
|
data = BOOT_BIN.read_bytes()
|
|
if len(data) != 512:
|
|
raise RuntimeError(f"boot image must be 512 bytes, got {len(data)}")
|
|
if data[510:512] != b"\x55\xaa":
|
|
raise RuntimeError("boot image is missing BIOS 0x55AA signature")
|
|
|
|
|
|
def spawn_node(index: int, attempt: int, memory: str) -> NodeRun:
|
|
log_path = ARRAY_DIR / f"node-{index:02d}.attempt-{attempt:02d}.serial.log"
|
|
log_path.unlink(missing_ok=True)
|
|
cmd = [
|
|
"qemu-system-x86_64",
|
|
"-machine",
|
|
"accel=tcg",
|
|
"-m",
|
|
memory,
|
|
"-display",
|
|
"none",
|
|
"-no-reboot",
|
|
"-drive",
|
|
f"format=raw,file={BOOT_BIN},if=floppy,readonly=on",
|
|
"-serial",
|
|
f"file:{log_path}",
|
|
"-monitor",
|
|
"none",
|
|
"-no-shutdown",
|
|
]
|
|
return NodeRun(
|
|
index=index,
|
|
attempt=attempt,
|
|
log_path=log_path,
|
|
process=subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE),
|
|
started_at=time.monotonic(),
|
|
)
|
|
|
|
|
|
def read_log(path: Path) -> str:
|
|
try:
|
|
return path.read_text(encoding="ascii", errors="replace")
|
|
except FileNotFoundError:
|
|
return ""
|
|
|
|
|
|
def mutate_observation(
|
|
text: str,
|
|
corrupt_rate: float,
|
|
duplicate_rate: float,
|
|
truncate_rate: float,
|
|
rng: random.Random,
|
|
) -> str:
|
|
if text and truncate_rate > 0.0 and rng.random() < truncate_rate:
|
|
text = text[: rng.randrange(0, len(text))]
|
|
if text and duplicate_rate > 0.0 and rng.random() < duplicate_rate:
|
|
text += "BOOT_OK\r\n"
|
|
if text and corrupt_rate > 0.0 and rng.random() < corrupt_rate:
|
|
edits = (
|
|
("BOOT_OK", "B00T_OK"),
|
|
("op=recover", "op=unknown"),
|
|
("admission=admitted", "admission=quarantined"),
|
|
("source=0x002A", "source=0xFFFF"),
|
|
)
|
|
old, new = rng.choice(edits)
|
|
text = text.replace(old, new, 1)
|
|
return text
|
|
|
|
|
|
def observed_log(
|
|
path: Path,
|
|
drop_rate: float,
|
|
stale_rate: float,
|
|
corrupt_rate: float,
|
|
duplicate_rate: float,
|
|
truncate_rate: float,
|
|
cache: dict[Path, str],
|
|
rng: random.Random,
|
|
) -> str:
|
|
if drop_rate > 0.0 and rng.random() < drop_rate:
|
|
return ""
|
|
if stale_rate > 0.0 and path in cache and rng.random() < stale_rate:
|
|
return cache[path]
|
|
text = read_log(path)
|
|
text = mutate_observation(text, corrupt_rate, duplicate_rate, truncate_rate, rng)
|
|
cache[path] = text
|
|
return text
|
|
|
|
|
|
def validate_log(text: str) -> str | None:
|
|
missing = [line for line in REQUIRED_LINES if line not in text]
|
|
if missing:
|
|
return "missing serial line(s): " + ", ".join(missing)
|
|
if text.count("BOOT_OK") != 1:
|
|
return f"expected one BOOT_OK marker, saw {text.count('BOOT_OK')}"
|
|
if "op=recover" not in text:
|
|
return "missing recovery op"
|
|
if "admission=admitted" not in text:
|
|
return "missing GCL admission"
|
|
return None
|
|
|
|
|
|
def stop_node(node: NodeRun) -> None:
|
|
if node.process.poll() is None:
|
|
node.process.terminate()
|
|
try:
|
|
node.process.wait(timeout=1.0)
|
|
except subprocess.TimeoutExpired:
|
|
node.process.kill()
|
|
node.process.wait(timeout=1.0)
|
|
|
|
|
|
def stderr_tail(node: NodeRun) -> str:
|
|
if node.process.stderr is None:
|
|
return ""
|
|
try:
|
|
data = node.process.stderr.read() or b""
|
|
except ValueError:
|
|
return ""
|
|
return data.decode("utf-8", errors="replace").strip()[-400:]
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--count", type=int, default=10, help="QEMU instances to launch")
|
|
parser.add_argument("--timeout", type=float, default=5.0, help="seconds per node")
|
|
parser.add_argument("--memory", default="8M", help="guest memory per node")
|
|
parser.add_argument(
|
|
"--scenario",
|
|
choices=("clean", "finicky", "dodgy"),
|
|
default="clean",
|
|
help="test profile; dodgy simulates a bargain-bin control plane",
|
|
)
|
|
parser.add_argument("--seed", type=int, default=510510, help="deterministic jitter seed")
|
|
parser.add_argument("--retries", type=int, default=0, help="extra attempts per node")
|
|
parser.add_argument("--quorum", type=int, default=None, help="minimum successful nodes")
|
|
parser.add_argument("--start-jitter", type=float, default=0.0, help="max launch delay per node")
|
|
parser.add_argument("--poll-min", type=float, default=0.05, help="minimum polling delay")
|
|
parser.add_argument("--poll-max", type=float, default=0.05, help="maximum polling delay")
|
|
parser.add_argument("--observe-drop-rate", type=float, default=0.0, help="chance to miss a log read")
|
|
parser.add_argument("--observe-stale-rate", type=float, default=0.0, help="chance to reuse old observer data")
|
|
parser.add_argument("--observe-corrupt-rate", type=float, default=0.0, help="chance to corrupt observer data")
|
|
parser.add_argument("--observe-duplicate-rate", type=float, default=0.0, help="chance to duplicate BOOT_OK")
|
|
parser.add_argument("--observe-truncate-rate", type=float, default=0.0, help="chance to truncate observer data")
|
|
args = parser.parse_args()
|
|
|
|
if args.count < 1:
|
|
parser.error("--count must be >= 1")
|
|
if args.retries < 0:
|
|
parser.error("--retries must be >= 0")
|
|
if not 0.0 <= args.observe_drop_rate <= 1.0:
|
|
parser.error("--observe-drop-rate must be between 0 and 1")
|
|
for rate_name in (
|
|
"observe_stale_rate",
|
|
"observe_corrupt_rate",
|
|
"observe_duplicate_rate",
|
|
"observe_truncate_rate",
|
|
):
|
|
if not 0.0 <= getattr(args, rate_name) <= 1.0:
|
|
parser.error(f"--{rate_name.replace('_', '-')} must be between 0 and 1")
|
|
if args.poll_min <= 0.0 or args.poll_max < args.poll_min:
|
|
parser.error("--poll-min/--poll-max must be positive and ordered")
|
|
|
|
if args.scenario in ("finicky", "dodgy"):
|
|
args.count = 12 if args.count == 10 else args.count
|
|
args.timeout = 0.45 if args.timeout == 5.0 else args.timeout
|
|
args.retries = 3 if args.retries == 0 else args.retries
|
|
args.quorum = args.count if args.quorum is None else args.quorum
|
|
args.start_jitter = 0.25 if args.start_jitter == 0.0 else args.start_jitter
|
|
args.poll_min = 0.02 if args.poll_min == 0.05 else args.poll_min
|
|
args.poll_max = 0.22 if args.poll_max == 0.05 else args.poll_max
|
|
args.observe_drop_rate = 0.35 if args.observe_drop_rate == 0.0 else args.observe_drop_rate
|
|
if args.scenario == "dodgy":
|
|
args.count = 16 if args.count == 12 else args.count
|
|
args.timeout = 0.30 if args.timeout == 0.45 else args.timeout
|
|
args.retries = 6 if args.retries == 3 else args.retries
|
|
args.quorum = 12 if args.quorum == args.count else args.quorum
|
|
args.start_jitter = 0.40 if args.start_jitter == 0.25 else args.start_jitter
|
|
args.poll_max = 0.35 if args.poll_max == 0.22 else args.poll_max
|
|
args.observe_drop_rate = 0.45 if args.observe_drop_rate == 0.35 else args.observe_drop_rate
|
|
args.observe_stale_rate = 0.25 if args.observe_stale_rate == 0.0 else args.observe_stale_rate
|
|
args.observe_corrupt_rate = 0.18 if args.observe_corrupt_rate == 0.0 else args.observe_corrupt_rate
|
|
args.observe_duplicate_rate = 0.12 if args.observe_duplicate_rate == 0.0 else args.observe_duplicate_rate
|
|
args.observe_truncate_rate = 0.20 if args.observe_truncate_rate == 0.0 else args.observe_truncate_rate
|
|
|
|
quorum = args.count if args.quorum is None else args.quorum
|
|
if quorum < 1 or quorum > args.count:
|
|
parser.error("--quorum must be between 1 and --count")
|
|
|
|
rng = random.Random(args.seed)
|
|
|
|
run(["make", "build/gcl_boot.bin"])
|
|
verify_boot_image()
|
|
ARRAY_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
current: dict[int, NodeRun] = {}
|
|
final: dict[int, NodeRun] = {}
|
|
attempts = {index: 0 for index in range(args.count)}
|
|
history: dict[int, list[str]] = {index: [] for index in range(args.count)}
|
|
observer_cache: dict[Path, str] = {}
|
|
|
|
def launch(index: int) -> None:
|
|
if args.start_jitter > 0.0:
|
|
time.sleep(rng.uniform(0.0, args.start_jitter))
|
|
current[index] = spawn_node(index, attempts[index], args.memory)
|
|
|
|
for node_index in range(args.count):
|
|
launch(node_index)
|
|
|
|
try:
|
|
while current:
|
|
now = time.monotonic()
|
|
for index, node in list(current.items()):
|
|
text = observed_log(
|
|
node.log_path,
|
|
args.observe_drop_rate,
|
|
args.observe_stale_rate,
|
|
args.observe_corrupt_rate,
|
|
args.observe_duplicate_rate,
|
|
args.observe_truncate_rate,
|
|
observer_cache,
|
|
rng,
|
|
)
|
|
timed_out = (now - node.started_at) >= args.timeout
|
|
if "BOOT_OK" in text:
|
|
node.boot_ok_at = now
|
|
node.error = validate_log(text)
|
|
stop_node(node)
|
|
del current[index]
|
|
if node.error and attempts[index] < args.retries:
|
|
history[index].append(f"attempt-{node.attempt:02d}: {node.error}")
|
|
attempts[index] += 1
|
|
launch(index)
|
|
else:
|
|
final[index] = node
|
|
elif node.process.poll() is not None:
|
|
node.error = f"qemu exited early with {node.process.returncode}: {stderr_tail(node)}"
|
|
del current[index]
|
|
if attempts[index] < args.retries:
|
|
history[index].append(f"attempt-{node.attempt:02d}: {node.error}")
|
|
attempts[index] += 1
|
|
launch(index)
|
|
else:
|
|
final[index] = node
|
|
elif timed_out:
|
|
visible_text = read_log(node.log_path)
|
|
if "BOOT_OK" in visible_text:
|
|
node.error = "observer missed BOOT_OK before timeout"
|
|
else:
|
|
node.error = f"timeout after {args.timeout:.2f}s"
|
|
stop_node(node)
|
|
del current[index]
|
|
if attempts[index] < args.retries:
|
|
history[index].append(f"attempt-{node.attempt:02d}: {node.error}")
|
|
attempts[index] += 1
|
|
launch(index)
|
|
else:
|
|
final[index] = node
|
|
if current:
|
|
time.sleep(rng.uniform(args.poll_min, args.poll_max))
|
|
finally:
|
|
for node in current.values():
|
|
stop_node(node)
|
|
|
|
failures = [node for node in final.values() if node.error]
|
|
successes = [node for node in final.values() if not node.error]
|
|
for node in [final[index] for index in sorted(final)]:
|
|
elapsed = (node.boot_ok_at or time.monotonic()) - node.started_at
|
|
status = "FAIL" if node.error else "OK"
|
|
print(
|
|
f"node-{node.index:02d} attempt-{node.attempt:02d} {status} "
|
|
f"{elapsed:.3f}s {node.log_path.relative_to(ROOT)}"
|
|
)
|
|
for event in history[node.index]:
|
|
print(f" recovered: {event}")
|
|
if node.error:
|
|
print(f" {node.error}")
|
|
text = read_log(node.log_path).strip()
|
|
if text:
|
|
print(" serial:")
|
|
for line in text.splitlines()[-8:]:
|
|
print(f" {line}")
|
|
|
|
if len(successes) < quorum:
|
|
print(
|
|
f"array result: quorum failed, {len(successes)}/{args.count} succeeded "
|
|
f"(quorum {quorum})",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
|
|
print(
|
|
f"array result: {len(successes)}/{args.count} booted and emitted valid "
|
|
f"recovery pulses under {args.scenario} visibility"
|
|
)
|
|
if failures:
|
|
print(f"array warning: {len(failures)} node(s) failed after retries but quorum held")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|