diff --git a/.atlas/benchmark.sh b/.atlas/benchmark.sh index 4531fef0..3d3278c4 100755 --- a/.atlas/benchmark.sh +++ b/.atlas/benchmark.sh @@ -14,7 +14,8 @@ OUT="$(mktemp)" # --- the repo's run command (auto-detected) -------------------------------- # Edit this line if the campaign should run something different. -( python3 "$ATLAS_OPTIMIZE_TARGET" ) >"$OUT" 2>&1 +TARGET="${ATLAS_OPTIMIZE_TARGET:-$1}" +( python3 "$TARGET" ) >"$OUT" 2>&1 RC=$? # --------------------------------------------------------------------------- diff --git a/.atlas/gate.sh b/.atlas/gate.sh index 2a16ee63..6818066a 100755 --- a/.atlas/gate.sh +++ b/.atlas/gate.sh @@ -8,7 +8,7 @@ # rewrites tracked files (checkpoints, logs, and .atlas/ are already ignored). set -uo pipefail WT="${1:-$PWD}" -TARGET="${2:-${ATLAS_OPTIMIZE_TARGET:-}}" +TARGET="${ATLAS_OPTIMIZE_TARGET:-$2}" REL="${TARGET#"$WT"/}" IGNORE_RE='^(outputs/|out/|checkpoints/|runs/|wandb/|\.atlas/|.*\.log$|.*\.ckpt$|.*\.pt$|.*\.bin$|.*\.safetensors$)' CHANGED="$(git -C "$WT" status --porcelain --untracked-files=no 2>/dev/null \ diff --git a/.atlas/score.py b/.atlas/score.py index 7d9f2173..1a8f8064 100755 --- a/.atlas/score.py +++ b/.atlas/score.py @@ -1,65 +1,73 @@ #!/usr/bin/env python3 -"""Generated by `atlas autoresearch`. Parse one metric out of a run's output -into the Atlas benchmark contract. - -EDIT the METRIC line if the wrong number is being read, or rewrite parse() for -a richer evaluator (per-example feedback is what makes GEPA converge — see -.atlas/benchmark-contract or the atlas-optimize skill).""" -import argparse -import json -import os -import re -import sys - -# autoresearch guessed this metric name from your --goal. -METRIC = "find_optimal_crossing" # <-- EDIT ME if this is the wrong metric -HIGHER_IS_BETTER = False - -# Matches " = 0.74", ": 0.74", " 0.74". -PATTERN = re.compile( - re.escape(METRIC) + r"\s*[:=]?\s*([+-]?[0-9]*\.?[0-9]+(?:[eE][+-]?[0-9]+)?)" -) +import argparse, json, os, re, sys +METRICS = { + "find_optimal_crossing": False, + "SA direct": False, +} def parse(text): - matches = PATTERN.findall(text) - if not matches: - return None - return float(matches[-1]) # last occurrence == final epoch / final eval - + results = {} + for name, higher in METRICS.items(): + pat = re.compile(re.escape(name) + r"\s*[:=]?\s*([+-]?[0-9]*\.?[0-9]+)") + for m in pat.finditer(text): + raw = m.group(0).strip() + val = float(m.group(1)) + line_before = text[max(0, m.start()-80):m.start()].split("\n")[-1].strip() + results[name] = (val, higher, raw, line_before) + return results def main(): ap = argparse.ArgumentParser() - ap.add_argument("--stdout", required=True, help="file holding the run's captured output") + ap.add_argument("--stdout", required=True) args = ap.parse_args() text = open(args.stdout, encoding="utf-8", errors="replace").read() - score = parse(text) - if score is None: - sys.stderr.write( - f"[score] could not find metric '{METRIC}' in the run output.\n" - " Fix: edit .atlas/score.py (METRIC=...), or pass an explicit\n" - " --benchmark to 'atlas autoresearch'.\n" - ) + parsed = parse(text) + + if "find_optimal_crossing" not in parsed: + sys.stderr.write(f"[score] could not find metric in output.\n") sys.exit(3) - direction = "higher is better" if HIGHER_IS_BETTER else "lower is better" + + primary_name = "find_optimal_crossing" + primary_val, _, primary_raw, ctx = parsed[primary_name] + direction = "lower is better" + + examples = [{ + "id": primary_name, + "score": primary_val, + "pass": True, + "feedback": f"{primary_raw} | context: {ctx}" if ctx else primary_raw, + }] + + if "SA direct" in parsed: + sa_val, _, sa_raw, sa_ctx = parsed["SA direct"] + examples.append({ + "id": "SA direct", + "score": sa_val, + "pass": sa_val < 100, + "feedback": f"{sa_raw} | context: {sa_ctx}" if sa_ctx else sa_raw, + }) + + diag_lines = [] + for line in text.strip().split("\n"): + stripped = line.strip() + if "find_optimal_crossing" in stripped or "SA direct" in stripped: + diag_lines.append(stripped) + diagnostic = "; ".join(diag_lines) if diag_lines else primary_raw + result = { - "score": score, - "examples": [{ - "id": METRIC, - "score": score, - "pass": True, - "feedback": f"{METRIC} = {score} (parsed from run output; {direction})", - }], - "feedback": f"Final {METRIC} = {score}.", + "score": primary_val, + "examples": examples, + "feedback": f"find_optimal_crossing={primary_val} ({direction}) | {diagnostic}", } + out = os.environ.get("ATLAS_OPTIMIZE_RESULT") payload = json.dumps(result) if out: - with open(out, "w", encoding="utf-8") as fh: + with open(out, "w") as fh: fh.write(payload) else: sys.stdout.write(payload) - if __name__ == "__main__": main()