mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-08-17 06:20:34 +00:00
65 lines
2.1 KiB
Python
Executable file
65 lines
2.1 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Generated by `atlas autoresearch`. Parse one metric out of a run's output
|
|
into the Atlas benchmark contract.
|
|
|
|
EDIT the METRIC line if the wrong number is being read, or rewrite parse() for
|
|
a richer evaluator (per-example feedback is what makes GEPA converge — see
|
|
.atlas/benchmark-contract or the atlas-optimize skill)."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
# autoresearch guessed this metric name from your --goal.
|
|
METRIC = "find_optimal_crossing" # <-- EDIT ME if this is the wrong metric
|
|
HIGHER_IS_BETTER = False
|
|
|
|
# Matches "<metric> = 0.74", "<metric>: 0.74", "<metric> 0.74".
|
|
PATTERN = re.compile(
|
|
re.escape(METRIC) + r"\s*[:=]?\s*([+-]?[0-9]*\.?[0-9]+(?:[eE][+-]?[0-9]+)?)"
|
|
)
|
|
|
|
|
|
def parse(text):
|
|
matches = PATTERN.findall(text)
|
|
if not matches:
|
|
return None
|
|
return float(matches[-1]) # last occurrence == final epoch / final eval
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--stdout", required=True, help="file holding the run's captured output")
|
|
args = ap.parse_args()
|
|
text = open(args.stdout, encoding="utf-8", errors="replace").read()
|
|
score = parse(text)
|
|
if score is None:
|
|
sys.stderr.write(
|
|
f"[score] could not find metric '{METRIC}' in the run output.\n"
|
|
" Fix: edit .atlas/score.py (METRIC=...), or pass an explicit\n"
|
|
" --benchmark to 'atlas autoresearch'.\n"
|
|
)
|
|
sys.exit(3)
|
|
direction = "higher is better" if HIGHER_IS_BETTER else "lower is better"
|
|
result = {
|
|
"score": score,
|
|
"examples": [{
|
|
"id": METRIC,
|
|
"score": score,
|
|
"pass": True,
|
|
"feedback": f"{METRIC} = {score} (parsed from run output; {direction})",
|
|
}],
|
|
"feedback": f"Final {METRIC} = {score}.",
|
|
}
|
|
out = os.environ.get("ATLAS_OPTIMIZE_RESULT")
|
|
payload = json.dumps(result)
|
|
if out:
|
|
with open(out, "w", encoding="utf-8") as fh:
|
|
fh.write(payload)
|
|
else:
|
|
sys.stdout.write(payload)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|