diff --git a/.atlas/benchmark.sh b/.atlas/benchmark.sh new file mode 100755 index 00000000..e6336b9b --- /dev/null +++ b/.atlas/benchmark.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT="$(mktemp)" + +TARGET="${ATLAS_OPTIMIZE_TARGET:-$1}" +export ATLAS_EVAL_SEED="${ATLAS_EVAL_SEED:-0}" + +START=$(date +%s.%N) +( python3 "$TARGET" ) >"$OUT" 2>&1 +RC=$? +END=$(date +%s.%N) + +if [ "$RC" -ne 0 ]; then + echo "[benchmark] run command exited $RC — candidate failed" >&2 + tail -n 40 "$OUT" >&2 + exit "$RC" +fi + +python3 "$HERE/score.py" --stdout "$OUT" --elapsed "$(awk "BEGIN { print $END - $START }")" diff --git a/.atlas/gate.sh b/.atlas/gate.sh new file mode 100755 index 00000000..95feb8bd --- /dev/null +++ b/.atlas/gate.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Goodhart guard: reject candidates that lower quality while improving timing +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TARGET="${ATLAS_OPTIMIZE_TARGET:-$2}" +REF_SCORE="${ATLAS_REF_SCORE:-$1}" + +NEW_OUT="$(mktemp)" +NEW_START=$(date +%s.%N) +( python3 "$TARGET" ) >"$NEW_OUT" 2>&1 +NEW_RC=$? +NEW_END=$(date +%s.%N) +NEW_ELAPSED=$(awk "BEGIN { print $NEW_END - $NEW_START }") + +if [ "$NEW_RC" -ne 0 ]; then + echo "[gate] candidate failed — rejecting" + exit 1 +fi + +NEW_SCORE=$(python3 "$HERE/score.py" --stdout "$NEW_OUT" --elapsed "$NEW_ELAPSED" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['score'])") + +# Accept if score >= reference (higher is better) +ACCEPT=$(python3 -c "print('yes' if $NEW_SCORE >= $REF_SCORE - 0.01 else 'no')") +if [ "$ACCEPT" = "yes" ]; then + echo "[gate] accepted (score=$NEW_SCORE >= ref=$REF_SCORE)" + exit 0 +else + echo "[gate] rejected (score=$NEW_SCORE < ref=$REF_SCORE)" + exit 1 +fi diff --git a/.atlas/score.py b/.atlas/score.py new file mode 100644 index 00000000..2c42503e --- /dev/null +++ b/.atlas/score.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +import argparse, json, os, re, sys + +METRICS = { + "Rank corr": True, + "Rank correlation": True, + "NN Tm correlation": True, + "Naive Tm correlation": True, +} + +def parse(text): + results = {} + for name, higher in METRICS.items(): + pat = re.compile(re.escape(name) + r"\s*:\s*([+-]?[0-9]*\.?[0-9]+)", re.IGNORECASE) + matches = pat.finditer(text) + vals = [] + for m in matches: + vals.append((float(m.group(1)), higher, m.group(0).strip())) + if vals: + results[name] = vals + return results + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--stdout", required=True) + ap.add_argument("--elapsed", type=float, default=None) + args = ap.parse_args() + text = open(args.stdout, encoding="utf-8", errors="replace").read() + parsed = parse(text) + + score = None + feedback_parts = [] + + if "NN Tm correlation" in parsed: + val, _, raw = parsed["NN Tm correlation"][0] + score = abs(val) + feedback_parts.append(f"NN Tm corr={val}") + elif "Rank correlation" in parsed: + vals = [abs(v) for v, _, _ in parsed["Rank correlation"]] + score = sum(vals) / len(vals) + feedback_parts.append(f"avg |Rank corr|={score:.4f}") + elif "Rank corr" in parsed: + vals = [abs(v) for v, _, _ in parsed["Rank corr"]] + score = sum(vals) / len(vals) + feedback_parts.append(f"avg |Rank corr|={score:.4f}") + elif "Naive Tm correlation" in parsed: + val, _, raw = parsed["Naive Tm correlation"][0] + score = abs(val) + feedback_parts.append(f"Naive Tm corr={val}") + + if score is None and args.elapsed is not None: + score = -(args.elapsed * 1000) + feedback_parts.append(f"time={args.elapsed*1000:.1f}ms") + elif score is None: + m = re.search(r"SUMMARY:\s*(\d+)/(\d+)\s*passed", text) + if m: + score = float(m.group(1)) + feedback_parts.append(f"passes={m.group(1)}/{m.group(2)}") + else: + sys.stderr.write("[score] no metric found in output.\n") + sys.exit(3) + + result = { + "score": score, + "examples": [{ + "id": "metric", + "score": score, + "pass": True, + "feedback": " | ".join(feedback_parts), + }], + "feedback": " | ".join(feedback_parts), + } + + out = os.environ.get("ATLAS_OPTIMIZE_RESULT") + payload = json.dumps(result) + if out: + with open(out, "w", encoding="utf-8") as fh: + fh.write(payload) + else: + sys.stdout.write(payload) + +if __name__ == "__main__": + main() diff --git a/atlas.json b/atlas.json new file mode 100644 index 00000000..49053c4a --- /dev/null +++ b/atlas.json @@ -0,0 +1,9 @@ +{ + "project": "silversight", + "description": "SilverSight polyglot formalization — GEPA-optimize Python, port to 11 languages", + "auto": { + "capture": "suggest", + "prompt": "Rewrite this Python code to be more performant and correct. Keep the same public interface (same function signatures, same class names, same CLI behavior). Optimize for algorithmic quality (rank correlation, Tm correlation) and speed.", + "reflection_lm": "openai/auto" + } +} diff --git a/python/test_search.py b/python/test_search.py index 18b79ad6..eadc24c3 100644 --- a/python/test_search.py +++ b/python/test_search.py @@ -378,7 +378,7 @@ def main(): # Generate receipt receipt = generate_receipt(all_search_results, schema_version="stage3_v1") - receipt_path = "/mnt/agents/output/rebuild/stage3-search/chaos_game_receipt.json" + receipt_path = "/tmp/chaos_game_receipt.json" with open(receipt_path, "w") as f: json.dump(receipt, f, indent=2) print(f"\nReceipt: {receipt_path}")