From 7fcfde3ca7d1966e1285bcd7a6ad53113290bcc4 Mon Sep 17 00:00:00 2001 From: allaun Date: Tue, 7 Jul 2026 01:08:41 -0500 Subject: [PATCH] feat(polyglot): add GEPA infrastructure for Python shim optimization - Add .atlas/ with benchmark.sh, score.py, gate.sh for GEPA campaigns - Add atlas.json project config (auto.capture=suggest, reflection_lm=openai/auto) - Fix test_search.py receipt path from /mnt/agents/ to /tmp/ Metric targets: dna_qubo_sort.py: avg |Rank corr| (higher=better, baseline=0.8547) dna_qubo_nn.py: NN Tm correlation (higher=better, baseline=0.4649) dna_lut.py: avg |Rank corr| (higher=better, baseline=0.5808) test_search.py: pass count (ceiling=43, for stability verification) --- .atlas/benchmark.sh | 20 +++++++++++ .atlas/gate.sh | 31 ++++++++++++++++ .atlas/score.py | 83 +++++++++++++++++++++++++++++++++++++++++++ atlas.json | 9 +++++ python/test_search.py | 2 +- 5 files changed, 144 insertions(+), 1 deletion(-) create mode 100755 .atlas/benchmark.sh create mode 100755 .atlas/gate.sh create mode 100644 .atlas/score.py create mode 100644 atlas.json diff --git a/.atlas/benchmark.sh b/.atlas/benchmark.sh new file mode 100755 index 00000000..e6336b9b --- /dev/null +++ b/.atlas/benchmark.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT="$(mktemp)" + +TARGET="${ATLAS_OPTIMIZE_TARGET:-$1}" +export ATLAS_EVAL_SEED="${ATLAS_EVAL_SEED:-0}" + +START=$(date +%s.%N) +( python3 "$TARGET" ) >"$OUT" 2>&1 +RC=$? +END=$(date +%s.%N) + +if [ "$RC" -ne 0 ]; then + echo "[benchmark] run command exited $RC — candidate failed" >&2 + tail -n 40 "$OUT" >&2 + exit "$RC" +fi + +python3 "$HERE/score.py" --stdout "$OUT" --elapsed "$(awk "BEGIN { print $END - $START }")" diff --git a/.atlas/gate.sh b/.atlas/gate.sh new file mode 100755 index 00000000..95feb8bd --- /dev/null +++ b/.atlas/gate.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Goodhart guard: reject candidates that lower quality while improving timing +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TARGET="${ATLAS_OPTIMIZE_TARGET:-$2}" +REF_SCORE="${ATLAS_REF_SCORE:-$1}" + +NEW_OUT="$(mktemp)" +NEW_START=$(date +%s.%N) +( python3 "$TARGET" ) >"$NEW_OUT" 2>&1 +NEW_RC=$? +NEW_END=$(date +%s.%N) +NEW_ELAPSED=$(awk "BEGIN { print $NEW_END - $NEW_START }") + +if [ "$NEW_RC" -ne 0 ]; then + echo "[gate] candidate failed — rejecting" + exit 1 +fi + +NEW_SCORE=$(python3 "$HERE/score.py" --stdout "$NEW_OUT" --elapsed "$NEW_ELAPSED" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['score'])") + +# Accept if score >= reference (higher is better) +ACCEPT=$(python3 -c "print('yes' if $NEW_SCORE >= $REF_SCORE - 0.01 else 'no')") +if [ "$ACCEPT" = "yes" ]; then + echo "[gate] accepted (score=$NEW_SCORE >= ref=$REF_SCORE)" + exit 0 +else + echo "[gate] rejected (score=$NEW_SCORE < ref=$REF_SCORE)" + exit 1 +fi diff --git a/.atlas/score.py b/.atlas/score.py new file mode 100644 index 00000000..2c42503e --- /dev/null +++ b/.atlas/score.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +import argparse, json, os, re, sys + +METRICS = { + "Rank corr": True, + "Rank correlation": True, + "NN Tm correlation": True, + "Naive Tm correlation": True, +} + +def parse(text): + results = {} + for name, higher in METRICS.items(): + pat = re.compile(re.escape(name) + r"\s*:\s*([+-]?[0-9]*\.?[0-9]+)", re.IGNORECASE) + matches = pat.finditer(text) + vals = [] + for m in matches: + vals.append((float(m.group(1)), higher, m.group(0).strip())) + if vals: + results[name] = vals + return results + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--stdout", required=True) + ap.add_argument("--elapsed", type=float, default=None) + args = ap.parse_args() + text = open(args.stdout, encoding="utf-8", errors="replace").read() + parsed = parse(text) + + score = None + feedback_parts = [] + + if "NN Tm correlation" in parsed: + val, _, raw = parsed["NN Tm correlation"][0] + score = abs(val) + feedback_parts.append(f"NN Tm corr={val}") + elif "Rank correlation" in parsed: + vals = [abs(v) for v, _, _ in parsed["Rank correlation"]] + score = sum(vals) / len(vals) + feedback_parts.append(f"avg |Rank corr|={score:.4f}") + elif "Rank corr" in parsed: + vals = [abs(v) for v, _, _ in parsed["Rank corr"]] + score = sum(vals) / len(vals) + feedback_parts.append(f"avg |Rank corr|={score:.4f}") + elif "Naive Tm correlation" in parsed: + val, _, raw = parsed["Naive Tm correlation"][0] + score = abs(val) + feedback_parts.append(f"Naive Tm corr={val}") + + if score is None and args.elapsed is not None: + score = -(args.elapsed * 1000) + feedback_parts.append(f"time={args.elapsed*1000:.1f}ms") + elif score is None: + m = re.search(r"SUMMARY:\s*(\d+)/(\d+)\s*passed", text) + if m: + score = float(m.group(1)) + feedback_parts.append(f"passes={m.group(1)}/{m.group(2)}") + else: + sys.stderr.write("[score] no metric found in output.\n") + sys.exit(3) + + result = { + "score": score, + "examples": [{ + "id": "metric", + "score": score, + "pass": True, + "feedback": " | ".join(feedback_parts), + }], + "feedback": " | ".join(feedback_parts), + } + + out = os.environ.get("ATLAS_OPTIMIZE_RESULT") + payload = json.dumps(result) + if out: + with open(out, "w", encoding="utf-8") as fh: + fh.write(payload) + else: + sys.stdout.write(payload) + +if __name__ == "__main__": + main() diff --git a/atlas.json b/atlas.json new file mode 100644 index 00000000..49053c4a --- /dev/null +++ b/atlas.json @@ -0,0 +1,9 @@ +{ + "project": "silversight", + "description": "SilverSight polyglot formalization — GEPA-optimize Python, port to 11 languages", + "auto": { + "capture": "suggest", + "prompt": "Rewrite this Python code to be more performant and correct. Keep the same public interface (same function signatures, same class names, same CLI behavior). Optimize for algorithmic quality (rank correlation, Tm correlation) and speed.", + "reflection_lm": "openai/auto" + } +} diff --git a/python/test_search.py b/python/test_search.py index 18b79ad6..eadc24c3 100644 --- a/python/test_search.py +++ b/python/test_search.py @@ -378,7 +378,7 @@ def main(): # Generate receipt receipt = generate_receipt(all_search_results, schema_version="stage3_v1") - receipt_path = "/mnt/agents/output/rebuild/stage3-search/chaos_game_receipt.json" + receipt_path = "/tmp/chaos_game_receipt.json" with open(receipt_path, "w") as f: json.dump(receipt, f, indent=2) print(f"\nReceipt: {receipt_path}")