feat(polyglot): add GEPA infrastructure for Python shim optimization

- Add .atlas/ with benchmark.sh, score.py, gate.sh for GEPA campaigns
- Add atlas.json project config (auto.capture=suggest, reflection_lm=openai/auto)
- Fix test_search.py receipt path from /mnt/agents/ to /tmp/

Metric targets:
  dna_qubo_sort.py: avg |Rank corr| (higher=better, baseline=0.8547)
  dna_qubo_nn.py:   NN Tm correlation (higher=better, baseline=0.4649)
  dna_lut.py:       avg |Rank corr| (higher=better, baseline=0.5808)
  test_search.py:   pass count (ceiling=43, for stability verification)
This commit is contained in:
allaun 2026-07-07 01:08:41 -05:00
parent 8b79a75ea4
commit 7fcfde3ca7
5 changed files with 144 additions and 1 deletions

20
.atlas/benchmark.sh Executable file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OUT="$(mktemp)"
TARGET="${ATLAS_OPTIMIZE_TARGET:-$1}"
export ATLAS_EVAL_SEED="${ATLAS_EVAL_SEED:-0}"
START=$(date +%s.%N)
( python3 "$TARGET" ) >"$OUT" 2>&1
RC=$?
END=$(date +%s.%N)
if [ "$RC" -ne 0 ]; then
echo "[benchmark] run command exited $RC — candidate failed" >&2
tail -n 40 "$OUT" >&2
exit "$RC"
fi
python3 "$HERE/score.py" --stdout "$OUT" --elapsed "$(awk "BEGIN { print $END - $START }")"

31
.atlas/gate.sh Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
# Goodhart guard: reject candidates that lower quality while improving timing
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TARGET="${ATLAS_OPTIMIZE_TARGET:-$2}"
REF_SCORE="${ATLAS_REF_SCORE:-$1}"
NEW_OUT="$(mktemp)"
NEW_START=$(date +%s.%N)
( python3 "$TARGET" ) >"$NEW_OUT" 2>&1
NEW_RC=$?
NEW_END=$(date +%s.%N)
NEW_ELAPSED=$(awk "BEGIN { print $NEW_END - $NEW_START }")
if [ "$NEW_RC" -ne 0 ]; then
echo "[gate] candidate failed — rejecting"
exit 1
fi
NEW_SCORE=$(python3 "$HERE/score.py" --stdout "$NEW_OUT" --elapsed "$NEW_ELAPSED" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['score'])")
# Accept if score >= reference (higher is better)
ACCEPT=$(python3 -c "print('yes' if $NEW_SCORE >= $REF_SCORE - 0.01 else 'no')")
if [ "$ACCEPT" = "yes" ]; then
echo "[gate] accepted (score=$NEW_SCORE >= ref=$REF_SCORE)"
exit 0
else
echo "[gate] rejected (score=$NEW_SCORE < ref=$REF_SCORE)"
exit 1
fi

83
.atlas/score.py Normal file
View file

@ -0,0 +1,83 @@
#!/usr/bin/env python3
import argparse, json, os, re, sys
METRICS = {
"Rank corr": True,
"Rank correlation": True,
"NN Tm correlation": True,
"Naive Tm correlation": True,
}
def parse(text):
results = {}
for name, higher in METRICS.items():
pat = re.compile(re.escape(name) + r"\s*:\s*([+-]?[0-9]*\.?[0-9]+)", re.IGNORECASE)
matches = pat.finditer(text)
vals = []
for m in matches:
vals.append((float(m.group(1)), higher, m.group(0).strip()))
if vals:
results[name] = vals
return results
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--stdout", required=True)
ap.add_argument("--elapsed", type=float, default=None)
args = ap.parse_args()
text = open(args.stdout, encoding="utf-8", errors="replace").read()
parsed = parse(text)
score = None
feedback_parts = []
if "NN Tm correlation" in parsed:
val, _, raw = parsed["NN Tm correlation"][0]
score = abs(val)
feedback_parts.append(f"NN Tm corr={val}")
elif "Rank correlation" in parsed:
vals = [abs(v) for v, _, _ in parsed["Rank correlation"]]
score = sum(vals) / len(vals)
feedback_parts.append(f"avg |Rank corr|={score:.4f}")
elif "Rank corr" in parsed:
vals = [abs(v) for v, _, _ in parsed["Rank corr"]]
score = sum(vals) / len(vals)
feedback_parts.append(f"avg |Rank corr|={score:.4f}")
elif "Naive Tm correlation" in parsed:
val, _, raw = parsed["Naive Tm correlation"][0]
score = abs(val)
feedback_parts.append(f"Naive Tm corr={val}")
if score is None and args.elapsed is not None:
score = -(args.elapsed * 1000)
feedback_parts.append(f"time={args.elapsed*1000:.1f}ms")
elif score is None:
m = re.search(r"SUMMARY:\s*(\d+)/(\d+)\s*passed", text)
if m:
score = float(m.group(1))
feedback_parts.append(f"passes={m.group(1)}/{m.group(2)}")
else:
sys.stderr.write("[score] no metric found in output.\n")
sys.exit(3)
result = {
"score": score,
"examples": [{
"id": "metric",
"score": score,
"pass": True,
"feedback": " | ".join(feedback_parts),
}],
"feedback": " | ".join(feedback_parts),
}
out = os.environ.get("ATLAS_OPTIMIZE_RESULT")
payload = json.dumps(result)
if out:
with open(out, "w", encoding="utf-8") as fh:
fh.write(payload)
else:
sys.stdout.write(payload)
if __name__ == "__main__":
main()

9
atlas.json Normal file
View file

@ -0,0 +1,9 @@
{
"project": "silversight",
"description": "SilverSight polyglot formalization — GEPA-optimize Python, port to 11 languages",
"auto": {
"capture": "suggest",
"prompt": "Rewrite this Python code to be more performant and correct. Keep the same public interface (same function signatures, same class names, same CLI behavior). Optimize for algorithmic quality (rank correlation, Tm correlation) and speed.",
"reflection_lm": "openai/auto"
}
}

View file

@ -378,7 +378,7 @@ def main():
# Generate receipt
receipt = generate_receipt(all_search_results, schema_version="stage3_v1")
receipt_path = "/mnt/agents/output/rebuild/stage3-search/chaos_game_receipt.json"
receipt_path = "/tmp/chaos_game_receipt.json"
with open(receipt_path, "w") as f:
json.dump(receipt, f, indent=2)
print(f"\nReceipt: {receipt_path}")