mirror of
https://github.com/allaunthefox/SilverSight.git
synced 2026-07-30 17:16:16 +00:00
feat(polyglot): add GEPA infrastructure for Python shim optimization
- Add .atlas/ with benchmark.sh, score.py, gate.sh for GEPA campaigns - Add atlas.json project config (auto.capture=suggest, reflection_lm=openai/auto) - Fix test_search.py receipt path from /mnt/agents/ to /tmp/ Metric targets: dna_qubo_sort.py: avg |Rank corr| (higher=better, baseline=0.8547) dna_qubo_nn.py: NN Tm correlation (higher=better, baseline=0.4649) dna_lut.py: avg |Rank corr| (higher=better, baseline=0.5808) test_search.py: pass count (ceiling=43, for stability verification)
This commit is contained in:
parent
8b79a75ea4
commit
7fcfde3ca7
5 changed files with 144 additions and 1 deletions
20
.atlas/benchmark.sh
Executable file
20
.atlas/benchmark.sh
Executable file
|
|
@ -0,0 +1,20 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
OUT="$(mktemp)"
|
||||
|
||||
TARGET="${ATLAS_OPTIMIZE_TARGET:-$1}"
|
||||
export ATLAS_EVAL_SEED="${ATLAS_EVAL_SEED:-0}"
|
||||
|
||||
START=$(date +%s.%N)
|
||||
( python3 "$TARGET" ) >"$OUT" 2>&1
|
||||
RC=$?
|
||||
END=$(date +%s.%N)
|
||||
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo "[benchmark] run command exited $RC — candidate failed" >&2
|
||||
tail -n 40 "$OUT" >&2
|
||||
exit "$RC"
|
||||
fi
|
||||
|
||||
python3 "$HERE/score.py" --stdout "$OUT" --elapsed "$(awk "BEGIN { print $END - $START }")"
|
||||
31
.atlas/gate.sh
Executable file
31
.atlas/gate.sh
Executable file
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/env bash
|
||||
# Goodhart guard: reject candidates that lower quality while improving timing
|
||||
set -uo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TARGET="${ATLAS_OPTIMIZE_TARGET:-$2}"
|
||||
REF_SCORE="${ATLAS_REF_SCORE:-$1}"
|
||||
|
||||
NEW_OUT="$(mktemp)"
|
||||
NEW_START=$(date +%s.%N)
|
||||
( python3 "$TARGET" ) >"$NEW_OUT" 2>&1
|
||||
NEW_RC=$?
|
||||
NEW_END=$(date +%s.%N)
|
||||
NEW_ELAPSED=$(awk "BEGIN { print $NEW_END - $NEW_START }")
|
||||
|
||||
if [ "$NEW_RC" -ne 0 ]; then
|
||||
echo "[gate] candidate failed — rejecting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NEW_SCORE=$(python3 "$HERE/score.py" --stdout "$NEW_OUT" --elapsed "$NEW_ELAPSED" 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['score'])")
|
||||
|
||||
# Accept if score >= reference (higher is better)
|
||||
ACCEPT=$(python3 -c "print('yes' if $NEW_SCORE >= $REF_SCORE - 0.01 else 'no')")
|
||||
if [ "$ACCEPT" = "yes" ]; then
|
||||
echo "[gate] accepted (score=$NEW_SCORE >= ref=$REF_SCORE)"
|
||||
exit 0
|
||||
else
|
||||
echo "[gate] rejected (score=$NEW_SCORE < ref=$REF_SCORE)"
|
||||
exit 1
|
||||
fi
|
||||
83
.atlas/score.py
Normal file
83
.atlas/score.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
#!/usr/bin/env python3
|
||||
import argparse, json, os, re, sys
|
||||
|
||||
METRICS = {
|
||||
"Rank corr": True,
|
||||
"Rank correlation": True,
|
||||
"NN Tm correlation": True,
|
||||
"Naive Tm correlation": True,
|
||||
}
|
||||
|
||||
def parse(text):
|
||||
results = {}
|
||||
for name, higher in METRICS.items():
|
||||
pat = re.compile(re.escape(name) + r"\s*:\s*([+-]?[0-9]*\.?[0-9]+)", re.IGNORECASE)
|
||||
matches = pat.finditer(text)
|
||||
vals = []
|
||||
for m in matches:
|
||||
vals.append((float(m.group(1)), higher, m.group(0).strip()))
|
||||
if vals:
|
||||
results[name] = vals
|
||||
return results
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--stdout", required=True)
|
||||
ap.add_argument("--elapsed", type=float, default=None)
|
||||
args = ap.parse_args()
|
||||
text = open(args.stdout, encoding="utf-8", errors="replace").read()
|
||||
parsed = parse(text)
|
||||
|
||||
score = None
|
||||
feedback_parts = []
|
||||
|
||||
if "NN Tm correlation" in parsed:
|
||||
val, _, raw = parsed["NN Tm correlation"][0]
|
||||
score = abs(val)
|
||||
feedback_parts.append(f"NN Tm corr={val}")
|
||||
elif "Rank correlation" in parsed:
|
||||
vals = [abs(v) for v, _, _ in parsed["Rank correlation"]]
|
||||
score = sum(vals) / len(vals)
|
||||
feedback_parts.append(f"avg |Rank corr|={score:.4f}")
|
||||
elif "Rank corr" in parsed:
|
||||
vals = [abs(v) for v, _, _ in parsed["Rank corr"]]
|
||||
score = sum(vals) / len(vals)
|
||||
feedback_parts.append(f"avg |Rank corr|={score:.4f}")
|
||||
elif "Naive Tm correlation" in parsed:
|
||||
val, _, raw = parsed["Naive Tm correlation"][0]
|
||||
score = abs(val)
|
||||
feedback_parts.append(f"Naive Tm corr={val}")
|
||||
|
||||
if score is None and args.elapsed is not None:
|
||||
score = -(args.elapsed * 1000)
|
||||
feedback_parts.append(f"time={args.elapsed*1000:.1f}ms")
|
||||
elif score is None:
|
||||
m = re.search(r"SUMMARY:\s*(\d+)/(\d+)\s*passed", text)
|
||||
if m:
|
||||
score = float(m.group(1))
|
||||
feedback_parts.append(f"passes={m.group(1)}/{m.group(2)}")
|
||||
else:
|
||||
sys.stderr.write("[score] no metric found in output.\n")
|
||||
sys.exit(3)
|
||||
|
||||
result = {
|
||||
"score": score,
|
||||
"examples": [{
|
||||
"id": "metric",
|
||||
"score": score,
|
||||
"pass": True,
|
||||
"feedback": " | ".join(feedback_parts),
|
||||
}],
|
||||
"feedback": " | ".join(feedback_parts),
|
||||
}
|
||||
|
||||
out = os.environ.get("ATLAS_OPTIMIZE_RESULT")
|
||||
payload = json.dumps(result)
|
||||
if out:
|
||||
with open(out, "w", encoding="utf-8") as fh:
|
||||
fh.write(payload)
|
||||
else:
|
||||
sys.stdout.write(payload)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
9
atlas.json
Normal file
9
atlas.json
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"project": "silversight",
|
||||
"description": "SilverSight polyglot formalization — GEPA-optimize Python, port to 11 languages",
|
||||
"auto": {
|
||||
"capture": "suggest",
|
||||
"prompt": "Rewrite this Python code to be more performant and correct. Keep the same public interface (same function signatures, same class names, same CLI behavior). Optimize for algorithmic quality (rank correlation, Tm correlation) and speed.",
|
||||
"reflection_lm": "openai/auto"
|
||||
}
|
||||
}
|
||||
|
|
@ -378,7 +378,7 @@ def main():
|
|||
|
||||
# Generate receipt
|
||||
receipt = generate_receipt(all_search_results, schema_version="stage3_v1")
|
||||
receipt_path = "/mnt/agents/output/rebuild/stage3-search/chaos_game_receipt.json"
|
||||
receipt_path = "/tmp/chaos_game_receipt.json"
|
||||
with open(receipt_path, "w") as f:
|
||||
json.dump(receipt, f, indent=2)
|
||||
print(f"\nReceipt: {receipt_path}")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue