mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
refactor(agent): simplify break-glass to model selection guide
Hermes already handles model routing. Remove MCP server, keep skill as a cost-optimized model selection guide. Key insight: DeepSeek V4 Pro is 55x cheaper than Claude Opus and nearly as good at math/code. Default to DeepSeek, escalate to Opus only when stuck. Removed: break_glass_mcp.py (unnecessary — Hermes handles routing)
This commit is contained in:
parent
e250787e8b
commit
9c5e5030a5
3 changed files with 65 additions and 381 deletions
10
.mcp.json
10
.mcp.json
|
|
@ -108,12 +108,10 @@
|
|||
}
|
||||
},
|
||||
"break-glass": {
|
||||
"_comment": "LAST RESORT: Multi-model panel via OpenRouter Fusion. Only for problems that have resisted all other approaches. Costs 3-5x single model call.",
|
||||
"command": "python3",
|
||||
"args": ["4-Infrastructure/shim/break_glass_mcp.py"],
|
||||
"env": {
|
||||
"OPENROUTER_API_KEY": "${OPENROUTER_API_KEY}"
|
||||
}
|
||||
"_comment": "REMOVED: Hermes already handles model routing. Use break-glass skill for model selection guidance.",
|
||||
"command": "echo",
|
||||
"args": ["break-glass MCP removed — use skill instead"],
|
||||
"env": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,34 +4,70 @@
|
|||
|
||||
Use this skill ONLY when a problem has resisted ALL other approaches:
|
||||
|
||||
1. Local model (Hermes3 / default) — tried and failed
|
||||
2. Single frontier model (Claude Opus / GPT-5.5) — tried and failed
|
||||
1. Local model (current default) — tried and failed
|
||||
2. Single frontier model — tried and failed
|
||||
3. Existing skills and tools — tried and failed
|
||||
|
||||
**Trigger:** `break glass: <problem statement>`
|
||||
|
||||
## What It Does
|
||||
|
||||
Assembles a focused multi-model panel via OpenRouter Fusion:
|
||||
Provides model selection guidance based on problem type. Hermes handles the actual model routing.
|
||||
|
||||
1. **Diagnoses** the problem type (lean_proof, code_bug, architecture, math_novel, infra_debug)
|
||||
2. **Selects** an optimal model panel for that problem type
|
||||
3. **Assembles** codebase context (relevant files, build logs, AGENTS.md constraints)
|
||||
4. **Calls** OpenRouter Fusion — sends the problem to multiple models in parallel
|
||||
5. **Synthesizes** — a judge model produces consensus, contradictions, blind spots
|
||||
6. **Returns** structured diagnosis + solution with confidence score
|
||||
## Model Selection Guide
|
||||
|
||||
## Cost
|
||||
### Cost Hierarchy (per 1M tokens, input/output)
|
||||
|
||||
Each invocation costs 3-5× a single model call (~$0.30-$1.50 depending on problem type).
|
||||
| Model | Input | Output | Provider | Use When |
|
||||
|-------|-------|--------|----------|----------|
|
||||
| **DeepSeek V4 Flash** | $0.07 | $0.28 | DeepSeek | Quick iterations, sanity checks |
|
||||
| **DeepSeek V4 Pro** | $0.27 | $1.10 | DeepSeek | Math reasoning, code, primary workhorse |
|
||||
| **Gemini 3 Flash** | $0.075 | $0.30 | Google | Budget panel diversity |
|
||||
| **Gemini 3.1 Pro** | $1.25 | $5.00 | Google | Different thinking style |
|
||||
| **Kimi K2.6** | $0.50 | $2.00 | Moonshot | Code, different training |
|
||||
| **Cohere Command A+** | $2.50 | $10.00 | Cohere | Structured output, tool-use |
|
||||
| **GLM-5.2** | $1.00 | $4.00 | Zhipu | Architectural diversity (753B) |
|
||||
| **GPT-5.5** | $10.00 | $30.00 | OpenAI | Breadth, ops knowledge |
|
||||
| **Claude Opus 4.8** | $15.00 | $75.00 | Anthropic | Formal proof, synthesis |
|
||||
|
||||
| Problem Type | Panel | Budget |
|
||||
|-------------|-------|--------|
|
||||
| lean_proof | Opus + DeepSeek V4 + Gemini 3.1 Pro | $1.00 |
|
||||
| code_bug | GPT-5.5 + Opus + Kimi K2.6 | $0.75 |
|
||||
| architecture | Opus + GPT-5.5 + Gemini 3.1 Pro | $1.50 |
|
||||
| math_novel | DeepSeek V4 + Opus + Gemini 3.1 Pro | $1.00 |
|
||||
| infra_debug | GPT-5.5 + Opus + DeepSeek V4 | $0.75 |
|
||||
### Problem Type → Model Recommendation
|
||||
|
||||
| Problem Type | First Try | If Stuck | Last Resort |
|
||||
|-------------|-----------|----------|-------------|
|
||||
| **lean_proof** | DeepSeek V4 Pro | + Claude Opus | + GPT-5.5 + Cohere |
|
||||
| **code_bug** | DeepSeek V4 Pro | + GPT-5.5 | + Claude Opus |
|
||||
| **architecture** | Claude Opus | + DeepSeek V4 Pro | + GPT-5.5 + Cohere |
|
||||
| **math_novel** | DeepSeek V4 Pro | + Claude Opus | + Gemini 3.1 Pro |
|
||||
| **infra_debug** | GPT-5.5 | + DeepSeek V4 Pro | + Claude Opus |
|
||||
| **hard_wall** | DeepSeek V4 Pro | + Claude Opus + GPT-5.5 | All 8 models |
|
||||
|
||||
### The "All 8 Models" Panel
|
||||
|
||||
For truly stuck problems, switch between all 8 models in sequence:
|
||||
1. DeepSeek V4 Pro ($0.27/$1.10) — math reasoning
|
||||
2. Claude Opus ($15/$75) — formal proof, synthesis
|
||||
3. GPT-5.5 ($10/$30) — breadth, ops
|
||||
4. Cohere Command A+ ($2.50/$10) — structured output
|
||||
5. Gemini 3.1 Pro ($1.25/$5) — different perspective
|
||||
6. GLM-5.2 ($1/$4) — different architecture
|
||||
7. Kimi K2.6 ($0.50/$2) — code
|
||||
8. DeepSeek V4 Flash ($0.07/$0.28) — fast iteration
|
||||
|
||||
### Cost-Optimal Strategy
|
||||
|
||||
**Default:** DeepSeek V4 Pro — 55x cheaper than Opus, nearly as good at math/code.
|
||||
|
||||
**Escalation path:**
|
||||
1. Start with DeepSeek V4 Pro ($0.27/$1.10)
|
||||
2. If stuck, add Claude Opus for synthesis ($15/$75)
|
||||
3. If still stuck, add GPT-5.5 for breadth ($10/$30)
|
||||
4. If truly stuck, use all 8 models sequentially
|
||||
|
||||
**Budget per problem:**
|
||||
- Routine: $0.10-0.50 (DeepSeek only)
|
||||
- Stuck: $1.00-3.00 (DeepSeek + Opus)
|
||||
- Hard wall: $5.00-10.00 (all 8 models)
|
||||
- Break glass: $15.00+ (all 8 + multiple rounds)
|
||||
|
||||
## How to Use
|
||||
|
||||
|
|
@ -40,23 +76,15 @@ break glass: cleanMerge_preservesGap sorry — List.zip/filter/all terms too lar
|
|||
```
|
||||
|
||||
The agent will:
|
||||
1. Auto-classify as `lean_proof`
|
||||
2. Read the relevant Lean files
|
||||
3. Pull build logs and AGENTS.md constraints
|
||||
4. Call Fusion with the Opus + DeepSeek + Gemini panel
|
||||
5. Return structured diagnosis + code
|
||||
|
||||
## Safety
|
||||
|
||||
- **Rate limit:** Max 3 invocations per session
|
||||
- **Cost cap:** $5.00 per session
|
||||
- **Human gate:** If estimated cost > $2.00, requires confirmation
|
||||
- **Audit trail:** Every invocation logged to ContextStream
|
||||
1. Classify the problem type
|
||||
2. Recommend which model to switch to
|
||||
3. Provide context to pass to that model
|
||||
4. Track cost
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `4-Infrastructure/shim/break_glass_mcp.py` | MCP server |
|
||||
| `6-Documentation/docs/specs/break_glass_mcp_agent.md` | Full design spec |
|
||||
| `.mcp.json` | MCP server registration |
|
||||
| `.opencode/skills/break-glass/SKILL.md` | This file |
|
||||
| `6-Documentation/docs/specs/break_glass_problem_statement.md` | Problem statement |
|
||||
| `6-Documentation/docs/specs/break_glass_mcp_agent.md` | Design spec (historical) |
|
||||
|
|
|
|||
|
|
@ -1,342 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
break_glass_mcp.py — Last-resort multi-model panel via OpenRouter Fusion.
|
||||
|
||||
Wraps OpenRouter Fusion as an MCP tool. Only invoked when all other
|
||||
approaches have failed on a novel problem.
|
||||
|
||||
Usage:
|
||||
python3 break_glass_mcp.py # starts MCP server on stdio
|
||||
|
||||
Requires: OPENROUTER_API_KEY env var
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
import httpx
|
||||
except ImportError:
|
||||
httpx = None
|
||||
|
||||
OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY", "")
|
||||
OPENROUTER_BASE = "https://openrouter.ai/api/v1"
|
||||
|
||||
# Problem type → model panel configuration
|
||||
PANELS = {
|
||||
"lean_proof": {
|
||||
"analysis_models": [
|
||||
"anthropic/claude-opus-4.8", # Lean expertise, formal proof
|
||||
"deepseek/deepseek-v4-pro", # Best math reasoning
|
||||
"openai/gpt-5.5", # Widest training data
|
||||
"cohere/command-a-03-2026", # Structured output, tool-use optimized
|
||||
"google/gemini-3.1-pro-preview", # Different thinking style
|
||||
"zhipu/glm-5.2", # 753B, genuinely different architecture
|
||||
"moonshotai/kimi-k2.6", # Strong code, different training
|
||||
"deepseek/deepseek-v4-flash", # Fast, good for quick iterations
|
||||
],
|
||||
"judge": "anthropic/claude-opus-4.8",
|
||||
"budget": 2.50,
|
||||
},
|
||||
"code_bug": {
|
||||
"analysis_models": [
|
||||
"openai/gpt-5.5", # Breadth, ops knowledge
|
||||
"anthropic/claude-opus-4.8", # Depth, precision
|
||||
"deepseek/deepseek-v4-pro", # Math/code reasoning
|
||||
"moonshotai/kimi-k2.6", # Different code style
|
||||
"cohere/command-a-03-2026", # Structured output
|
||||
"deepseek/deepseek-v4-flash", # Fast iterations
|
||||
],
|
||||
"judge": "anthropic/claude-opus-4.8",
|
||||
"budget": 1.80,
|
||||
},
|
||||
"architecture": {
|
||||
"analysis_models": [
|
||||
"anthropic/claude-opus-4.8", # Deep reasoning
|
||||
"openai/gpt-5.5", # Breadth
|
||||
"deepseek/deepseek-v4-pro", # Math rigor
|
||||
"cohere/command-a-03-2026", # Structured output
|
||||
"google/gemini-3.1-pro-preview", # Different perspective
|
||||
"zhipu/glm-5.2", # Architectural diversity
|
||||
"moonshotai/kimi-k2.6", # Code-first thinking
|
||||
"deepseek/deepseek-v4-flash", # Quick iterations
|
||||
],
|
||||
"judge": "anthropic/claude-opus-4.8",
|
||||
"budget": 2.50,
|
||||
},
|
||||
"math_novel": {
|
||||
"analysis_models": [
|
||||
"deepseek/deepseek-v4-pro", # Best math reasoning
|
||||
"anthropic/claude-opus-4.8", # Formal rigor
|
||||
"google/gemini-3.1-pro-preview", # Breadth
|
||||
"cohere/command-a-03-2026", # Structured output
|
||||
"zhipu/glm-5.2", # Different math tradition
|
||||
"deepseek/deepseek-v4-flash", # Quick checks
|
||||
],
|
||||
"judge": "anthropic/claude-opus-4.8",
|
||||
"budget": 2.00,
|
||||
},
|
||||
"infra_debug": {
|
||||
"analysis_models": [
|
||||
"openai/gpt-5.5", # Ops knowledge
|
||||
"anthropic/claude-opus-4.8", # Precision
|
||||
"deepseek/deepseek-v4-pro", # Systems reasoning
|
||||
"cohere/command-a-03-2026", # Structured tool chains
|
||||
"deepseek/deepseek-v4-flash", # Quick checks
|
||||
],
|
||||
"judge": "anthropic/claude-opus-4.8",
|
||||
"budget": 1.50,
|
||||
},
|
||||
"hard_wall": {
|
||||
"analysis_models": [
|
||||
"anthropic/claude-opus-4.8",
|
||||
"deepseek/deepseek-v4-pro",
|
||||
"openai/gpt-5.5",
|
||||
"cohere/command-a-03-2026",
|
||||
"google/gemini-3.1-pro-preview",
|
||||
"zhipu/glm-5.2",
|
||||
"moonshotai/kimi-k2.6",
|
||||
"deepseek/deepseek-v4-flash",
|
||||
],
|
||||
"judge": "anthropic/claude-opus-4.8",
|
||||
"budget": 3.50,
|
||||
},
|
||||
}
|
||||
|
||||
SYSTEM_PROMPT = """You are a specialist agent assembled to attack a specific
|
||||
novel problem that has resisted all常规 approaches. You have access to:
|
||||
|
||||
1. The full codebase context (provided below)
|
||||
2. The project's operating constraints
|
||||
3. The specific problem statement
|
||||
4. Previous attempts and their failure modes
|
||||
|
||||
YOUR TASK:
|
||||
- Diagnose the root cause
|
||||
- Propose a concrete solution with exact file paths and code
|
||||
- If the problem is undecidable, explain why and propose what would be needed
|
||||
|
||||
CONSTRAINTS:
|
||||
- Lean is the source of truth for formal claims
|
||||
- No Float in compute paths (use Q0_16 or Q16_16)
|
||||
- No sorry in committed code unless with TODO(lean-port) + human sign-off
|
||||
- All new formal work goes to SilverSight (not Research Stack)
|
||||
|
||||
OUTPUT FORMAT (JSON):
|
||||
{
|
||||
"diagnosis": "...",
|
||||
"solution": {
|
||||
"approach": "...",
|
||||
"confidence": 0.0-1.0,
|
||||
"files_to_change": ["..."],
|
||||
"code_snippets": {"file": "code"}
|
||||
},
|
||||
"alternatives": ["..."],
|
||||
"risks": ["..."],
|
||||
"cost_estimate": "$X.XX"
|
||||
}"""
|
||||
|
||||
|
||||
def classify_problem(problem: str) -> str:
|
||||
"""Auto-classify problem type from text."""
|
||||
lower = problem.lower()
|
||||
if any(w in lower for w in ["sorry", "lean", "theorem", "proof", "lake build"]):
|
||||
return "lean_proof"
|
||||
if any(w in lower for w in ["architecture", "design", "refactor", "structure"]):
|
||||
return "architecture"
|
||||
if any(w in lower for w in ["math", "formula", "eigenvalue", "convergence"]):
|
||||
return "math_novel"
|
||||
if any(w in lower for w in ["infra", "deploy", "k3s", "docker", "ssh"]):
|
||||
return "infra_debug"
|
||||
if any(w in lower for w in ["stuck", "wall", "impossible", "no idea", "tried everything"]):
|
||||
return "hard_wall"
|
||||
return "code_bug"
|
||||
|
||||
|
||||
def call_fusion(problem: str, context: str, problem_type: str,
|
||||
budget: float = 1.00) -> dict:
|
||||
"""Call OpenRouter Fusion with the assembled context."""
|
||||
if not OPENROUTER_API_KEY:
|
||||
return {"error": "OPENROUTER_API_KEY not set"}
|
||||
if httpx is None:
|
||||
return {"error": "httpx not installed (pip install httpx)"}
|
||||
|
||||
panel = PANELS.get(problem_type, PANELS["code_bug"])
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": f"""## Problem Statement
|
||||
{problem}
|
||||
|
||||
## Codebase Context
|
||||
{context}
|
||||
|
||||
## Problem Type
|
||||
{problem_type}
|
||||
|
||||
Please diagnose and propose a solution."""},
|
||||
]
|
||||
|
||||
payload = {
|
||||
"model": "openrouter/fusion",
|
||||
"messages": messages,
|
||||
"plugins": [{
|
||||
"id": "fusion",
|
||||
"model": panel["judge"],
|
||||
"analysis_models": panel["analysis_models"],
|
||||
}],
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {OPENROUTER_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
"HTTP-Referer": "https://researchstack.info",
|
||||
"X-Title": "break-glass-agent",
|
||||
}
|
||||
|
||||
try:
|
||||
with httpx.Client(timeout=120) as client:
|
||||
resp = client.post(
|
||||
f"{OPENROUTER_BASE}/chat/completions",
|
||||
json=payload,
|
||||
headers=headers,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# Extract cost
|
||||
usage = data.get("usage", {})
|
||||
cost = usage.get("total_cost", 0)
|
||||
|
||||
# Extract response
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
|
||||
return {
|
||||
"response": content,
|
||||
"cost": cost,
|
||||
"model": data.get("model", "openrouter/fusion"),
|
||||
"usage": usage,
|
||||
}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
def handle_tool_call(tool_name: str, arguments: dict) -> dict:
|
||||
"""Handle an MCP tool call."""
|
||||
if tool_name != "break_glass":
|
||||
return {"error": f"Unknown tool: {tool_name}"}
|
||||
|
||||
problem = arguments.get("problem", "")
|
||||
problem_type = arguments.get("problem_type") or classify_problem(problem)
|
||||
budget = arguments.get("budget", 1.00)
|
||||
context_files = arguments.get("context_files", [])
|
||||
context = arguments.get("context", "")
|
||||
|
||||
# Cost gate
|
||||
panel = PANELS.get(problem_type, PANELS["code_bug"])
|
||||
if budget > panel["budget"] * 3:
|
||||
return {
|
||||
"error": f"Budget ${budget:.2f} exceeds threshold ${panel['budget'] * 3:.2f}. "
|
||||
f"Escalate to human.",
|
||||
"problem_type": problem_type,
|
||||
}
|
||||
|
||||
result = call_fusion(problem, context, problem_type, budget)
|
||||
|
||||
return {
|
||||
"problem_type": problem_type,
|
||||
"panel": [panel["judge"]] + panel["analysis_models"],
|
||||
"result": result,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def mcp_server():
|
||||
"""Simple MCP server on stdio (JSON-RPC)."""
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
req = json.loads(line.strip())
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
method = req.get("method", "")
|
||||
req_id = req.get("id")
|
||||
params = req.get("params", {})
|
||||
|
||||
if method == "initialize":
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": req_id,
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {"tools": {}},
|
||||
"serverInfo": {
|
||||
"name": "break-glass",
|
||||
"version": "0.1.0",
|
||||
},
|
||||
},
|
||||
}
|
||||
elif method == "tools/list":
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": req_id,
|
||||
"result": {
|
||||
"tools": [{
|
||||
"name": "break_glass",
|
||||
"description": (
|
||||
"Last-resort multi-model panel for novel problems. "
|
||||
"Costs 3-5x a single model call. "
|
||||
"Requires explicit 'break glass:' trigger."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"problem": {
|
||||
"type": "string",
|
||||
"description": "The specific problem statement",
|
||||
},
|
||||
"context": {
|
||||
"type": "string",
|
||||
"description": "Relevant codebase context",
|
||||
},
|
||||
"problem_type": {
|
||||
"type": "string",
|
||||
"enum": list(PANELS.keys()),
|
||||
"description": "Problem classification (auto-detected if omitted)",
|
||||
},
|
||||
"budget": {
|
||||
"type": "number",
|
||||
"description": "Maximum cost in USD (default: 1.00)",
|
||||
},
|
||||
},
|
||||
"required": ["problem"],
|
||||
},
|
||||
}],
|
||||
},
|
||||
}
|
||||
elif method == "tools/call":
|
||||
tool_name = params.get("name", "")
|
||||
arguments = params.get("arguments", {})
|
||||
result = handle_tool_call(tool_name, arguments)
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": req_id,
|
||||
"result": {
|
||||
"content": [{"type": "text", "text": json.dumps(result, indent=2)}],
|
||||
},
|
||||
}
|
||||
else:
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": req_id,
|
||||
"error": {"code": -32601, "message": f"Method not found: {method}"},
|
||||
}
|
||||
|
||||
print(json.dumps(resp), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mcp_server()
|
||||
Loading…
Add table
Reference in a new issue