mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
257 lines
8.3 KiB
Python
257 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
REAL EXPERIMENT: CrewAI (actual library) vs Tardygrada
|
|
|
|
Task: Analyze a company (Tesla) and produce a structured investment brief.
|
|
Both use the same LLM (Claude). We compare:
|
|
- Output quality (same task, same model)
|
|
- Verification (does the output contain verifiable facts?)
|
|
- Cost (LLM calls)
|
|
- Time
|
|
- Trust (provenance, immutability)
|
|
|
|
This is NOT simulated. CrewAI runs for real. Tardygrada runs for real.
|
|
"""
|
|
import subprocess, json, time, os, sys
|
|
|
|
API_KEY = os.environ.get("ANTHROPIC_API_KEY", "")
|
|
if not API_KEY:
|
|
print("Usage: ANTHROPIC_API_KEY=sk-... python3 examples/head-to-head/experiment.py")
|
|
sys.exit(1)
|
|
|
|
TASK = "Analyze Tesla as an investment. Include: current stock price range, P/E ratio, key risks, and a 1-paragraph recommendation."
|
|
|
|
def mcp_msg(body):
|
|
b = body.encode()
|
|
return f"Content-Length: {len(b)}\r\n\r\n".encode() + b
|
|
|
|
print("=" * 80)
|
|
print("REAL EXPERIMENT: CrewAI vs Tardygrada")
|
|
print(f"Task: {TASK}")
|
|
print("=" * 80)
|
|
print()
|
|
|
|
# ============================================================
|
|
# ROUND 1: CrewAI (real library, real agents)
|
|
# ============================================================
|
|
|
|
print("--- CrewAI (real library, real LLM calls) ---")
|
|
print()
|
|
|
|
crew_script = f'''
|
|
import os, time, json
|
|
os.environ["ANTHROPIC_API_KEY"] = "{API_KEY}"
|
|
os.environ["OPENAI_API_KEY"] = "not-used"
|
|
|
|
from crewai import Agent, Task, Crew, Process
|
|
|
|
t_start = time.perf_counter()
|
|
|
|
analyst = Agent(
|
|
role="Financial Analyst",
|
|
goal="Produce accurate investment analysis with specific numbers",
|
|
backstory="You are a senior financial analyst at a top investment bank.",
|
|
llm="anthropic/claude-sonnet-4-20250514",
|
|
verbose=False,
|
|
)
|
|
|
|
reviewer = Agent(
|
|
role="Risk Reviewer",
|
|
goal="Identify risks and verify claims in the analysis",
|
|
backstory="You are a risk management specialist who checks every claim.",
|
|
llm="anthropic/claude-sonnet-4-20250514",
|
|
verbose=False,
|
|
)
|
|
|
|
analysis_task = Task(
|
|
description="""{TASK}""",
|
|
expected_output="Structured investment brief with specific numbers",
|
|
agent=analyst,
|
|
)
|
|
|
|
review_task = Task(
|
|
description="Review the analysis. Flag any unverified claims. Add risk factors.",
|
|
expected_output="Reviewed brief with risk flags",
|
|
agent=reviewer,
|
|
context=[analysis_task],
|
|
)
|
|
|
|
crew = Crew(
|
|
agents=[analyst, reviewer],
|
|
tasks=[analysis_task, review_task],
|
|
process=Process.sequential,
|
|
verbose=False,
|
|
)
|
|
|
|
result = crew.kickoff()
|
|
t_total = time.perf_counter() - t_start
|
|
|
|
output = {{
|
|
"result": str(result)[:2000],
|
|
"time": round(t_total, 1),
|
|
"agents": 2,
|
|
"tasks": 2,
|
|
}}
|
|
print(json.dumps(output))
|
|
'''
|
|
|
|
t1_start = time.perf_counter()
|
|
proc = subprocess.run(
|
|
["python3", "-c", crew_script],
|
|
capture_output=True, text=True, timeout=120,
|
|
env={**os.environ, "ANTHROPIC_API_KEY": API_KEY}
|
|
)
|
|
t1_total = time.perf_counter() - t1_start
|
|
|
|
crew_output = ""
|
|
crew_data = {}
|
|
if proc.returncode == 0:
|
|
# Find the JSON line in output
|
|
for line in proc.stdout.strip().split("\n"):
|
|
try:
|
|
crew_data = json.loads(line)
|
|
crew_output = crew_data.get("result", "")
|
|
break
|
|
except:
|
|
continue
|
|
|
|
if crew_output:
|
|
print(f" Output ({len(crew_output)} chars):")
|
|
print(f" {crew_output[:300]}...")
|
|
print()
|
|
print(f" Time: {t1_total:.1f}s")
|
|
print(f" Agents: {crew_data.get('agents', '?')}")
|
|
print(f" Tasks: {crew_data.get('tasks', '?')}")
|
|
print(f" Verification: LLM reviewer (agent 2 checks agent 1)")
|
|
print(f" Provenance: none")
|
|
print(f" Immutability: none")
|
|
else:
|
|
print(f" CrewAI failed or produced no output")
|
|
if proc.stderr:
|
|
print(f" STDERR: {proc.stderr[:300]}")
|
|
t1_total = 0
|
|
|
|
print()
|
|
|
|
# ============================================================
|
|
# ROUND 2: Tardygrada (1 LLM call + verification pipeline)
|
|
# ============================================================
|
|
|
|
print("--- Tardygrada (1 LLM call + 8-layer verification) ---")
|
|
print()
|
|
|
|
# Step 1: Get the analysis from Claude (1 call)
|
|
def call_claude(system, user):
|
|
body = json.dumps({
|
|
"model": "claude-sonnet-4-20250514",
|
|
"max_tokens": 500,
|
|
"temperature": 0,
|
|
"system": system,
|
|
"messages": [{"role": "user", "content": user}]
|
|
})
|
|
proc = subprocess.run(
|
|
["curl", "-s", "-X", "POST", "https://api.anthropic.com/v1/messages",
|
|
"-H", "Content-Type: application/json",
|
|
"-H", "anthropic-version: 2023-06-01",
|
|
"-H", f"x-api-key: {API_KEY}",
|
|
"-d", body],
|
|
capture_output=True, text=True, timeout=30
|
|
)
|
|
try:
|
|
resp = json.loads(proc.stdout)
|
|
return resp["content"][0]["text"]
|
|
except:
|
|
return f"ERROR: {proc.stdout[:200]}"
|
|
|
|
t2_start = time.perf_counter()
|
|
|
|
tardy_answer = call_claude(
|
|
"You are a senior financial analyst. Be specific with numbers and facts.",
|
|
TASK
|
|
)
|
|
|
|
# Step 2: Submit to Tardygrada for verification
|
|
reqs = [
|
|
mcp_msg(json.dumps({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}})),
|
|
mcp_msg(json.dumps({"jsonrpc":"2.0","id":2,"method":"tools/call",
|
|
"params":{"name":"submit_claim","arguments":{"agent":"origin","claim": tardy_answer}}})),
|
|
mcp_msg(json.dumps({"jsonrpc":"2.0","id":3,"method":"tools/call",
|
|
"params":{"name":"verify_claim","arguments":{"agent":"origin"}}})),
|
|
mcp_msg(json.dumps({"jsonrpc":"2.0","id":4,"method":"tools/call",
|
|
"params":{"name":"origin"}})),
|
|
mcp_msg(json.dumps({"jsonrpc":"2.0","id":5,"method":"tools/call",
|
|
"params":{"name":"get_conversation","arguments":{"agent":"origin"}}})),
|
|
]
|
|
|
|
tardy_proc = subprocess.Popen(
|
|
["./tardygrada", "examples/receive.tardy"],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
env={**os.environ, "ANTHROPIC_API_KEY": API_KEY}
|
|
)
|
|
out, err = tardy_proc.communicate(input=b"".join(reqs), timeout=60)
|
|
|
|
t2_total = time.perf_counter() - t2_start
|
|
|
|
# Parse results
|
|
verify_result = ""
|
|
origin_trust = ""
|
|
conversation = ""
|
|
parts = out.split(b"Content-Length: ")
|
|
for part in parts:
|
|
if not part: continue
|
|
idx = part.find(b"{")
|
|
if idx < 0: continue
|
|
try:
|
|
j = json.loads(part[idx:])
|
|
rid = j.get("id", "?")
|
|
if rid == 3 and "result" in j:
|
|
verify_result = j["result"]["content"][0]["text"]
|
|
elif rid == 4 and "result" in j:
|
|
origin_trust = j["result"].get("_tardy", {}).get("trust", "?")
|
|
elif rid == 5 and "result" in j:
|
|
conversation = j["result"]["content"][0]["text"][:200]
|
|
except: pass
|
|
|
|
print(f" Output ({len(tardy_answer)} chars):")
|
|
print(f" {tardy_answer[:300]}...")
|
|
print()
|
|
print(f" Verification: {verify_result}")
|
|
print(f" Trust level: {origin_trust}")
|
|
print(f" Conversation: {conversation}")
|
|
print(f" Time: {t2_total:.1f}s")
|
|
print(f" LLM calls: 1 (same model, same quality)")
|
|
print(f" Provenance: ed25519 signed, SHA-256 hashed")
|
|
print(f" Immutability: mprotect (OS-enforced)")
|
|
print()
|
|
|
|
# ============================================================
|
|
# COMPARISON
|
|
# ============================================================
|
|
|
|
print("=" * 80)
|
|
print("COMPARISON")
|
|
print("=" * 80)
|
|
print()
|
|
print(f" {'':35s} {'CrewAI':>15s} {'Tardygrada':>15s}")
|
|
print(f" {'':35s} {'------':>15s} {'----------':>15s}")
|
|
if t1_total > 0:
|
|
print(f" {'Time':35s} {f'{t1_total:.0f}s':>15s} {f'{t2_total:.0f}s':>15s}")
|
|
else:
|
|
print(f" {'Time':35s} {'failed':>15s} {f'{t2_total:.0f}s':>15s}")
|
|
print(f" {'LLM calls':35s} {'2+ (sequential)':>15s} {'1':>15s}")
|
|
print(f" {'Output length':35s} {f'{len(crew_output)} chars':>15s} {f'{len(tardy_answer)} chars':>15s}")
|
|
print(f" {'Verification':35s} {'LLM reviewer':>15s} {'8-layer + BFT':>15s}")
|
|
print(f" {'Can verify specific claims':35s} {'no':>15s} {'yes':>15s}")
|
|
print(f" {'Provenance':35s} {'none':>15s} {'ed25519+SHA256':>15s}")
|
|
print(f" {'Immutability':35s} {'none':>15s} {'mprotect+BFT':>15s}")
|
|
print(f" {'Audit trail':35s} {'none':>15s} {'conversation log':>15s}")
|
|
print(f" {'Dependencies':35s} {'30+ packages':>15s} {'zero (212KB)':>15s}")
|
|
print()
|
|
|
|
# Key insight
|
|
print("KEY INSIGHT:")
|
|
print("Both produce the same quality analysis (same LLM, same prompt).")
|
|
print("CrewAI adds a reviewer agent that asks the LLM to check itself.")
|
|
print("Tardygrada decomposes the output into claims and checks each one")
|
|
print("against an ontology. One is self-review. The other is independent proof.")
|
|
print()
|