Research-Stack/shared-data/pist_route_repair_benchmark.json
2026-05-26 11:38:01 -05:00

656 lines
No EOL
22 KiB
JSON

{
"n_theorems": 9,
"recovered": 1,
"total_attempts": 19,
"results": [
{
"theorem_name": "simp_add_comm",
"initial_status": "failed",
"predicted_tactic_family": "normalization",
"confidence": 0.67,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "simp",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 11,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/simp_add_comm_repair_0-028f5c63f99e4290.lean:2:2: error: `simp` made no progress\n",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 7,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/simp_add_comm_repair_1-3d5420573f571a49.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/simp_add_comm_repair_1-3d5420573f571a49.lean:1:41: error",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 7,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/simp_add_comm_repair_1-3d5420573f571a49.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/simp_add_comm_repair_1-3d5420573f571a49.lean:1:41: error",
"stderr_preview": ""
},
"recovered": false,
"elapsed_s": 5.87
},
{
"theorem_name": "simp_add_assoc",
"initial_status": "failed",
"predicted_tactic_family": "normalization",
"confidence": 0.67,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "simp",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 12,
"variables_before": 3,
"variables_after": 3,
"stdout_preview": "/var/lib/language-proof-server/work/simp_add_assoc_repair_0-e042c815747f6968.lean:2:2: error: `simp` made no progress\n",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 8,
"variables_before": 3,
"variables_after": 3,
"stdout_preview": "/var/lib/language-proof-server/work/simp_add_assoc_repair_1-e4f2173c672f4be3.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/simp_add_assoc_repair_1-e4f2173c672f4be3.lean:1:55: err",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 8,
"variables_before": 3,
"variables_after": 3,
"stdout_preview": "/var/lib/language-proof-server/work/simp_add_assoc_repair_1-e4f2173c672f4be3.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/simp_add_assoc_repair_1-e4f2173c672f4be3.lean:1:55: err",
"stderr_preview": ""
},
"recovered": false,
"elapsed_s": 4.06
},
{
"theorem_name": "rw_mul_comm",
"initial_status": "failed",
"predicted_tactic_family": "normalization",
"confidence": 0.67,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "simp",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 9,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/rw_mul_comm_repair_0-cd5a6bf1f732e09b.lean:2:2: error: `simp` made no progress\n",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 5,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/rw_mul_comm_repair_1-97e55676c9783090.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/rw_mul_comm_repair_1-97e55676c9783090.lean:1:41: error: un",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 5,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/rw_mul_comm_repair_1-97e55676c9783090.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/rw_mul_comm_repair_1-97e55676c9783090.lean:1:41: error: un",
"stderr_preview": ""
},
"recovered": false,
"elapsed_s": 4.05
},
{
"theorem_name": "rw_add_comm",
"initial_status": "failed",
"predicted_tactic_family": "normalization",
"confidence": 0.67,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "simp",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 13,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/rw_add_comm_repair_0-028f5c63f99e4290.lean:2:2: error: `simp` made no progress\n",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 9,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/rw_add_comm_repair_1-3d5420573f571a49.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/rw_add_comm_repair_1-3d5420573f571a49.lean:1:41: error: un",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 9,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/rw_add_comm_repair_1-3d5420573f571a49.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/rw_add_comm_repair_1-3d5420573f571a49.lean:1:41: error: un",
"stderr_preview": ""
},
"recovered": false,
"elapsed_s": 3.78
},
{
"theorem_name": "induct_add_zero",
"initial_status": "failed",
"predicted_tactic_family": "case_analysis",
"confidence": 0.67,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "cases n",
"tactic_family": "case_analysis",
"status": "failed",
"goal_complexity_delta": 40,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/induct_add_zero_repair_0-a5c8f0c8af055333.lean:1:35: error: unsolved goals\ncase zero\n\u22a2 0 + 0 = 0\n\ncase succ\nn\u271d : Nat\n\u22a2 n\u271d + 1 + 0 = n\u271d + 1\n",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "constructor",
"tactic_family": "case_analysis",
"status": "verified",
"goal_complexity_delta": 36,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "constructor",
"tactic_family": "case_analysis",
"status": "verified",
"goal_complexity_delta": 36,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "",
"stderr_preview": ""
},
"recovered": true,
"elapsed_s": 3.7
},
{
"theorem_name": "omega_chain_unsat",
"initial_status": "failed",
"predicted_tactic_family": "normalization",
"confidence": 0.67,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "simp",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 1,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/omega_chain_unsat_repair_0-74119ff4a607a187.lean:1:31: error: unsolved goals\nx : Nat\n\u22a2 False\n",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": -3,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/omega_chain_unsat_repair_1-5947476429d3579b.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/omega_chain_unsat_repair_1-5947476429d3579b.lean:1:3",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": -3,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/omega_chain_unsat_repair_1-5947476429d3579b.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/omega_chain_unsat_repair_1-5947476429d3579b.lean:1:3",
"stderr_preview": ""
},
"recovered": false,
"elapsed_s": 3.76
},
{
"theorem_name": "fail_unsat",
"initial_status": "failed",
"predicted_tactic_family": "normalization",
"confidence": 0.67,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "simp",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": 1,
"variables_before": 2,
"variables_after": 2,
"stdout_preview": "/var/lib/language-proof-server/work/fail_unsat_repair_0-448f459908ccaed3.lean:2:2: error: `simp` made no progress\n",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": -3,
"variables_before": 2,
"variables_after": 2,
"stdout_preview": "/var/lib/language-proof-server/work/fail_unsat_repair_1-04befe8d56d52da7.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/fail_unsat_repair_1-04befe8d56d52da7.lean:1:43: error: unso",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": -3,
"variables_before": 2,
"variables_after": 2,
"stdout_preview": "/var/lib/language-proof-server/work/fail_unsat_repair_1-04befe8d56d52da7.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/fail_unsat_repair_1-04befe8d56d52da7.lean:1:43: error: unso",
"stderr_preview": ""
},
"recovered": false,
"elapsed_s": 3.71
},
{
"theorem_name": "fail_type_mismatch",
"initial_status": "failed",
"predicted_tactic_family": "normalization",
"confidence": 0.67,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "simp",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": -1,
"variables_before": 2,
"variables_after": 2,
"stdout_preview": "/var/lib/language-proof-server/work/fail_type_mismatch_repair_0-68c00734d7081b91.lean:1:26: error: Type mismatch\n \"hello\"\nhas type\n String\nbut is expected to have type\n Nat\n/var/lib/language-proof-",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": -5,
"variables_before": 2,
"variables_after": 2,
"stdout_preview": "/var/lib/language-proof-server/work/fail_type_mismatch_repair_1-c49504955a4f3b42.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/fail_type_mismatch_repair_1-c49504955a4f3b42.lean:1",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "norm_num",
"tactic_family": "normalization",
"status": "failed",
"goal_complexity_delta": -5,
"variables_before": 2,
"variables_after": 2,
"stdout_preview": "/var/lib/language-proof-server/work/fail_type_mismatch_repair_1-c49504955a4f3b42.lean:2:3: error: unknown tactic\n/var/lib/language-proof-server/work/fail_type_mismatch_repair_1-c49504955a4f3b42.lean:1",
"stderr_preview": ""
},
"recovered": false,
"elapsed_s": 3.63
},
{
"theorem_name": "logic_or",
"initial_status": "failed",
"predicted_tactic_family": "discharge",
"confidence": 1.0,
"nearest_motifs": [
{
"id": "f993dc56-735b-4ec7-9a1f-162b83e14c3e",
"frequency": 7,
"tactic_family": "arithmetic"
},
{
"id": "2ae046a6-11e7-4f23-83ec-71dfe6a3473b",
"frequency": 7,
"tactic_family": "unknown"
},
{
"id": "30bc1d20-5f7b-4afe-9890-8a79ab5d95d7",
"frequency": 7,
"tactic_family": "discharge"
},
{
"id": "2f2ceebc-a166-45ce-9e1e-5abf7bc7c1e3",
"frequency": 5,
"tactic_family": "rewrite"
},
{
"id": "0c407456-55b2-446a-82bb-bc5ba0fdd864",
"frequency": 5,
"tactic_family": "normalization"
}
],
"attempts": [
{
"attempt": 1,
"candidate_patch": "exact A",
"tactic_family": "discharge",
"status": "failed",
"goal_complexity_delta": 40,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/logic_or_repair_0-ec89c3fc2dc1c4fd.lean:2:2: error: Type mismatch\n A\nhas type\n Prop\nof sort `Type` but is expected to have type\n A \u2228 B \u2192 B \u2228 A\nof sort `Prop`\n",
"stderr_preview": ""
},
{
"attempt": 2,
"candidate_patch": "assumption",
"tactic_family": "discharge",
"status": "failed",
"goal_complexity_delta": 37,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/logic_or_repair_1-7edfc94c3cd2318d.lean:2:2: error: Tactic `assumption` failed\n\nA B : Prop\n\u22a2 A \u2228 B \u2192 B \u2228 A\n",
"stderr_preview": ""
},
{
"attempt": 3,
"candidate_patch": "apply A",
"tactic_family": "discharge",
"status": "failed",
"goal_complexity_delta": 40,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/logic_or_repair_2-443739a1d6289bb8.lean:2:2: error: Tactic `apply` failed: could not unify the type of `A`\n Prop\nwith the goal\n A \u2228 B \u2192 B \u2228 A\n\nA B : Prop\n\u22a2 A \u2228 B ",
"stderr_preview": ""
}
],
"best_attempt": {
"attempt": 2,
"candidate_patch": "assumption",
"tactic_family": "discharge",
"status": "failed",
"goal_complexity_delta": 37,
"variables_before": 1,
"variables_after": 1,
"stdout_preview": "/var/lib/language-proof-server/work/logic_or_repair_1-7edfc94c3cd2318d.lean:2:2: error: Tactic `assumption` failed\n\nA B : Prop\n\u22a2 A \u2228 B \u2192 B \u2228 A\n",
"stderr_preview": ""
},
"recovered": false,
"elapsed_s": 5.31
}
],
"summary": {
"recovery_rate": 0.1111,
"per_family": {
"normalization": 0.0,
"case_analysis": 1.0,
"discharge": 0.0
}
}
}