mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
175 lines
6.6 KiB
Text
175 lines
6.6 KiB
Text
/- Copyright (c) 2026 Sovereign Research Stack. All rights reserved.
|
||
Released under Apache 2.0 license as described in the file LICENSE.
|
||
Authors: Research Stack Team
|
||
|
||
SigmaGateBenchmark.lean — Creation OS Benchmark Verification
|
||
|
||
Auto-generated from Creation OS benchmarks/suite/full_results.json
|
||
via scripts/creation_os_shim.py (JSON → Lean #eval shim).
|
||
|
||
Per AGENTS.md §1.4: Float conversion at I/O boundary only.
|
||
-/
|
||
import Semantics.SigmaGate
|
||
|
||
namespace Semantics.SigmaGate.Benchmark
|
||
|
||
open Semantics.SigmaGate
|
||
open Semantics.SigmaGate.Conformal
|
||
|
||
-- ════════════════════════════════════════════════════════════
|
||
-- §1 Creation OS Benchmark Data (Auto-Imported)
|
||
-- ════════════════════════════════════════════════════════════
|
||
|
||
/-- Benchmark rows from Creation OS multi-dataset σ-gate suite (SCI-6).
|
||
Schema: cos.suite_sci.v1
|
||
Source: https://github.com/spektre-labs/creation-os
|
||
BitNet-b1.58-2B, cos chat, pipeline mode filter.
|
||
-/
|
||
/-- Dataset: truthfulqa, N=817, accuracy=0.3357 -/
|
||
def dataset_truthfulqa : ShimScoredItem := ⟨
|
||
0.391316, -- sigma_mean
|
||
0.335714, -- accuracy
|
||
"truthfulqa"
|
||
⟩
|
||
|
||
/-- Threshold for truthfulqa: α=0.8, τ=0.6552 -/
|
||
def threshold_truthfulqa : ConformalThreshold := {
|
||
tau := ⟨0x53DB⟩,
|
||
alpha := ⟨0x6665⟩,
|
||
delta := ⟨0x0001⟩,
|
||
calibratedOn := 817
|
||
}
|
||
|
||
/-- Dataset: arc_challenge, N=1172, accuracy=0.3371 -/
|
||
def dataset_arc_challenge : ShimScoredItem := ⟨
|
||
0.507841, -- sigma_mean
|
||
0.337148, -- accuracy
|
||
"arc_challenge"
|
||
⟩
|
||
|
||
/-- Threshold for arc_challenge: α=0.8, τ=0.6500 -/
|
||
def threshold_arc_challenge : ConformalThreshold := {
|
||
tau := ⟨0x5332⟩,
|
||
alpha := ⟨0x6665⟩,
|
||
delta := ⟨0x0001⟩,
|
||
calibratedOn := 1172
|
||
}
|
||
|
||
/-- Dataset: arc_easy, N=2376, accuracy=0.4197 -/
|
||
def dataset_arc_easy : ShimScoredItem := ⟨
|
||
0.477290, -- sigma_mean
|
||
0.419742, -- accuracy
|
||
"arc_easy"
|
||
⟩
|
||
|
||
/-- Threshold for arc_easy: α=0.8, τ=0.6500 -/
|
||
def threshold_arc_easy : ConformalThreshold := {
|
||
tau := ⟨0x5332⟩,
|
||
alpha := ⟨0x6665⟩,
|
||
delta := ⟨0x0001⟩,
|
||
calibratedOn := 2376
|
||
}
|
||
|
||
/-- Dataset: gsm8k, N=1319, accuracy=0.1250 -/
|
||
def dataset_gsm8k : ShimScoredItem := ⟨
|
||
0.481380, -- sigma_mean
|
||
0.125000, -- accuracy
|
||
"gsm8k"
|
||
⟩
|
||
|
||
/-- Threshold for gsm8k: α=0.8, τ=0.3300 -/
|
||
def threshold_gsm8k : ConformalThreshold := {
|
||
tau := ⟨0x2A3D⟩,
|
||
alpha := ⟨0x6665⟩,
|
||
delta := ⟨0x0001⟩,
|
||
calibratedOn := 1319
|
||
}
|
||
|
||
/-- Dataset: hellaswag, N=500, accuracy=0.2854 -/
|
||
def dataset_hellaswag : ShimScoredItem := ⟨
|
||
0.533180, -- sigma_mean
|
||
0.285417, -- accuracy
|
||
"hellaswag"
|
||
⟩
|
||
|
||
/-- Threshold for hellaswag: α=0.8, τ=0.6500 -/
|
||
def threshold_hellaswag : ConformalThreshold := {
|
||
tau := ⟨0x5332⟩,
|
||
alpha := ⟨0x6665⟩,
|
||
delta := ⟨0x0001⟩,
|
||
calibratedOn := 500
|
||
}
|
||
|
||
-- ════════════════════════════════════════════════════════════
|
||
-- §2 Verification #eval Witnesses
|
||
-- ════════════════════════════════════════════════════════════
|
||
|
||
/-- Verify all Creation OS datasets against conservative threshold.
|
||
Each dataset is checked for valid threshold structure.
|
||
-/
|
||
#eval! isValidConformalThreshold threshold_truthfulqa -- truthfulqa
|
||
#eval! isValidConformalThreshold threshold_arc_challenge -- arc_challenge
|
||
#eval! isValidConformalThreshold threshold_arc_easy -- arc_easy
|
||
#eval! isValidConformalThreshold threshold_gsm8k -- gsm8k
|
||
#eval! isValidConformalThreshold threshold_hellaswag -- hellaswag
|
||
|
||
/-- Verify empirical coverage: does threshold structure match dataset?
|
||
This is a structural check, not a full statistical test.
|
||
-/
|
||
#eval! verifyShimCoverage threshold_truthfulqa #[dataset_truthfulqa] -- truthfulqa
|
||
#eval! verifyShimCoverage threshold_arc_challenge #[dataset_arc_challenge] -- arc_challenge
|
||
#eval! verifyShimCoverage threshold_arc_easy #[dataset_arc_easy] -- arc_easy
|
||
#eval! verifyShimCoverage threshold_gsm8k #[dataset_gsm8k] -- gsm8k
|
||
#eval! verifyShimCoverage threshold_hellaswag #[dataset_hellaswag] -- hellaswag
|
||
|
||
/-- Assemble all datasets for combined threshold verification.
|
||
Tests if the conservative threshold generalizes across tasks.
|
||
-/
|
||
def allDatasets : Array ShimScoredItem := #[
|
||
dataset_truthfulqa,
|
||
dataset_arc_challenge,
|
||
dataset_arc_easy,
|
||
dataset_gsm8k,
|
||
dataset_hellaswag
|
||
]
|
||
|
||
/-- Combined verification across all tasks. -/
|
||
#eval! verifyShimCoverage ConformalThreshold.conservative allDatasets
|
||
|
||
-- ════════════════════════════════════════════════════════════
|
||
-- §3 Sigma Score Reconstruction from Entropy Measures
|
||
-- ════════════════════════════════════════════════════════════
|
||
|
||
/-- Convert Creation OS sigma_mean to Lean SigmaScore.
|
||
Float bridge at I/O boundary (AGENTS.md §1.4 compliant).
|
||
-/
|
||
def score_truthfulqa : SigmaScore := ⟨⟨0x3216⟩, "creation_os_truthfulqa", 1⟩
|
||
#eval! score_truthfulqa.value
|
||
|
||
def score_arc_challenge : SigmaScore := ⟨⟨0x4100⟩, "creation_os_arc_challenge", 1⟩
|
||
#eval! score_arc_challenge.value
|
||
|
||
def score_arc_easy : SigmaScore := ⟨⟨0x3D17⟩, "creation_os_arc_easy", 1⟩
|
||
#eval! score_arc_easy.value
|
||
|
||
def score_gsm8k : SigmaScore := ⟨⟨0x3D9D⟩, "creation_os_gsm8k", 1⟩
|
||
#eval! score_gsm8k.value
|
||
|
||
def score_hellaswag : SigmaScore := ⟨⟨0x443E⟩, "creation_os_hellaswag", 1⟩
|
||
#eval! score_hellaswag.value
|
||
|
||
-- ════════════════════════════════════════════════════════════
|
||
-- §4 Threshold Calibration Cross-Check
|
||
-- ════════════════════════════════════════════════════════════
|
||
|
||
/-- Conservative threshold (α=0.80, δ=0.10) cross-check.
|
||
Verifies structural validity against 5.5σ target.
|
||
-/
|
||
#eval! isValidConformalThreshold ConformalThreshold.conservative
|
||
|
||
/-- High-confidence threshold (targeting 6.5σ) cross-check.
|
||
Structural validity only — full 6.5σ requires 1M+ samples.
|
||
-/
|
||
#eval! isValidConformalThreshold ConformalThreshold.sixSigma
|
||
|
||
end Semantics.SigmaGate.Benchmark
|