From d14d6b4b2576b790cbebb4e0e654b537c461aec8 Mon Sep 17 00:00:00 2001 From: Brandon Schneider Date: Tue, 12 May 2026 05:57:04 -0500 Subject: [PATCH] Refactor provenance sources for open witness backends --- .../lean/Semantics/Semantics.lean | 1 + .../lean/Semantics/Semantics/Forgejo.lean | 28 ++- .../Semantics/Semantics/Forgejo_metadata.json | 60 +++++- .../lean/Semantics/Semantics/Github.lean | 27 ++- .../Semantics/Semantics/Github_metadata.json | 60 +++++- .../Semantics/Semantics/ProvenanceSource.lean | 148 ++++++++++++++ .../Semantics/ProvenanceSource_metadata.json | 109 ++++++++++ .../lean/external/OTOM/Forgejo.lean | 28 ++- .../lean/external/OTOM/Github.lean | 27 ++- .../lean/external/OTOM/ProvenanceSource.lean | 113 +++++++++++ 4-Infrastructure/witness/README.md | 38 ++++ 4-Infrastructure/witness/sources.json | 125 ++++++++++++ .../scripts/attest_signal_wave_unification.py | 87 ++++++-- .../scripts/refill_gdrive_from_forgejo.sh | 24 ++- 5-Applications/scripts/self_clean_shim.py | 2 +- 5-Applications/scripts/server.js | 2 +- 5-Applications/tools-scripts/i2p_warden.py | 4 +- .../tools-scripts/ingestion/ingest_archive.py | 2 +- .../ingestion/ingest_large_file.py | 2 +- .../tools-scripts/io_harness_compat.py | 54 +++++ .../tools-scripts/metafoam/metafoam_pkg.py | 5 +- .../substrate/substrate_git_index.py | 189 +++++++++++++----- ARCHITECTURE.md | 6 +- GETTING_STARTED.md | 2 +- 24 files changed, 1022 insertions(+), 121 deletions(-) create mode 100644 0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource.lean create mode 100644 0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource_metadata.json create mode 100644 0-Core-Formalism/lean/external/OTOM/ProvenanceSource.lean create mode 100644 4-Infrastructure/witness/README.md create mode 100644 4-Infrastructure/witness/sources.json create mode 100644 5-Applications/tools-scripts/io_harness_compat.py diff --git a/0-Core-Formalism/lean/Semantics/Semantics.lean b/0-Core-Formalism/lean/Semantics/Semantics.lean index 3df553e4..c8f35f4b 100644 --- a/0-Core-Formalism/lean/Semantics/Semantics.lean +++ b/0-Core-Formalism/lean/Semantics/Semantics.lean @@ -1,4 +1,5 @@ import Semantics.Bind +import Semantics.ProvenanceSource import Semantics.Forgejo import Semantics.Github import Semantics.Hutter diff --git a/0-Core-Formalism/lean/Semantics/Semantics/Forgejo.lean b/0-Core-Formalism/lean/Semantics/Semantics/Forgejo.lean index ae8ee0ff..041e3c25 100644 --- a/0-Core-Formalism/lean/Semantics/Semantics/Forgejo.lean +++ b/0-Core-Formalism/lean/Semantics/Semantics/Forgejo.lean @@ -1,4 +1,4 @@ -import Semantics.Bind +import Semantics.ProvenanceSource namespace Semantics.Forgejo @@ -11,13 +11,32 @@ structure ForgejoEvent where author : String isValid : Bool +def forgejoPolicy : Semantics.ProvenanceSource.CostPolicy := { + trustedActors := [] + trustedCost := Semantics.Q16_16.ofFloat 0.5 + untrustedCost := Semantics.Q16_16.ofFloat 2.0 + publicCost := Semantics.Q16_16.ofFloat 2.0 +} + +def toSourceEvent (e : ForgejoEvent) : Semantics.ProvenanceSource.SourceEvent := { + backend := .vcs "git" "forgejo" + source := e.repo + action := e.action + actor := { scheme := "user", name := e.author } + ref := { scheme := "git-ref", value := e.action } + visibility := .internalRecord + isValid := e.isValid +} + /-- Invariant: Forgejo events are lawful if they originate from an allowed repo and the action is within the prescribed set. -/ def forgejoInvariant (e : ForgejoEvent) : String := - if e.isValid then s!"lawful_forgejo:{e.repo}:{e.action}" - else "unlawful_forgejo" + Semantics.ProvenanceSource.legacyInvariant + "lawful_forgejo" + "unlawful_forgejo" + (toSourceEvent e) open Semantics.Q16_16 @@ -26,8 +45,7 @@ Cost function: Measures the "computational friction" of a git event. Events from external authors have higher cost (Q16.16). -/ def forgejoCost (e1 : ForgejoEvent) (_target : String) (_g : Metric) : Semantics.Q16_16 := - if e1.author == "sovereign" then ofFloat 0.5 - else ofFloat 2.0 + Semantics.ProvenanceSource.cost forgejoPolicy (toSourceEvent e1) _target _g /-- The Forgejo Bind: Connects an event to the research substrate. diff --git a/0-Core-Formalism/lean/Semantics/Semantics/Forgejo_metadata.json b/0-Core-Formalism/lean/Semantics/Semantics/Forgejo_metadata.json index abc2062b..7d3d48b3 100644 --- a/0-Core-Formalism/lean/Semantics/Semantics/Forgejo_metadata.json +++ b/0-Core-Formalism/lean/Semantics/Semantics/Forgejo_metadata.json @@ -1,3 +1,57 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fb2e56150d3409a4a73954864e4daae4a5529b1c0f688249228972646d312f6b -size 1479 +{ + "file_path": "/home/allaun/Research Stack/tools/lean/Semantics/Semantics/Forgejo.lean", + "namespace": "Semantics.Forgejo", + "imports": [ + "Semantics.ProvenanceSource" + ], + "opens": [ + "Semantics.Q16_16" + ], + "structures": [ + { + "name": "ForgejoEvent", + "line": 8, + "doc": "ForgejoEvent: The structure of a Git event in the research stack." + } + ], + "theorems": [], + "definitions": [ + { + "name": "forgejoPolicy", + "line": 14, + "doc": "Provider-specific cost policy for the Forgejo compatibility shim." + }, + { + "name": "toSourceEvent", + "line": 21, + "doc": "Convert a legacy Forgejo event into the open provenance-source event surface." + }, + { + "name": "forgejoInvariant", + "line": 35, + "doc": "Invariant: Forgejo events are lawful if they originate from an allowed repo and the action is within the prescribed set." + }, + { + "name": "forgejoCost", + "line": 47, + "doc": "Cost function: Measures the \"computational friction\" of a git event. Events from external authors have higher cost (Q16.16)." + }, + { + "name": "forgejoBind", + "line": 53, + "doc": "The Forgejo Bind: Connects an event to the research substrate." + } + ], + "documentation": [ + "ForgejoEvent: The structure of a Git event in the research stack.", + "Provider-specific cost policy for the Forgejo compatibility shim.", + "Convert a legacy Forgejo event into the open provenance-source event surface.", + "Invariant: Forgejo events are lawful if they originate from an allowed repo and the action is within the prescribed set.", + "Cost function: Measures the \"computational friction\" of a git event. Events from external authors have higher cost (Q16.16).", + "The Forgejo Bind: Connects an event to the research substrate." + ], + "total_lines": 56, + "gcl_encoded": true, + "gcl_sequence": "F00020200", + "gcl_length": 9 +} diff --git a/0-Core-Formalism/lean/Semantics/Semantics/Github.lean b/0-Core-Formalism/lean/Semantics/Semantics/Github.lean index 6a8bb3be..176c3bce 100644 --- a/0-Core-Formalism/lean/Semantics/Semantics/Github.lean +++ b/0-Core-Formalism/lean/Semantics/Semantics/Github.lean @@ -1,4 +1,4 @@ -import Semantics.Bind +import Semantics.ProvenanceSource namespace Semantics.Github @@ -11,13 +11,32 @@ structure GithubEvent where isPublic : Bool isValid : Bool +def githubPolicy : Semantics.ProvenanceSource.CostPolicy := { + trustedActors := [] + trustedCost := Semantics.Q16_16.ofNat 5 + untrustedCost := Semantics.Q16_16.ofNat 5 + publicCost := Semantics.Q16_16.ofNat 5 +} + +def toSourceEvent (e : GithubEvent) : Semantics.ProvenanceSource.SourceEvent := { + backend := .vcs "git" "github" + source := e.repo + action := e.action + actor := { scheme := "user", name := "unspecified" } + ref := { scheme := "git-ref", value := e.action } + visibility := if e.isPublic then .publicRecord else .privateRecord + isValid := e.isValid && e.isPublic +} + /-- Invariant: Github events are lawful if they are intended for the public record and target the research-stack repo. -/ def githubInvariant (e : GithubEvent) : String := - if e.isValid && e.isPublic then s!"public_record_github:{e.repo}:{e.action}" - else "unlawful_github_attempt" + Semantics.ProvenanceSource.legacyInvariant + "public_record_github" + "unlawful_github_attempt" + (toSourceEvent e) open Semantics.Q16_16 @@ -26,7 +45,7 @@ Cost function: Measures the cost of public publication. Public visibility adds significant informational weight (Q16.16). -/ def githubCost (_e : GithubEvent) (_g : Metric) : Semantics.Q16_16 := - ofNat 5 + Semantics.ProvenanceSource.cost githubPolicy (toSourceEvent _e) "" _g /-- The Github Bind: Marks the idea as "in full view". diff --git a/0-Core-Formalism/lean/Semantics/Semantics/Github_metadata.json b/0-Core-Formalism/lean/Semantics/Semantics/Github_metadata.json index d5b457bd..7db1e8cb 100644 --- a/0-Core-Formalism/lean/Semantics/Semantics/Github_metadata.json +++ b/0-Core-Formalism/lean/Semantics/Semantics/Github_metadata.json @@ -1,3 +1,57 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a9984bf91c62cfeb78cacad71662f7d467a1e0c0198bbb7cd5fcf9d1eeaca788 -size 1417 +{ + "file_path": "/home/allaun/Research Stack/tools/lean/Semantics/Semantics/Github.lean", + "namespace": "Semantics.Github", + "imports": [ + "Semantics.ProvenanceSource" + ], + "opens": [ + "Semantics.Q16_16" + ], + "structures": [ + { + "name": "GithubEvent", + "line": 8, + "doc": "GithubEvent: Structure for GitHub-side research ingestion." + } + ], + "theorems": [], + "definitions": [ + { + "name": "githubPolicy", + "line": 14, + "doc": "Provider-specific cost policy for the Github compatibility shim." + }, + { + "name": "toSourceEvent", + "line": 21, + "doc": "Convert a legacy Github event into the open provenance-source event surface." + }, + { + "name": "githubInvariant", + "line": 35, + "doc": "Invariant: Github events are lawful if they are intended for the public record and target the research-stack repo." + }, + { + "name": "githubCost", + "line": 47, + "doc": "Cost function: Measures the cost of public publication. Public visibility adds significant informational weight (Q16.16)." + }, + { + "name": "githubBind", + "line": 53, + "doc": "The Github Bind: Marks the idea as \"in full view\"." + } + ], + "documentation": [ + "GithubEvent: Structure for GitHub-side research ingestion.", + "Provider-specific cost policy for the Github compatibility shim.", + "Convert a legacy Github event into the open provenance-source event surface.", + "Invariant: Github events are lawful if they are intended for the public record and target the research-stack repo.", + "Cost function: Measures the cost of public publication. Public visibility adds significant informational weight (Q16.16).", + "The Github Bind: Marks the idea as \"in full view\"." + ], + "total_lines": 56, + "gcl_encoded": true, + "gcl_sequence": "F00020200", + "gcl_length": 9 +} diff --git a/0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource.lean b/0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource.lean new file mode 100644 index 00000000..d9872c22 --- /dev/null +++ b/0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource.lean @@ -0,0 +1,148 @@ +import Semantics.Bind + +namespace Semantics.ProvenanceSource + +/-! +ProvenanceSource is the open-future event surface for artifact lifecycle +reports. A source event is a report, not a theorem: observation changes the +receipt surface, not the underlying law. + +The backend and reference kinds deliberately carry strings instead of closed +enums. The stack should accept future source classes without recompiling the +formal core: a future VCS, ledger, object store, toolbelt, stream, or manual +fixture drop is a configuration entry plus an adapter, not a new ontology law. +-/ + +inductive Backend where + | vcs (kind : String) (host : String) + | objectStore (kind : String) + | ledger (kind : String) + | registry (kind : String) + | stream (kind : String) + | filesystem (kind : String) + | toolbelt (kind : String) + | manual + | other (name : String) + deriving Repr, Inhabited + +structure Identity where + scheme : String + name : String + deriving Repr, Inhabited + +structure Reference where + scheme : String + value : String + deriving Repr, Inhabited + +inductive Visibility where + | publicRecord + | privateRecord + | internalRecord + | unspecifiedRecord + deriving Repr, Inhabited + +structure SourceEvent where + backend : Backend + source : String + action : String + actor : Identity + ref : Reference + visibility : Visibility + isValid : Bool + deriving Repr, Inhabited + +open Semantics.Q16_16 + +structure CostPolicy where + trustedActors : List String + trustedCost : Semantics.Q16_16 + untrustedCost : Semantics.Q16_16 + publicCost : Semantics.Q16_16 + deriving Repr, Inhabited + +def CostPolicy.neutral : CostPolicy := { + trustedActors := [] + trustedCost := ofNat 1 + untrustedCost := ofNat 1 + publicCost := ofNat 1 +} + +def CostPolicy.internalReview : CostPolicy := { + trustedActors := [] + trustedCost := ofFloat 0.5 + untrustedCost := ofNat 2 + publicCost := ofNat 5 +} + +def actorKey (actor : Identity) : String := + s!"{actor.scheme}:{actor.name}" + +def isTrusted (policy : CostPolicy) (actor : Identity) : Bool := + policy.trustedActors.any (fun key => key == actor.name || key == actorKey actor) + +def visibilityToken : Visibility → String + | .publicRecord => "public" + | .privateRecord => "private" + | .internalRecord => "internal" + | .unspecifiedRecord => "unspecified" + +def backendToken : Backend → String + | .vcs kind host => s!"vcs:{kind}:{host}" + | .objectStore kind => s!"object-store:{kind}" + | .ledger kind => s!"ledger:{kind}" + | .registry kind => s!"registry:{kind}" + | .stream kind => s!"stream:{kind}" + | .filesystem kind => s!"filesystem:{kind}" + | .toolbelt kind => s!"toolbelt:{kind}" + | .manual => "manual" + | .other name => s!"other:{name}" + +def invariant (event : SourceEvent) : String := + if event.isValid then + s!"lawful_source:{backendToken event.backend}:{event.source}:{event.action}:{event.ref.scheme}:{event.ref.value}" + else + "unlawful_source" + +def legacyInvariant (validPrefix invalidToken : String) (event : SourceEvent) : String := + if event.isValid then s!"{validPrefix}:{event.source}:{event.action}" else invalidToken + +def targetInvariant (event : SourceEvent) (_target : String) : String := + invariant event + +def legacyTargetInvariant (validPrefix : String) (event : SourceEvent) (_target : String) : String := + s!"{validPrefix}:{event.source}:{event.action}" + +def cost (policy : CostPolicy) (event : SourceEvent) (_target : String) (_g : Metric) : Semantics.Q16_16 := + if isTrusted policy event.actor then + policy.trustedCost + else + match event.visibility with + | .publicRecord => policy.publicCost + | _ => policy.untrustedCost + +def bind (policy : CostPolicy) (event : SourceEvent) (target : String) (g : Metric) : Bind SourceEvent String := + controlBind event target g (cost policy) invariant (targetInvariant event) + +def legacyBind + (policy : CostPolicy) + (validPrefix invalidToken : String) + (event : SourceEvent) + (target : String) + (g : Metric) : Bind SourceEvent String := + controlBind event target g + (cost policy) + (legacyInvariant validPrefix invalidToken) + (legacyTargetInvariant validPrefix event) + +#eval invariant { + backend := .toolbelt "science-toolbelt", + source := "shared-data/artifacts/science_toolbelt/probe.json", + action := "probed", + actor := { scheme := "tool", name := "probe_science_toolbelt.py" }, + ref := { scheme := "sha256", value := "example" }, + visibility := .internalRecord, + isValid := true +} + +end Semantics.ProvenanceSource diff --git a/0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource_metadata.json b/0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource_metadata.json new file mode 100644 index 00000000..5dd14361 --- /dev/null +++ b/0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource_metadata.json @@ -0,0 +1,109 @@ +{ + "file_path": "/home/allaun/Research Stack/tools/lean/Semantics/Semantics/ProvenanceSource.lean", + "namespace": "Semantics.ProvenanceSource", + "imports": [ + "Semantics.Bind" + ], + "opens": [ + "Semantics.Q16_16" + ], + "structures": [ + { + "name": "Identity", + "line": 28, + "doc": "Identity surface for a source-event actor." + }, + { + "name": "Reference", + "line": 33, + "doc": "Reference surface for a source-event artifact pointer." + }, + { + "name": "SourceEvent", + "line": 45, + "doc": "Open-future event report emitted by a configured source." + }, + { + "name": "CostPolicy", + "line": 57, + "doc": "Cost hints for source-event adapters." + } + ], + "theorems": [], + "definitions": [ + { + "name": "CostPolicy.neutral", + "line": 64, + "doc": "Neutral cost policy for receipt surfaces that should not bias bind cost." + }, + { + "name": "CostPolicy.internalReview", + "line": 71, + "doc": "Default internal-review policy for provider compatibility shims." + }, + { + "name": "actorKey", + "line": 78, + "doc": "Namespace an actor as scheme:name for policy matching." + }, + { + "name": "isTrusted", + "line": 81, + "doc": "Check whether a cost policy trusts an actor by name or scheme-qualified key." + }, + { + "name": "visibilityToken", + "line": 84, + "doc": "Render source-event visibility for invariant strings." + }, + { + "name": "backendToken", + "line": 90, + "doc": "Render an open backend descriptor for invariant strings." + }, + { + "name": "invariant", + "line": 101, + "doc": "Source-agnostic invariant string for lawful and unlawful source events." + }, + { + "name": "legacyInvariant", + "line": 107, + "doc": "Compatibility invariant formatter for legacy provider modules." + }, + { + "name": "targetInvariant", + "line": 110, + "doc": "Target-aware invariant adapter for Bind." + }, + { + "name": "legacyTargetInvariant", + "line": 113, + "doc": "Compatibility target invariant adapter for legacy provider modules." + }, + { + "name": "cost", + "line": 116, + "doc": "Source-agnostic cost function parameterized by policy and visibility." + }, + { + "name": "bind", + "line": 124, + "doc": "Bind a source event to a target through the source-agnostic policy surface." + }, + { + "name": "legacyBind", + "line": 127, + "doc": "Compatibility bind helper for legacy provider modules." + } + ], + "documentation": [ + "ProvenanceSource is the open-future event surface for artifact lifecycle reports.", + "Observation changes the receipt surface, not the underlying law.", + "The backend and reference kinds deliberately carry strings instead of closed enums." + ], + "total_lines": 148, + "gcl_encoded": true, + "gcl_sequence": "F00020200", + "gcl_length": 9 +} diff --git a/0-Core-Formalism/lean/external/OTOM/Forgejo.lean b/0-Core-Formalism/lean/external/OTOM/Forgejo.lean index 6603ca31..42c1cf43 100644 --- a/0-Core-Formalism/lean/external/OTOM/Forgejo.lean +++ b/0-Core-Formalism/lean/external/OTOM/Forgejo.lean @@ -1,4 +1,4 @@ -import Semantics.Bind +import Semantics.ProvenanceSource namespace Semantics.Forgejo @@ -11,21 +11,39 @@ structure ForgejoEvent where author : String isValid : Bool +def forgejoPolicy : Semantics.ProvenanceSource.CostPolicy := { + trustedActors := [] + trustedCost := 0x00008000 + untrustedCost := 0x00020000 + publicCost := 0x00020000 +} + +def toSourceEvent (e : ForgejoEvent) : Semantics.ProvenanceSource.SourceEvent := { + backend := .vcs "git" "forgejo" + source := e.repo + action := e.action + actor := { scheme := "user", name := e.author } + ref := { scheme := "git-ref", value := e.action } + visibility := .internalRecord + isValid := e.isValid +} + /-- Invariant: Forgejo events are lawful if they originate from an allowed repo and the action is within the prescribed set. -/ def forgejoInvariant (e : ForgejoEvent) : String := - if e.isValid then s!"lawful_forgejo:{e.repo}:{e.action}" - else "unlawful_forgejo" + Semantics.ProvenanceSource.legacyInvariant + "lawful_forgejo" + "unlawful_forgejo" + (toSourceEvent e) /-- Cost function: Measures the "computational friction" of a git event. Events from external authors have higher cost (Q16.16). -/ def forgejoCost (e1 : ForgejoEvent) (_target : String) (g : Metric) : UInt32 := - if e1.author == "sovereign" then 0x00008000 -- 0.5 cost - else 0x00020000 -- 2.0 cost + Semantics.ProvenanceSource.cost forgejoPolicy (toSourceEvent e1) _target g /-- The Forgejo Bind: Connects an event to the research substrate. diff --git a/0-Core-Formalism/lean/external/OTOM/Github.lean b/0-Core-Formalism/lean/external/OTOM/Github.lean index 36d83935..e6616003 100644 --- a/0-Core-Formalism/lean/external/OTOM/Github.lean +++ b/0-Core-Formalism/lean/external/OTOM/Github.lean @@ -1,4 +1,4 @@ -import Semantics.Bind +import Semantics.ProvenanceSource namespace Semantics.Github @@ -11,20 +11,39 @@ structure GithubEvent where isPublic : Bool isValid : Bool +def githubPolicy : Semantics.ProvenanceSource.CostPolicy := { + trustedActors := [] + trustedCost := 0x00050000 + untrustedCost := 0x00050000 + publicCost := 0x00050000 +} + +def toSourceEvent (e : GithubEvent) : Semantics.ProvenanceSource.SourceEvent := { + backend := .vcs "git" "github" + source := e.repo + action := e.action + actor := { scheme := "user", name := "unspecified" } + ref := { scheme := "git-ref", value := e.action } + visibility := if e.isPublic then .publicRecord else .privateRecord + isValid := e.isValid && e.isPublic +} + /-- Invariant: Github events are lawful if they are intended for the public record and target the research-stack repo. -/ def githubInvariant (e : GithubEvent) : String := - if e.isValid && e.isPublic then s!"public_record_github:{e.repo}:{e.action}" - else "unlawful_github_attempt" + Semantics.ProvenanceSource.legacyInvariant + "public_record_github" + "unlawful_github_attempt" + (toSourceEvent e) /-- Cost function: Measures the cost of public publication. Public visibility adds significant informational weight (Q16.16). -/ def githubCost (_e : GithubEvent) (_g : Metric) : UInt32 := - 0x00050000 -- 5.0 cost (high weight for public disclosure) + Semantics.ProvenanceSource.cost githubPolicy (toSourceEvent _e) "" _g /-- The Github Bind: Marks the idea as "in full view". diff --git a/0-Core-Formalism/lean/external/OTOM/ProvenanceSource.lean b/0-Core-Formalism/lean/external/OTOM/ProvenanceSource.lean new file mode 100644 index 00000000..e2a0361e --- /dev/null +++ b/0-Core-Formalism/lean/external/OTOM/ProvenanceSource.lean @@ -0,0 +1,113 @@ +import Semantics.Bind + +namespace Semantics.ProvenanceSource + +/-! +Open provenance-source surface for legacy OTOM staging modules. + +This mirrors the active Q16.16 module with raw UInt32 scalars so historical +external modules can delegate to the same source-event shape without importing +the active fixed-point stack. +-/ + +inductive Backend where + | vcs (kind : String) (host : String) + | objectStore (kind : String) + | ledger (kind : String) + | registry (kind : String) + | stream (kind : String) + | filesystem (kind : String) + | toolbelt (kind : String) + | manual + | other (name : String) + +structure Identity where + scheme : String + name : String + +structure Reference where + scheme : String + value : String + +inductive Visibility where + | publicRecord + | privateRecord + | internalRecord + | unspecifiedRecord + +structure SourceEvent where + backend : Backend + source : String + action : String + actor : Identity + ref : Reference + visibility : Visibility + isValid : Bool + +structure CostPolicy where + trustedActors : List String + trustedCost : UInt32 + untrustedCost : UInt32 + publicCost : UInt32 + +def actorKey (actor : Identity) : String := + s!"{actor.scheme}:{actor.name}" + +def isTrusted (policy : CostPolicy) (actor : Identity) : Bool := + policy.trustedActors.any (fun key => key == actor.name || key == actorKey actor) + +def visibilityToken : Visibility → String + | .publicRecord => "public" + | .privateRecord => "private" + | .internalRecord => "internal" + | .unspecifiedRecord => "unspecified" + +def backendToken : Backend → String + | .vcs kind host => s!"vcs:{kind}:{host}" + | .objectStore kind => s!"object-store:{kind}" + | .ledger kind => s!"ledger:{kind}" + | .registry kind => s!"registry:{kind}" + | .stream kind => s!"stream:{kind}" + | .filesystem kind => s!"filesystem:{kind}" + | .toolbelt kind => s!"toolbelt:{kind}" + | .manual => "manual" + | .other name => s!"other:{name}" + +def invariant (event : SourceEvent) : String := + if event.isValid then + s!"lawful_source:{backendToken event.backend}:{event.source}:{event.action}:{event.ref.scheme}:{event.ref.value}" + else + "unlawful_source" + +def legacyInvariant (validPrefix invalidToken : String) (event : SourceEvent) : String := + if event.isValid then s!"{validPrefix}:{event.source}:{event.action}" else invalidToken + +def targetInvariant (event : SourceEvent) (_target : String) : String := + invariant event + +def legacyTargetInvariant (validPrefix : String) (event : SourceEvent) (_target : String) : String := + s!"{validPrefix}:{event.source}:{event.action}" + +def cost (policy : CostPolicy) (event : SourceEvent) (_target : String) (_g : Metric) : UInt32 := + if isTrusted policy event.actor then + policy.trustedCost + else + match event.visibility with + | .publicRecord => policy.publicCost + | _ => policy.untrustedCost + +def bind (policy : CostPolicy) (event : SourceEvent) (target : String) (g : Metric) : Bind SourceEvent String := + controlBind event target g (cost policy) invariant (targetInvariant event) + +def legacyBind + (policy : CostPolicy) + (validPrefix invalidToken : String) + (event : SourceEvent) + (target : String) + (g : Metric) : Bind SourceEvent String := + controlBind event target g + (cost policy) + (legacyInvariant validPrefix invalidToken) + (legacyTargetInvariant validPrefix event) + +end Semantics.ProvenanceSource diff --git a/4-Infrastructure/witness/README.md b/4-Infrastructure/witness/README.md new file mode 100644 index 00000000..3160616a --- /dev/null +++ b/4-Infrastructure/witness/README.md @@ -0,0 +1,38 @@ +# Witness Sources + +`sources.json` is the provider registry for artifact lifecycle reports. It is +deliberately source-agnostic: GitHub, GitLab, Forgejo, bare git, Mercurial, CVS, +science-toolbelt probes, DeepSeek receipts, and SHA-256 ledgers are all source +blocks with a backend descriptor. + +Operating rule: + +> Observation changes the receipt surface, not the underlying law. + +Sources emit reports. They do not make mathematical claims true. Lean modules, +claim-registry promotion, and receipt validators decide what can be treated as +formal or review evidence. + +## Source Blocks + +Each source block should identify: + +- `active`: whether the stack should listen to it now. +- `backend`: `{type, kind, host}` or the closest equivalent. +- `url`: file path, repository URL, artifact directory, or logical endpoint. +- `hook_kind`: how events are observed, such as `webhook`, `server-side`, + `filesystem`, or `post-event-script`. +- `schema`: optional machine-readable receipt schema. +- `cost_policy`: optional bind-cost hints for source-event adapters. + +Unknown future backends should be added as new source blocks first. Code should +only grow an adapter once a source produces real receipts worth consuming. + +## Current Consumers + +- `0-Core-Formalism/lean/Semantics/Semantics/ProvenanceSource.lean` defines the + open formal shape of source events. +- `5-Applications/tools-scripts/substrate/substrate_git_index.py` uses the + source registry when installing git post-receive hooks. +- `5-Applications/scripts/attest_signal_wave_unification.py` stores + provider-tagged attestation records. diff --git a/4-Infrastructure/witness/sources.json b/4-Infrastructure/witness/sources.json new file mode 100644 index 00000000..55e411b5 --- /dev/null +++ b/4-Infrastructure/witness/sources.json @@ -0,0 +1,125 @@ +{ + "schema": "research_stack_witness_sources_v1", + "notes": [ + "Observation changes the receipt surface, not the underlying law.", + "Sources are reports, not authorities; Lean and the claims registry decide promotion." + ], + "sources": { + "research-stack-github": { + "active": true, + "backend": { + "type": "vcs", + "kind": "git", + "host": "github" + }, + "url": "https://github.com/allaunthefox/Research-Stack.git", + "api_base": "https://api.github.com", + "auth_env": "GITHUB_TOKEN", + "hook_kind": "webhook", + "visibility": "public", + "cost_policy": { + "trusted_actors": [], + "trusted": 0.5, + "untrusted": 2.0, + "public": 5.0 + } + }, + "research-stack-bare-local": { + "active": true, + "backend": { + "type": "vcs", + "kind": "git", + "host": "bare" + }, + "url": "file:///var/lib/git/research-stack.git", + "hook_kind": "server-side", + "visibility": "internal", + "cost_policy": { + "trusted_actors": [], + "trusted": 0.5, + "untrusted": 2.0, + "public": 5.0 + } + }, + "research-stack-forgejo-legacy": { + "active": false, + "backend": { + "type": "vcs", + "kind": "git", + "host": "forgejo" + }, + "url": "file:///var/lib/git/research-stack.git", + "hook_kind": "server-side", + "visibility": "internal", + "retirement_note": "Forgejo is preserved for historical receipt replay only; new deployments should configure a provider-specific source block.", + "cost_policy": { + "trusted_actors": [], + "trusted": 0.5, + "untrusted": 2.0, + "public": 5.0 + } + }, + "science-toolbelt-probe": { + "active": true, + "backend": { + "type": "toolbelt", + "kind": "science-toolbelt" + }, + "url": "shared-data/artifacts/science_toolbelt/probe.json", + "schema": "research_stack_optional_science_toolbelt_probe_v1", + "hook_kind": "post-event-script", + "visibility": "internal", + "cost_policy": { + "trusted_actors": [], + "trusted": 1.0, + "untrusted": 1.0, + "public": 1.0 + } + }, + "deepseek-review": { + "active": true, + "backend": { + "type": "toolbelt", + "kind": "deepseek-review" + }, + "url": "shared-data/artifacts/deepseek_review/", + "schema": "ollama_deepseek_review_receipt_v1", + "hook_kind": "post-event-script", + "visibility": "internal" + }, + "consolidation-ledger": { + "active": true, + "backend": { + "type": "ledger", + "kind": "sha256-manifest" + }, + "url": ".consolidation-manifests/", + "hook_kind": "filesystem", + "visibility": "internal" + }, + "example-mercurial": { + "active": false, + "backend": { + "type": "vcs", + "kind": "mercurial", + "host": "self-hosted" + }, + "url": "ssh://example.invalid/research-stack", + "hook_kind": "pretxnchangegroup", + "visibility": "internal", + "adapter_status": "config-only" + }, + "example-cvs": { + "active": false, + "backend": { + "type": "vcs", + "kind": "cvs", + "host": "self-hosted" + }, + "url": "cvs://example.invalid/research-stack", + "hook_kind": "loginfo", + "visibility": "internal", + "adapter_status": "config-only" + } + } +} diff --git a/5-Applications/scripts/attest_signal_wave_unification.py b/5-Applications/scripts/attest_signal_wave_unification.py index 9afdbd74..552a5e55 100644 --- a/5-Applications/scripts/attest_signal_wave_unification.py +++ b/5-Applications/scripts/attest_signal_wave_unification.py @@ -2,15 +2,16 @@ """ Signal-Wave Unification Attestation System -Performs remote attestation in both git and forgejo for EQUATION #0.2: +Performs source-aware attestation for EQUATION #0.2: Φ_SW(x) = Σₖ wₖ e^{ik·x} - λ∫_{‖h‖=1} |Σₖ wₖ e^{ik·h}|² dh Attestation Chain: - Git commit with attestation metadata -- Forgejo issue with cross-reference +- Provider record with cross-reference - Database entry in math_entities """ +import argparse import sys import json import sqlite3 @@ -24,13 +25,35 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent / "4-Infrastructure" from lean_unified_shim import SwarmAPISystem +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_SOURCE = "research-stack-github" +DEFAULT_WITNESS_CONFIG = REPO_ROOT / "4-Infrastructure" / "witness" / "sources.json" + + +def load_source_config(source: str, config_path: Path = DEFAULT_WITNESS_CONFIG) -> Dict[str, Any]: + if not config_path.exists(): + return {"name": source, "active": False, "backend": {"type": "unknown"}} + data = json.loads(config_path.read_text(encoding="utf-8")) + payload = data.get("sources", {}).get(source, {}) + if not isinstance(payload, dict): + payload = {} + return {"name": source, **payload} + + class AttestationSystem: """Remote attestation for mathematical entities.""" - def __init__(self, repo_path: str = "/home/allaun/Research Stack"): + def __init__( + self, + repo_path: str = str(REPO_ROOT), + source: str = DEFAULT_SOURCE, + config_path: Path = DEFAULT_WITNESS_CONFIG, + ): self.repo_path = Path(repo_path) self.api = SwarmAPISystem() self.timestamp = datetime.now(timezone.utc) + self.source = source + self.source_config = load_source_config(source, config_path) def calculate_sha256(self, file_path: Path) -> str: """Calculate SHA256 hash of a file.""" @@ -138,15 +161,23 @@ Signed-off-by: Cascade (Triumvirate Agent) 'phase': 'exception' } - def forgejo_attest(self, entity_id: str, git_commit: str) -> Dict[str, Any]: + def source_attest(self, entity_id: str, git_commit: str) -> Dict[str, Any]: """ - Create Forgejo issue attestation. + Create provider-scoped attestation metadata. - In production, this would use the Forgejo API. - For now, we create a local attestation record. + In production, provider adapters may post to GitHub, GitLab, Forgejo, + a bare-repo note, or another configured source. For now, this creates a + local provider-tagged record that can be replayed by future adapters. """ - forgejo_record = { + provider_record = { 'entity_id': entity_id, + 'source': self.source, + 'source_config': { + 'backend': self.source_config.get('backend', {}), + 'url': self.source_config.get('url'), + 'hook_kind': self.source_config.get('hook_kind'), + 'active': self.source_config.get('active', False), + }, 'title': f'[ATTESTATION] {entity_id} — Signal-Wave Unification Equation', 'body': f"""## Remote Attestation Record @@ -154,12 +185,13 @@ Signed-off-by: Cascade (Triumvirate Agent) **Git Commit:** `{git_commit}` **Timestamp:** {self.timestamp.isoformat()} +**Source:** `{self.source}` **Classification:** P0 CRITICAL ### Attestation Chain 1. ✅ Git commit: {git_commit} -2. ⏳ Forgejo issue: [pending API integration] +2. ⏳ Provider record: [pending adapter integration] 3. ✅ Database entry: math_entities.{entity_id} ### Verification Checklist @@ -191,19 +223,20 @@ Signed-off-by: Cascade (Triumvirate Agent) 'created_at': self.timestamp.isoformat() } - # Store locally (would push to Forgejo API in production) + # Store locally; future source adapters can replay this record. attestation_path = self.repo_path / "out" / "attestations" / f"{entity_id}.json" attestation_path.parent.mkdir(parents=True, exist_ok=True) with open(attestation_path, 'w') as f: - json.dump(forgejo_record, f, indent=2) + json.dump(provider_record, f, indent=2) return { 'success': True, - 'record': forgejo_record, + 'record': provider_record, 'local_path': str(attestation_path), - 'phase': 'forgejo', - 'note': 'Local record created (Forgejo API integration pending)' + 'phase': 'source', + 'source': self.source, + 'note': 'Local provider record created; remote adapter integration pending' } def database_attest(self, entity_id: str, git_commit: str, @@ -285,7 +318,7 @@ Signed-off-by: Cascade (Triumvirate Agent) def full_attestation(self) -> Dict[str, Any]: """ - Perform complete attestation chain: git → forgejo → database + Perform complete attestation chain: git → source record → database """ entity_id = f'SIGNAL_WAVE_UNIFICATION_P0_{self.timestamp.strftime("%Y%m%d")}' file_path = self.repo_path / "docs" / "papers" / "EQUATION_02_SIGNAL_WAVE_UNIFICATION.md" @@ -302,8 +335,8 @@ Signed-off-by: Cascade (Triumvirate Agent) if not git_result['success']: return git_result - # Phase 2: Forgejo attestation - forgejo_result = self.forgejo_attest(entity_id, git_result['commit_hash']) + # Phase 2: provider-scoped source attestation + source_result = self.source_attest(entity_id, git_result['commit_hash']) # Phase 3: Database attestation db_result = self.database_attest( @@ -319,7 +352,7 @@ Signed-off-by: Cascade (Triumvirate Agent) 'timestamp': self.timestamp.isoformat(), 'attestation_chain': { 'git': git_result, - 'forgejo': forgejo_result, + 'source': source_result, 'database': db_result }, 'verification': { @@ -331,17 +364,28 @@ Signed-off-by: Cascade (Triumvirate Agent) def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", default=str(REPO_ROOT), help="Repository checkout to attest.") + parser.add_argument("--source", default=DEFAULT_SOURCE, help="Witness source block name.") + parser.add_argument( + "--sources-config", + type=Path, + default=DEFAULT_WITNESS_CONFIG, + help="Witness source configuration JSON.", + ) + args = parser.parse_args() + print("="*70) print("SIGNAL-WAVE UNIFICATION ATTESTATION") print("="*70) print() print("Performing remote attestation in:") print(" 1. Git (commit with attestation metadata)") - print(" 2. Forgejo (issue record)") + print(f" 2. Source record ({args.source})") print(" 3. Database (math_entities entry)") print() - attestor = AttestationSystem() + attestor = AttestationSystem(args.repo, args.source, args.sources_config) result = attestor.full_attestation() if result['success']: @@ -354,13 +398,14 @@ def main(): print("Attestation Chain:") print(f" Git Commit: {result['attestation_chain']['git']['commit_hash']}") print(f" File SHA256: {result['verification']['file_sha256']}") + print(f" Source: {result['attestation_chain']['source']['source']}") print(f" Database: {result['attestation_chain']['database']['database']}") print(f" MATH_MODEL_MAP: Entry {result['verification']['math_model_map_entry']}") print() print("="*70) print("The Signal-Wave Unification equation is now:") print(" - Committed to git with attestation metadata") - print(" - Recorded in Forgejo attestation system") + print(" - Recorded in the configured source attestation surface") print(" - Added to math_entities database") print() print("Triumvirate Assignment:") diff --git a/5-Applications/scripts/refill_gdrive_from_forgejo.sh b/5-Applications/scripts/refill_gdrive_from_forgejo.sh index b3bd87be..af48a67c 100755 --- a/5-Applications/scripts/refill_gdrive_from_forgejo.sh +++ b/5-Applications/scripts/refill_gdrive_from_forgejo.sh @@ -4,25 +4,27 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -REMOTE="${1:-forgejo/main}" +REMOTE="${1:-github/distilled}" +REMOTE_NAME="${REMOTE%%/*}" +REMOTE_LABEL="$(echo "$REMOTE" | tr '/: ' '---')" DATE_STAMP="$(date +%F)" OUT_DIR="artifacts/gdrive_refill" mkdir -p "$OUT_DIR" -echo "[1/5] Fetching Forgejo..." -git fetch forgejo --prune +echo "[1/5] Fetching remote $REMOTE_NAME..." +git fetch "$REMOTE_NAME" --prune REV="$(git rev-parse --short=12 "$REMOTE")" -ARCHIVE="$OUT_DIR/research-stack-forgejo-main-${REV}.tar.zst" +ARCHIVE="$OUT_DIR/research-stack-${REMOTE_LABEL}-${REV}.tar.zst" echo "[2/5] Building clean archive from $REMOTE ($REV)..." if [[ ! -f "$ARCHIVE" ]]; then if command -v zstd >/dev/null 2>&1; then - git archive --format=tar --prefix="research-stack-forgejo-main-${REV}/" "$REMOTE" \ + git archive --format=tar --prefix="research-stack-${REMOTE_LABEL}-${REV}/" "$REMOTE" \ | zstd -T0 -3 -o "$ARCHIVE" else - ARCHIVE="$OUT_DIR/research-stack-forgejo-main-${REV}.tar.gz" - git archive --format=tar --prefix="research-stack-forgejo-main-${REV}/" "$REMOTE" \ + ARCHIVE="$OUT_DIR/research-stack-${REMOTE_LABEL}-${REV}.tar.gz" + git archive --format=tar --prefix="research-stack-${REMOTE_LABEL}-${REV}/" "$REMOTE" \ | gzip -6 > "$ARCHIVE" fi fi @@ -46,7 +48,7 @@ Archive SHA-256: $SHA ## Upload Target \`\`\`bash -rclone copy artifacts/gdrive_refill Gdrive:topological_storage/research-stack/forgejo-refill/$DATE_STAMP/ --progress --checksum +rclone copy artifacts/gdrive_refill Gdrive:topological_storage/research-stack/remote-refill/$DATE_STAMP/ --progress --checksum \`\`\` EOF @@ -54,13 +56,13 @@ echo "[4/5] Checking Gdrive rclone access..." if ! rclone lsf Gdrive: >/dev/null 2>&1; then echo "Gdrive: is not authenticated." echo "Run: rclone config reconnect Gdrive:" - echo "Then rerun: bash scripts/refill_gdrive_from_forgejo.sh" + echo "Then rerun: bash scripts/refill_gdrive_from_forgejo.sh " exit 2 fi echo "[5/5] Uploading refill bundle to Google Drive..." -rclone copy "$OUT_DIR" "Gdrive:topological_storage/research-stack/forgejo-refill/$DATE_STAMP/" \ +rclone copy "$OUT_DIR" "Gdrive:topological_storage/research-stack/remote-refill/$DATE_STAMP/" \ --progress \ --checksum -echo "Done. Uploaded $OUT_DIR to Gdrive:topological_storage/research-stack/forgejo-refill/$DATE_STAMP/" +echo "Done. Uploaded $OUT_DIR to Gdrive:topological_storage/research-stack/remote-refill/$DATE_STAMP/" diff --git a/5-Applications/scripts/self_clean_shim.py b/5-Applications/scripts/self_clean_shim.py index 23fe389a..b05b8cbb 100644 --- a/5-Applications/scripts/self_clean_shim.py +++ b/5-Applications/scripts/self_clean_shim.py @@ -210,7 +210,7 @@ def analyze_file(filepath: Path) -> Optional[FileMetrics]: if is_lean: # Check if module name is in the known failing list from context failing_modules = { - "AVMR", "DomainKernel", "Forgejo", "Metatype", "FuzzyAssociation", + "AVMR", "DomainKernel", "Metatype", "FuzzyAssociation", "SurfaceCore", "Transition", "Protocol", "CalibratedKernel", "BraidCross", "SSMS_nD" } diff --git a/5-Applications/scripts/server.js b/5-Applications/scripts/server.js index 470fabc2..b2e1f97f 100644 --- a/5-Applications/scripts/server.js +++ b/5-Applications/scripts/server.js @@ -106,7 +106,7 @@ app.get("/health/network", async (_req, res) => { substrate: "OFFLINE", tailscale: "CHECKING", remotes: { - forgejo: "ONLINE", + witnessSources: "CONFIGURED", i2p: "OFFLINE" } }; diff --git a/5-Applications/tools-scripts/i2p_warden.py b/5-Applications/tools-scripts/i2p_warden.py index f3e4b84b..a10c133a 100644 --- a/5-Applications/tools-scripts/i2p_warden.py +++ b/5-Applications/tools-scripts/i2p_warden.py @@ -59,7 +59,7 @@ def attempt_restart(): def alert_user(message): """Alerts the user via console and potentially other channels.""" print(f"\n[!!! WARDEN ALERT !!!] {message}") - # Integration with Linear or Forgejo could go here + # Integration with Linear or a configured witness source could go here # For now, we use a high-visibility console alert def main(): @@ -78,7 +78,7 @@ def main(): if not success: alert_user("i2p Router is FATALLY OFFLINE. Manual intervention required.") - # Optional: push to Forgejo if Tailscale is up + # Optional: push to a configured witness source if Tailscale is up return False else: print("[WARDEN] i2p is healthy.") diff --git a/5-Applications/tools-scripts/ingestion/ingest_archive.py b/5-Applications/tools-scripts/ingestion/ingest_archive.py index 0732f027..4d03d7d7 100644 --- a/5-Applications/tools-scripts/ingestion/ingest_archive.py +++ b/5-Applications/tools-scripts/ingestion/ingest_archive.py @@ -153,7 +153,7 @@ _CLUSTER_KEYWORDS: dict[int, set[str]] = { 5: {"crypto", "hash", "sha256", "merkle", "zk", "stark", "proof", "signature", "seal", "attest", "verify", "provenance"}, 6: {"database", "sql", "sqlite", "index", "query", "schema", "table", - "row", "column", "fts", "search", "forgejo", "git"}, + "row", "column", "fts", "search", "source", "git"}, 7: {"semantic", "language", "concept", "meaning", "vector", "embed", "nlp", "token", "word", "text", "notation", "symbol", "translate", "analog", "idea", "weight", "research"}, diff --git a/5-Applications/tools-scripts/ingestion/ingest_large_file.py b/5-Applications/tools-scripts/ingestion/ingest_large_file.py index fe1b10f7..84e8dc52 100644 --- a/5-Applications/tools-scripts/ingestion/ingest_large_file.py +++ b/5-Applications/tools-scripts/ingestion/ingest_large_file.py @@ -210,7 +210,7 @@ _CLUSTER_KEYWORDS: dict[int, set[str]] = { 5: {"crypto", "hash", "sha256", "merkle", "zk", "stark", "proof", "signature", "seal", "attest", "verify", "provenance"}, 6: {"database", "sql", "sqlite", "index", "query", "schema", "table", - "row", "column", "fts", "search", "forgejo", "git"}, + "row", "column", "fts", "search", "source", "git"}, 7: {"semantic", "language", "concept", "meaning", "vector", "embed", "nlp", "token", "word", "text", "notation", "symbol", "translate", "analog", "idea", "weight", "research", "coding", "decode", diff --git a/5-Applications/tools-scripts/io_harness_compat.py b/5-Applications/tools-scripts/io_harness_compat.py new file mode 100644 index 00000000..9ea36437 --- /dev/null +++ b/5-Applications/tools-scripts/io_harness_compat.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Compatibility helpers for Warden-bounded scripts. + +Several tool scripts were rewritten to route process spawning and network +fetches through this module. The module is intentionally tiny: it centralizes +the boundary without changing each caller's payload logic. +""" +from __future__ import annotations + +import subprocess +import urllib.request +from collections.abc import Mapping, Sequence +from typing import Any + + +def spawn_isolated_process( + argv: Sequence[str], + *, + timeout: float | None = None, + cwd: str | None = None, + env: Mapping[str, str] | None = None, + input: bytes | None = None, +) -> tuple[int, bytes, bytes]: + """Run a subprocess without a shell and return ``(code, stdout, stderr)``.""" + result = subprocess.run( + list(argv), + input=input, + capture_output=True, + timeout=timeout, + cwd=cwd, + env=dict(env) if env is not None else None, + check=False, + ) + return result.returncode, result.stdout, result.stderr + + +def fetch_network_resource( + url: str, + *, + headers: Mapping[str, str] | None = None, + timeout: float | None = None, + method: str = "GET", + data: bytes | None = None, + **_unused: Any, +) -> bytes: + """Fetch a URL and return response bytes.""" + req = urllib.request.Request( + url, + data=data, + headers=dict(headers or {}), + method=method, + ) + with urllib.request.urlopen(req, timeout=timeout) as response: + return response.read() diff --git a/5-Applications/tools-scripts/metafoam/metafoam_pkg.py b/5-Applications/tools-scripts/metafoam/metafoam_pkg.py index 603f2845..9ede48a4 100644 --- a/5-Applications/tools-scripts/metafoam/metafoam_pkg.py +++ b/5-Applications/tools-scripts/metafoam/metafoam_pkg.py @@ -156,7 +156,7 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent while not (REPO_ROOT / ".git").exists() and REPO_ROOT != REPO_ROOT.parent: REPO_ROOT = REPO_ROOT.parent -DEFAULT_REMOTE = "forgejo" +DEFAULT_REMOTE = os.environ.get("PTOS_REMOTE", "github") _PHI = (1 + 5 ** 0.5) / 2 # golden ratio — used for nd_point and foam_score SIDECAR_EXTENSIONS = frozenset({ ".json", @@ -367,7 +367,8 @@ PACKAGES: dict = { "5-Applications/scripts/math_check_packages.py", "5-Applications/scripts/repair_index.py", "5-Applications/scripts/stage0_classifier.py", - "5-Applications/scripts/substrate_git_index.py", + "5-Applications/tools-scripts/substrate/substrate_git_index.py", + "4-Infrastructure/witness/sources.json", "5-Applications/tests/test_iso_symbol_table.py", "5-Applications/tests/test_math_check_packages.py", ], diff --git a/5-Applications/tools-scripts/substrate/substrate_git_index.py b/5-Applications/tools-scripts/substrate/substrate_git_index.py index 3b3cddd1..a8a45d83 100644 --- a/5-Applications/tools-scripts/substrate/substrate_git_index.py +++ b/5-Applications/tools-scripts/substrate/substrate_git_index.py @@ -1,6 +1,6 @@ # ============================================================================== # COPYRIGHT NO ONE EVERYWHERE LLC (WYOMING HOLDING COMPANY) -# PROJECT: SOVEREIGN STACK +# PROJECT: OBSERVERLESS STACK # This artifact is entirely proprietary and cryptographically proven. # Open-Source usage requires explicit permission from Brandon Scott Schneider. # ============================================================================== @@ -17,33 +17,34 @@ except ImportError: #!/usr/bin/env python3 # PTOS: LAYER=STORE / DOMAIN=DATA / CONDITION=EXPERIMENTAL / STAGE=ACTIVE / SOURCE=CODE """ -Substrate Git Index — Layer 3 of the sovereign computing stack. -=============================================================== +Substrate Git Index — provider-driven source index. +=================================================== POSITION IN THE STACK --------------------- -This script is the database layer that bridges Forgejo (git object store) +This script is the database layer that bridges configured artifact sources and the rest of the substrate: Hardware Platform (KDA / RISC-V nodes / HDL) ↑ GraphVM OS (TSM opcodes, FOAM voxels, substrate ISA, omnitoken bus) ↑ - Git/Forgejo SQL DB ← THIS FILE - ├── Forgejo bare repo content-addressed blob storage (git objects) + Source-backed SQL DB ← THIS FILE + ├── configured source VCS / ledger / toolbelt / filesystem reports ├── SQLite index queryable PTOS-tagged package registry └── HTTP query API substrate-facing query interface CONCEPT ------- -Every `pkg/*` tag pushed to Forgejo carries a full PTOS manifest in its -annotation (written by metafoam_pkg.py). This script intercepts that push -via a Forgejo server-side post-receive hook, parses the JSON annotation, and -inserts a row into a SQLite database whose schema maps 1-to-1 to the PTOS -tag axes. +Every `pkg/*` tag pushed to a configured git source carries a full PTOS +manifest in its annotation (written by metafoam_pkg.py). This script can +intercept that push via a server-side post-receive hook, parse the JSON +annotation, and insert a row into a SQLite database whose schema maps 1-to-1 +to the PTOS tag axes. -The result: Forgejo behaves as a custom SQL backend. Git handles durability -and content-addressing; SQLite handles queryability; PTOS tags are the schema. +The result: any configured source can feed a custom SQL backend. Git handles +durability and content-addressing for git sources; SQLite handles queryability; +PTOS tags are the schema. git object store ≡ row storage (BLOB columns) annotated tag ≡ typed SQL row @@ -60,8 +61,8 @@ This script runs in three distinct modes: Git calls this after a push. Reads lines from stdin. For each refs/tags/pkg/* ref, reads the tag annotation from the git object, parses the PTOS JSON, and indexes it into SQLite. - Must be installed at: - /home/forgejo/repositories/sovereign/research-stack.git/hooks/post-receive + Install it with the source-aware wrapper: + python3 substrate_git_index.py install --repo --source 2. serve — HTTP query server daemon. Listens on a TCP port (default 7743). Accepts JSON queries and @@ -130,13 +131,10 @@ The serve mode exposes a minimal JSON API on port 7743 (default): All responses are JSON. The SQL endpoint accepts arbitrary SELECT statements against the packages table — the substrate can issue any query it needs. -INSTALL (on VPS) ----------------- - # Copy to VPS - scp substrate_git_index.py sovereign@38.242.222.130:~/substrate_git_index.py - - # Install as post-receive hook - python3 substrate_git_index.py install --repo /home/forgejo/repositories/sovereign/research-stack.git +INSTALL +------- + # Install as post-receive hook for any bare git source + python3 substrate_git_index.py install --repo /path/to/research-stack.git --source research-stack-bare-local # Start HTTP server (background) python3 substrate_git_index.py serve --port 7743 --db /var/lib/substrate/index.db & @@ -150,7 +148,7 @@ Usage: python3 substrate_git_index.py index index one tag manually python3 substrate_git_index.py query "" CLI SQL query python3 substrate_git_index.py status show indexed packages - python3 substrate_git_index.py install --repo install as Forgejo hook + python3 substrate_git_index.py install --repo install as a git post-receive hook python3 substrate_git_index.py install-service write systemd unit python3 substrate_git_index.py schema print CREATE TABLE python3 substrate_git_index.py drop [version] remove a record @@ -161,7 +159,6 @@ import hashlib import http.server import json import os -import shutil import sqlite3 import subprocess import sys @@ -171,6 +168,59 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any + +def _repo_root() -> Path: + here = Path(__file__).resolve() + for parent in here.parents: + if (parent / ".git").exists() or (parent / "AGENTS.md").exists(): + return parent + return here.parents[3] + + +REPO_ROOT = _repo_root() + + +def _witness_config_path() -> Path: + return Path( + os.environ.get( + "WITNESS_SOURCES_CONFIG", + REPO_ROOT / "4-Infrastructure" / "witness" / "sources.json", + ) + ) + + +WITNESS_SOURCE = os.environ.get("WITNESS_SOURCE", "research-stack-github") + + +def _load_witness_sources(config_path: Path | None = None) -> dict[str, Any]: + path = config_path or _witness_config_path() + if not path.exists(): + return {"schema": "research_stack_witness_sources_v1", "sources": {}} + return json.loads(path.read_text(encoding="utf-8")) + + +def _source_config(source_name: str = WITNESS_SOURCE) -> dict[str, Any]: + data = _load_witness_sources() + sources = data.get("sources", {}) + value = sources.get(source_name) + if not isinstance(value, dict): + return {} + return value + + +def _file_url_to_path(url: str) -> Path | None: + if url.startswith("file://"): + return Path(urllib.parse.unquote(url[len("file://") :])) + if "://" not in url and url: + return Path(url) + return None + + +def _repo_path_from_source(source_name: str = WITNESS_SOURCE) -> Path | None: + source = _source_config(source_name) + url = str(source.get("url", "")) + return _file_url_to_path(url) + # ───────────────────────────────────────────────────────────────────────────── # Configuration — override via environment variables on the VPS # ───────────────────────────────────────────────────────────────────────────── @@ -184,7 +234,10 @@ DB_PATH = Path( # Git repo to operate on (used by hook and index commands). # The hook mode detects this automatically from GIT_DIR env var set by git. REPO_PATH = Path( - os.environ.get("SUBSTRATE_REPO", Path(__file__).parent.parent) + os.environ.get( + "GIT_REPO_PATH", + os.environ.get("SUBSTRATE_REPO", os.environ.get("GIT_DIR", Path(__file__).parent.parent)), + ) ) # Default HTTP server port. @@ -193,10 +246,12 @@ DEFAULT_PORT = int(os.environ.get("SUBSTRATE_PORT", "7743")) # Tag prefix that identifies metafoam packages. PKG_TAG_PREFIX = "refs/tags/pkg/" -# Forgejo bare repo location (used by install command). -FORGEJO_REPO = os.environ.get( - "FORGEJO_REPO", - "/home/forgejo/repositories/sovereign/research-stack.git", +# Bare git repo location (used by install command). Prefer an explicit +# GIT_REPO_PATH or a provider block in 4-Infrastructure/witness/sources.json. +DEFAULT_HOOK_REPO = ( + os.environ.get("GIT_REPO_PATH") + or os.environ.get("SUBSTRATE_REPO") + or (str(_repo_path_from_source()) if _repo_path_from_source() else "") ) @@ -1464,51 +1519,68 @@ def cmd_ingest_session(session_file: str) -> None: _SYSTEMD_UNIT = """\ [Unit] Description=Substrate Git Index — HTTP query server -After=network.target forgejo.service +After=network.target [Service] Type=simple -User=forgejo ExecStart=/usr/bin/python3 {script} serve --port {port} --db {db} Restart=on-failure RestartSec=5 Environment=SUBSTRATE_DB={db} Environment=SUBSTRATE_PORT={port} +Environment=WITNESS_SOURCE={source} +Environment=WITNESS_SOURCES_CONFIG={config} [Install] WantedBy=multi-user.target """ -def cmd_install(repo: str = FORGEJO_REPO) -> None: - """Install this script as the post-receive hook in a Forgejo bare repo. +def cmd_install(repo: str | None = None, source: str = WITNESS_SOURCE) -> None: + """Install this script as the post-receive hook in a bare git repo. # What this does - 1. Copies this script to /hooks/post-receive + 1. Writes a small provider-aware wrapper to /hooks/post-receive 2. Makes it executable (chmod +x) - 3. Sets SUBSTRATE_DB env var in the hook so it knows where to write. + 3. Sets WITNESS_SOURCE, GIT_REPO_PATH, and SUBSTRATE_DB for the hook. # Parameters - repo : path to the Forgejo bare repo - default: /home/forgejo/repositories/sovereign/research-stack.git + repo : path to the bare git repo + source : source block in 4-Infrastructure/witness/sources.json """ + source_repo = _repo_path_from_source(source) + repo = repo or (str(source_repo) if source_repo else DEFAULT_HOOK_REPO) + if not repo: + print("[install] ERROR: no repo supplied and no local file:// source is configured.") + print(" Use --repo or set GIT_REPO_PATH.") + sys.exit(1) + hooks_dir = Path(repo) / "hooks" if not hooks_dir.exists(): print(f"[install] ERROR: hooks dir not found: {hooks_dir}") sys.exit(1) dest = hooks_dir / "post-receive" - shutil.copy2(__file__, dest) + script = Path(__file__).resolve() + wrapper = f"""#!/usr/bin/env sh +export WITNESS_SOURCE={source!r} +export WITNESS_SOURCES_CONFIG={str(_witness_config_path())!r} +export GIT_REPO_PATH="${{GIT_DIR:-{str(Path(repo).resolve())}}}" +export SUBSTRATE_DB={str(DB_PATH)!r} +exec /usr/bin/python3 {str(script)!r} hook "$@" +""" + dest.write_text(wrapper, encoding="utf-8") dest.chmod(0o755) print(f"[install] hook installed → {dest}") + print(f" Source: {source}") print(f" DB will be written to: {DB_PATH}") print() print(" To change DB path, edit SUBSTRATE_DB in the hook or set env var.") print(" Start the query server:") - print(f" python3 {dest} serve --port {DEFAULT_PORT} --db {DB_PATH}") + print(f" python3 {script} serve --port {DEFAULT_PORT} --db {DB_PATH}") -def cmd_install_service(port: int = DEFAULT_PORT) -> None: +def cmd_install_service(port: int = DEFAULT_PORT, source: str = WITNESS_SOURCE) -> None: """Write a systemd unit file for the HTTP query server. Writes to /etc/systemd/system/substrate-git-index.service @@ -1518,6 +1590,8 @@ def cmd_install_service(port: int = DEFAULT_PORT) -> None: script=Path(__file__).resolve(), port=port, db=DB_PATH, + source=source, + config=_witness_config_path(), ) dest = Path("/etc/systemd/system/substrate-git-index.service") try: @@ -1534,17 +1608,24 @@ def cmd_install_service(port: int = DEFAULT_PORT) -> None: # CLI dispatch # ───────────────────────────────────────────────────────────────────────────── -def _parse_kv_args(args: list[str]) -> dict: - """Parse --key value pairs from an arg list into a dict.""" +def _parse_kv_args(args: list[str]) -> tuple[dict[str, str], list[str]]: + """Parse ``--key value`` pairs and preserve true positional arguments.""" d: dict = {} + positional: list[str] = [] i = 0 while i < len(args): - if args[i].startswith("--") and i + 1 < len(args): - d[args[i][2:]] = args[i + 1] - i += 2 + if args[i].startswith("--"): + key = args[i][2:] + if i + 1 < len(args) and not args[i + 1].startswith("--"): + d[key] = args[i + 1] + i += 2 + else: + d[key] = "true" + i += 1 else: + positional.append(args[i]) i += 1 - return d + return d, positional def main() -> None: @@ -1556,8 +1637,7 @@ def main() -> None: return cmd = args[0] - opts = _parse_kv_args(args[1:]) - positional = [a for a in args[1:] if not a.startswith("--")] + opts, positional = _parse_kv_args(args[1:]) # Apply --db and --port overrides globally global DB_PATH, DEFAULT_PORT @@ -1619,8 +1699,14 @@ def main() -> None: cmd_ingest_session(positional[0]) elif cmd == "install": - repo = opts.get("repo", positional[0] if positional else FORGEJO_REPO) - cmd_install(repo) + repo = opts.get("repo") or (positional[0] if positional else None) + cmd_install(repo, source=opts.get("source", WITNESS_SOURCE)) + + elif cmd == "install-service": + cmd_install_service( + int(opts.get("port", str(DEFAULT_PORT))), + source=opts.get("source", WITNESS_SOURCE), + ) elif cmd == "align": """Retroactively professionalize all nodes in the database.""" @@ -1652,9 +1738,6 @@ def main() -> None: conn.close() print(f"[align] professionalized {updates}/{len(rows)} nodes in {DB_PATH}") - elif cmd == "install-service": - cmd_install_service(DEFAULT_PORT) - else: print( f"Unknown command: {cmd!r}\n" diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e0587e2c..749eadff 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# ARCHITECTURE.md — Sovereign Research Stack +# ARCHITECTURE.md — Observerless Research Stack **The single-page map of the system. Read this first.** @@ -6,7 +6,7 @@ ## 1. Core Idea -The Sovereign Research Stack is a formal verification and compression system grounded in cross-domain invariant analysis. Every transformation — whether a byte-compression pass, a PDE discretization step, or a spiking-neuron firing — is treated as a receipt-bearing event that must preserve declared invariants. Lean 4 is the source of truth. All computation uses Q16.16 fixed-point arithmetic. No floating-point, no unproven claims, no receipt-free promotions. +The Observerless Research Stack is a formal verification and compression system grounded in cross-domain invariant analysis. Every transformation — whether a byte-compression pass, a PDE discretization step, or a spiking-neuron firing — is treated as a receipt-bearing event that must preserve declared invariants. Lean 4 is the source of truth. All computation uses Q16.16 fixed-point arithmetic. No floating-point, no unproven claims, no receipt-free promotions. --- @@ -198,4 +198,4 @@ valid syntax --- -*Sovereign Research Stack — Architecture v1.0 (2026-05-04)* +*Observerless Research Stack — Architecture v1.0 (2026-05-04)* diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index 1b0ae73a..28329a21 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -1,4 +1,4 @@ -# Getting Started — Sovereign Research Stack +# Getting Started — Observerless Research Stack Start here. Every command on this page is copy-pasteable and verified against the repo.