mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
cleanup(ene): make provenance node/tailscale configurable via env vars and CLI arg
This commit is contained in:
Allaun Silverfox
parent
1181849cea
commit
e36cff917b
1 changed files with 191 additions and 263 deletions
|
|
@ -1,41 +1,38 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive Text Container to JSON-L Converter
|
||||
"""Comprehensive Text Container → JSON-L Converter (legacy shim).
|
||||
|
||||
Converts all information-bearing text containers (MD, JSON, CSV, TSV, TXT) to
|
||||
JSON-L format compatible with UNIFIED_JSONL_SCHEMA.md.
|
||||
|
||||
Handles:
|
||||
- .md files (Markdown documents)
|
||||
- .json files (JSON structures)
|
||||
- .jsonl files (already JSON-L, wrap as documents)
|
||||
- .csv files (tabular data)
|
||||
- .tsv files (tabular data)
|
||||
- .txt files (plain text documents)
|
||||
|
||||
Excludes:
|
||||
- Tool/library files (node_modules, .lake, tools/search, .git)
|
||||
- Config files (setup.json, package.json, etc.)
|
||||
- vendored dependencies
|
||||
Hardcoded node/provenance assumptions were removed:
|
||||
- node id is configurable via --node-id or ENE_NODE_ID
|
||||
- tailscale ip via ENE_TAILSCALE_IP
|
||||
- workspace root via ENE_WORKSPACE_ROOT
|
||||
|
||||
This script is an ingest shim. Treat outputs as non-authoritative.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import csv
|
||||
import os
|
||||
import sys
|
||||
import hashlib
|
||||
import time
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Configuration
|
||||
WORKSPACE_ROOT = Path("/home/allaun/Documents/Research Stack")
|
||||
|
||||
def env_default(name: str, default: str) -> str:
|
||||
try:
|
||||
import os
|
||||
|
||||
v = os.environ.get(name)
|
||||
except Exception:
|
||||
v = None
|
||||
return v if v is not None and v != "" else default
|
||||
|
||||
|
||||
WORKSPACE_ROOT = Path(env_default("ENE_WORKSPACE_ROOT", "/home/allaun/Documents/Research Stack"))
|
||||
MANIFEST_PATH = WORKSPACE_ROOT / "data" / "manifest.jsonl"
|
||||
|
||||
# Directories to skip
|
||||
EXCLUDE_PATTERNS = [
|
||||
".git",
|
||||
"node_modules",
|
||||
|
|
@ -50,10 +47,8 @@ EXCLUDE_PATTERNS = [
|
|||
"venv_",
|
||||
]
|
||||
|
||||
# File types to process
|
||||
PROCESS_EXTENSIONS = {".md", ".json", ".jsonl", ".csv", ".tsv", ".txt"}
|
||||
|
||||
# Archetype mapping
|
||||
ARCHETYPE_MAP = {
|
||||
".md": "markdown_document",
|
||||
".json": "json_structure",
|
||||
|
|
@ -66,7 +61,6 @@ ARCHETYPE_MAP = {
|
|||
|
||||
@dataclass
|
||||
class FileContainer:
|
||||
"""Represents a text-based information container."""
|
||||
filepath: Path
|
||||
ext: str
|
||||
size: int
|
||||
|
|
@ -75,50 +69,53 @@ class FileContainer:
|
|||
|
||||
|
||||
def should_skip(filepath: Path) -> Tuple[bool, str]:
|
||||
"""Check if file should be skipped."""
|
||||
path_str = str(filepath)
|
||||
|
||||
# Skip 4-Infrastructure/config/metadata files
|
||||
|
||||
config_files = {
|
||||
"package.json", "package-lock.json", "tsconfig.json", "devcontainer.json",
|
||||
"settings.json", "launch.json", "tasks.json", ".stylelintrc.json",
|
||||
"biome.json", "lake-manifest.json", "pyrightconfig.json",
|
||||
"manifest.json", "requirements.txt", "app.json", "acme.json",
|
||||
".vscode", ".devcontainer"
|
||||
"package.json",
|
||||
"package-lock.json",
|
||||
"tsconfig.json",
|
||||
"devcontainer.json",
|
||||
"settings.json",
|
||||
"launch.json",
|
||||
"tasks.json",
|
||||
".stylelintrc.json",
|
||||
"biome.json",
|
||||
"lake-manifest.json",
|
||||
"pyrightconfig.json",
|
||||
"manifest.json",
|
||||
"requirements.txt",
|
||||
"app.json",
|
||||
"acme.json",
|
||||
".vscode",
|
||||
".devcontainer",
|
||||
}
|
||||
|
||||
|
||||
if filepath.name in config_files:
|
||||
return True, "config"
|
||||
|
||||
# Skip excluded paths
|
||||
|
||||
for pattern in EXCLUDE_PATTERNS:
|
||||
if pattern in path_str:
|
||||
return True, f"excluded:{pattern}"
|
||||
|
||||
# Skip large binary-like files
|
||||
if filepath.stat().st_size > 100_000_000: # > 100MB
|
||||
|
||||
if filepath.stat().st_size > 100_000_000:
|
||||
return True, "too_large"
|
||||
|
||||
|
||||
return False, ""
|
||||
|
||||
|
||||
def compute_sha256(filepath: Path) -> str:
|
||||
"""Compute SHA256 hash of file."""
|
||||
sha256 = hashlib.sha256()
|
||||
try:
|
||||
with open(filepath, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
sha256.update(chunk)
|
||||
return f"sha256:{sha256.hexdigest()}"
|
||||
except Exception as e:
|
||||
return f"sha256:error:{str(e)}"
|
||||
with open(filepath, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
sha256.update(chunk)
|
||||
return f"sha256:{sha256.hexdigest()}"
|
||||
|
||||
|
||||
def infer_domain(filepath: Path, content: str = "") -> str:
|
||||
"""Infer domain from filename and path."""
|
||||
name = filepath.stem.upper()
|
||||
path = str(filepath).upper()
|
||||
|
||||
|
||||
domain_hints = {
|
||||
"LEAN|SEMANTIC|FORMALIZATION": "formalization",
|
||||
"MATH|EQUATION|MODEL": "mathematics",
|
||||
|
|
@ -132,38 +129,34 @@ def infer_domain(filepath: Path, content: str = "") -> str:
|
|||
"DATA|DATASET|CSV|TSV": "data_science",
|
||||
"CONFIG|SETTING": "infrastructure",
|
||||
}
|
||||
|
||||
|
||||
for patterns, domain in domain_hints.items():
|
||||
if any(p in name or p in path for p in patterns.split("|")):
|
||||
return domain
|
||||
|
||||
# Default based on extension
|
||||
if filepath.suffix == ".csv" or filepath.suffix == ".tsv":
|
||||
|
||||
if filepath.suffix in {".csv", ".tsv"}:
|
||||
return "data_science"
|
||||
elif filepath.suffix == ".json":
|
||||
if filepath.suffix == ".json":
|
||||
return "specification"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def infer_tier(filepath: Path) -> str:
|
||||
"""Infer tier from path."""
|
||||
path = str(filepath).lower()
|
||||
if "6-Documentation/docs/semantics" in path or "docs" in path:
|
||||
return "CORE"
|
||||
elif "shared-data/data/germane" in path:
|
||||
if "shared-data/data/germane" in path:
|
||||
return "AUX"
|
||||
elif "out" in path:
|
||||
if "out" in path:
|
||||
return "DERIVED"
|
||||
return "AUX"
|
||||
|
||||
|
||||
def read_text_safely(filepath: Path, max_size: int = 1_000_000) -> str:
|
||||
"""Read text file safely with encoding fallback."""
|
||||
encodings = ["utf-8", "utf-8-sig", "latin-1", "ascii", "cp1252"]
|
||||
|
||||
|
||||
size = filepath.stat().st_size
|
||||
if size > max_size:
|
||||
# Read first and last chunks
|
||||
with open(filepath, "rb") as f:
|
||||
start = f.read(500)
|
||||
f.seek(max(0, size - 500))
|
||||
|
|
@ -172,275 +165,210 @@ def read_text_safely(filepath: Path, max_size: int = 1_000_000) -> str:
|
|||
else:
|
||||
with open(filepath, "rb") as f:
|
||||
content_bytes = f.read()
|
||||
|
||||
|
||||
for encoding in encodings:
|
||||
try:
|
||||
return content_bytes.decode(encoding, errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
|
||||
return "<unable to decode>"
|
||||
|
||||
|
||||
def extract_summary(content: str, max_chars: int = 250) -> str:
|
||||
"""Extract summary from content."""
|
||||
lines = content.split("\n")
|
||||
summary_lines = []
|
||||
|
||||
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped and not stripped.startswith("#") and not stripped.startswith("{"):
|
||||
summary_lines.append(stripped)
|
||||
if len(" ".join(summary_lines)) >= max_chars:
|
||||
break
|
||||
|
||||
|
||||
summary = " ".join(summary_lines)[:max_chars]
|
||||
return summary or "<empty or binary file>"
|
||||
|
||||
|
||||
def csv_to_dict_list(filepath: Path, limit_rows: int = 100) -> List[Dict]:
|
||||
"""Read CSV file into list of dicts."""
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
|
||||
reader = csv.DictReader(f)
|
||||
return list(islice(reader, limit_rows))
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def compute_genome(filepath: Path, content: str = "") -> Dict[str, int]:
|
||||
"""Compute 6D genome signature."""
|
||||
try:
|
||||
size = filepath.stat().st_size
|
||||
lines = len(content.split("\n")) if content else 10
|
||||
|
||||
|
||||
mu = (lines % 256) // 32
|
||||
rho = min(7, (size // 1024) % 8)
|
||||
c = 4
|
||||
m = 4
|
||||
ne = min(7, len(filepath.stem) // 10)
|
||||
sig = 0 if filepath.suffix != ".json" else 3
|
||||
|
||||
|
||||
return {"mu": mu, "rho": rho, "c": c, "m": m, "ne": ne, "sig": sig}
|
||||
except Exception:
|
||||
return {"mu": 0, "rho": 0, "c": 4, "m": 4, "ne": 0, "sig": 0}
|
||||
|
||||
|
||||
def text_to_jsonl_entry(filepath: Path, node_id: str = "qfox") -> Optional[Dict[str, Any]]:
|
||||
"""Convert a text container to JSON-L entry."""
|
||||
|
||||
try:
|
||||
should_skip_file, reason = should_skip(filepath)
|
||||
if should_skip_file:
|
||||
return None
|
||||
|
||||
# Read content
|
||||
content = read_text_safely(filepath)
|
||||
file_hash = compute_sha256(filepath)
|
||||
file_stat = filepath.stat()
|
||||
mtime_unix = file_stat.st_mtime
|
||||
file_size = file_stat.st_size
|
||||
|
||||
# Compute metadata
|
||||
domain = infer_domain(filepath, content)
|
||||
tier = infer_tier(filepath)
|
||||
genome = compute_genome(filepath, content)
|
||||
|
||||
# Create pkg identifier
|
||||
rel_path = filepath.relative_to(WORKSPACE_ROOT)
|
||||
pkg = f"ene/text/{filepath.suffix[1:]}/{rel_path.stem}".replace("\\", "/")
|
||||
version = datetime.fromtimestamp(mtime_unix, tz=timezone.utc).isoformat()
|
||||
|
||||
concept_anchor = {
|
||||
"domain": domain,
|
||||
"concept": filepath.stem.lower().replace(" ", "_").replace("-", "_").replace(".", "_"),
|
||||
"resolution": "STABLE"
|
||||
}
|
||||
|
||||
# Extract data payload based on file type
|
||||
summary = extract_summary(content)
|
||||
|
||||
data_payload = {
|
||||
"pkg": pkg,
|
||||
"version": version,
|
||||
"tier": tier,
|
||||
"domain": domain,
|
||||
"archetype": ARCHETYPE_MAP.get(filepath.suffix, "text_document"),
|
||||
"concept_anchor": concept_anchor,
|
||||
"file_path": str(rel_path).replace("\\", "/"),
|
||||
"file_ext": filepath.suffix,
|
||||
"file_hash": file_hash,
|
||||
"byte_count": file_size,
|
||||
"line_count": len(content.split("\n")),
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
# Add format-specific metadata
|
||||
if filepath.suffix == ".json":
|
||||
try:
|
||||
obj = json.loads(content)
|
||||
data_payload["json_keys"] = list(obj.keys() if isinstance(obj, dict) else [])
|
||||
except Exception:
|
||||
data_payload["json_keys"] = []
|
||||
|
||||
elif filepath.suffix in {".csv", ".tsv"}:
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
|
||||
reader = csv.DictReader(f)
|
||||
first_row = next(reader, None)
|
||||
if first_row:
|
||||
data_payload["columns"] = list(first_row.keys())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Provenance
|
||||
provenance = {
|
||||
"node": node_id,
|
||||
"lake_seed": "text_converter",
|
||||
"tailscale_ip": "127.0.0.1",
|
||||
"attestation_hash": file_hash,
|
||||
"prev_id": None
|
||||
}
|
||||
|
||||
# Bind
|
||||
bind = {
|
||||
"lawful": True,
|
||||
"cost": 0x00010000,
|
||||
"invariant": "documentConsistency",
|
||||
"class": "informational_bind"
|
||||
}
|
||||
|
||||
# Full JSON-L entry
|
||||
entry = {
|
||||
"t": mtime_unix,
|
||||
"src": "ene",
|
||||
"id": f"ene:{pkg}:{version}",
|
||||
"op": "upsert",
|
||||
"data": data_payload,
|
||||
"genome": genome,
|
||||
"bind": bind,
|
||||
"provenance": provenance
|
||||
}
|
||||
|
||||
return entry
|
||||
|
||||
except Exception as e:
|
||||
print(f" ⚠️ Error processing {filepath}: {e}", file=sys.stderr)
|
||||
def text_to_jsonl_entry(filepath: Path, node_id: str) -> Optional[Dict[str, Any]]:
|
||||
should_skip_file, _reason = should_skip(filepath)
|
||||
if should_skip_file:
|
||||
return None
|
||||
|
||||
content = read_text_safely(filepath)
|
||||
file_hash = compute_sha256(filepath)
|
||||
file_stat = filepath.stat()
|
||||
mtime_unix = file_stat.st_mtime
|
||||
file_size = file_stat.st_size
|
||||
|
||||
domain = infer_domain(filepath, content)
|
||||
tier = infer_tier(filepath)
|
||||
genome = compute_genome(filepath, content)
|
||||
|
||||
rel_path = filepath.relative_to(WORKSPACE_ROOT)
|
||||
pkg = f"ene/text/{filepath.suffix[1:]}/{rel_path.stem}".replace("\\", "/")
|
||||
version = datetime.fromtimestamp(mtime_unix, tz=timezone.utc).isoformat()
|
||||
|
||||
concept_anchor = {
|
||||
"domain": domain,
|
||||
"concept": filepath.stem.lower().replace(" ", "_").replace("-", "_").replace(".", "_"),
|
||||
"resolution": "STABLE",
|
||||
}
|
||||
|
||||
summary = extract_summary(content)
|
||||
|
||||
data_payload: Dict[str, Any] = {
|
||||
"pkg": pkg,
|
||||
"version": version,
|
||||
"tier": tier,
|
||||
"domain": domain,
|
||||
"archetype": ARCHETYPE_MAP.get(filepath.suffix, "text_document"),
|
||||
"concept_anchor": concept_anchor,
|
||||
"file_path": str(rel_path).replace("\\", "/"),
|
||||
"file_ext": filepath.suffix,
|
||||
"file_hash": file_hash,
|
||||
"byte_count": file_size,
|
||||
"line_count": len(content.split("\n")),
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
if filepath.suffix == ".json":
|
||||
try:
|
||||
obj = json.loads(content)
|
||||
data_payload["json_keys"] = list(obj.keys() if isinstance(obj, dict) else [])
|
||||
except Exception:
|
||||
data_payload["json_keys"] = []
|
||||
|
||||
if filepath.suffix in {".csv", ".tsv"}:
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
|
||||
reader = csv.DictReader(f)
|
||||
first_row = next(reader, None)
|
||||
if first_row:
|
||||
data_payload["columns"] = list(first_row.keys())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
provenance = {
|
||||
"node": node_id,
|
||||
"lake_seed": env_default("ENE_LAKE_SEED", "text_converter"),
|
||||
"tailscale_ip": env_default("ENE_TAILSCALE_IP", "127.0.0.1"),
|
||||
"attestation_hash": file_hash,
|
||||
"prev_id": None,
|
||||
}
|
||||
|
||||
bind = {
|
||||
"lawful": True,
|
||||
"cost": 0x00010000,
|
||||
"invariant": "documentConsistency",
|
||||
"class": "informational_bind",
|
||||
}
|
||||
|
||||
return {
|
||||
"t": mtime_unix,
|
||||
"src": "ene",
|
||||
"id": f"ene:{pkg}:{version}",
|
||||
"op": "upsert",
|
||||
"data": data_payload,
|
||||
"genome": genome,
|
||||
"bind": bind,
|
||||
"provenance": provenance,
|
||||
}
|
||||
|
||||
|
||||
def find_all_text_containers() -> List[Path]:
|
||||
"""Find all text containers in workspace."""
|
||||
containers = []
|
||||
|
||||
for ext in PROCESS_EXTENSIONS:
|
||||
for file_path in WORKSPACE_ROOT.rglob(f"*{ext}"):
|
||||
if file_path.is_file():
|
||||
containers.append(file_path)
|
||||
|
||||
return sorted(list(set(containers)))
|
||||
|
||||
|
||||
def load_existing_manifest() -> set:
|
||||
"""Load IDs from existing manifest."""
|
||||
existing_ids = set()
|
||||
if MANIFEST_PATH.exists():
|
||||
try:
|
||||
with open(MANIFEST_PATH, "r") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
existing_ids.add(entry.get("id", ""))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read existing manifest: {e}")
|
||||
with open(MANIFEST_PATH, "r") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
existing_ids.add(entry.get("id", ""))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return existing_ids
|
||||
|
||||
|
||||
def convert_all_text_containers(output_file: Optional[str] = None) -> Tuple[int, int, int, int]:
|
||||
"""
|
||||
Convert all text containers to JSON-L.
|
||||
|
||||
Returns: (total_files, newly_converted, already_exists, skipped)
|
||||
"""
|
||||
from itertools import islice
|
||||
|
||||
def convert_all_text_containers(node_id: str, output_file: Optional[str] = None) -> Tuple[int, int, int, int]:
|
||||
output_path = Path(output_file) if output_file else MANIFEST_PATH
|
||||
|
||||
print(f"🔍 Scanning for text containers...")
|
||||
|
||||
containers = find_all_text_containers()
|
||||
print(f"✅ Found {len(containers)} text containers")
|
||||
|
||||
existing_ids = load_existing_manifest()
|
||||
print(f"📋 Manifest already has {len(existing_ids)} entries")
|
||||
print()
|
||||
|
||||
|
||||
converted = 0
|
||||
skipped = 0
|
||||
already_exists = 0
|
||||
|
||||
ext_stats = {}
|
||||
|
||||
|
||||
with open(output_path, "a") as manifest_f:
|
||||
for i, container in enumerate(containers, 1):
|
||||
ext = container.suffix
|
||||
ext_stats[ext] = ext_stats.get(ext, 0) + 1
|
||||
|
||||
entry = text_to_jsonl_entry(container)
|
||||
|
||||
for container in containers:
|
||||
entry = text_to_jsonl_entry(container, node_id=node_id)
|
||||
if entry is None:
|
||||
skipped += 1
|
||||
status = "⏭️ SKIP"
|
||||
continue
|
||||
|
||||
entry_id = entry.get("id", "")
|
||||
if entry_id in existing_ids:
|
||||
already_exists += 1
|
||||
else:
|
||||
entry_id = entry.get("id", "")
|
||||
if entry_id in existing_ids:
|
||||
already_exists += 1
|
||||
status = "⏪ DUP"
|
||||
else:
|
||||
manifest_f.write(json.dumps(entry) + "\n")
|
||||
manifest_f.flush()
|
||||
converted += 1
|
||||
status = "✅ CONV"
|
||||
|
||||
rel_path = container.relative_to(WORKSPACE_ROOT)
|
||||
|
||||
# Less verbose output
|
||||
if i % 10 == 0 or status == "✅ CONV":
|
||||
print(f"[{i:4d}/{len(containers)}] {status} {rel_path}")
|
||||
|
||||
manifest_f.write(json.dumps(entry) + "\n")
|
||||
manifest_f.flush()
|
||||
converted += 1
|
||||
|
||||
return len(containers), converted, already_exists, skipped
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
output_file = None
|
||||
if len(sys.argv) > 2 and sys.argv[1] == "--output-file":
|
||||
output_file = sys.argv[2]
|
||||
|
||||
print("=" * 75)
|
||||
print("🌐 Comprehensive Text Container to JSON-L Converter")
|
||||
print(f"Workspace: {WORKSPACE_ROOT}")
|
||||
print(f"Output: {output_file or MANIFEST_PATH}")
|
||||
print("=" * 75)
|
||||
print()
|
||||
|
||||
total, converted, already_exists, skipped = convert_all_text_containers(output_file)
|
||||
|
||||
print()
|
||||
print("=" * 75)
|
||||
print(f"📊 Summary:")
|
||||
print(f" Total files scanned: {total}")
|
||||
print(f" Newly converted: {converted}")
|
||||
print(f" Already in manifest: {already_exists}")
|
||||
print(f" Skipped: {skipped}")
|
||||
print(f" Output file: {output_file or MANIFEST_PATH}")
|
||||
print("=" * 75)
|
||||
|
||||
return 0 if converted > 0 else 1
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output-file", default=None)
|
||||
parser.add_argument("--node-id", default=env_default("ENE_NODE_ID", "qfox"))
|
||||
args = parser.parse_args()
|
||||
|
||||
total, converted, already_exists, skipped = convert_all_text_containers(
|
||||
node_id=args.node_id, output_file=args.output_file
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"total": total,
|
||||
"converted": converted,
|
||||
"already_exists": already_exists,
|
||||
"skipped": skipped,
|
||||
"node_id": args.node_id,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
raise SystemExit(main())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue