From 29f93f9dfd542ffed899cda51ffc4c3f5354701c Mon Sep 17 00:00:00 2001 From: Allaun Silverfox <28494262+allaunthefox@users.noreply.github.com> Date: Tue, 26 May 2026 16:42:03 -0500 Subject: [PATCH] cleanup(ene): make provenance node/tailscale configurable via env vars and CLI arg --- .../scripts/text_container_to_jsonl.py | 454 ++++++++---------- 1 file changed, 191 insertions(+), 263 deletions(-) diff --git a/5-Applications/scripts/text_container_to_jsonl.py b/5-Applications/scripts/text_container_to_jsonl.py index efcc535e..f780fd5e 100644 --- a/5-Applications/scripts/text_container_to_jsonl.py +++ b/5-Applications/scripts/text_container_to_jsonl.py @@ -1,41 +1,38 @@ #!/usr/bin/env python3 -""" -Comprehensive Text Container to JSON-L Converter +"""Comprehensive Text Container → JSON-L Converter (legacy shim). -Converts all information-bearing text containers (MD, JSON, CSV, TSV, TXT) to -JSON-L format compatible with UNIFIED_JSONL_SCHEMA.md. - -Handles: - - .md files (Markdown documents) - - .json files (JSON structures) - - .jsonl files (already JSON-L, wrap as documents) - - .csv files (tabular data) - - .tsv files (tabular data) - - .txt files (plain text documents) - -Excludes: - - Tool/library files (node_modules, .lake, tools/search, .git) - - Config files (setup.json, package.json, etc.) - - vendored dependencies +Hardcoded node/provenance assumptions were removed: +- node id is configurable via --node-id or ENE_NODE_ID +- tailscale ip via ENE_TAILSCALE_IP +- workspace root via ENE_WORKSPACE_ROOT +This script is an ingest shim. Treat outputs as non-authoritative. """ +import argparse import json import csv -import os import sys import hashlib -import time from pathlib import Path from datetime import datetime, timezone from typing import Dict, Any, List, Optional, Tuple from dataclasses import dataclass -# Configuration -WORKSPACE_ROOT = Path("/home/allaun/Documents/Research Stack") + +def env_default(name: str, default: str) -> str: + try: + import os + + v = os.environ.get(name) + except Exception: + v = None + return v if v is not None and v != "" else default + + +WORKSPACE_ROOT = Path(env_default("ENE_WORKSPACE_ROOT", "/home/allaun/Documents/Research Stack")) MANIFEST_PATH = WORKSPACE_ROOT / "data" / "manifest.jsonl" -# Directories to skip EXCLUDE_PATTERNS = [ ".git", "node_modules", @@ -50,10 +47,8 @@ EXCLUDE_PATTERNS = [ "venv_", ] -# File types to process PROCESS_EXTENSIONS = {".md", ".json", ".jsonl", ".csv", ".tsv", ".txt"} -# Archetype mapping ARCHETYPE_MAP = { ".md": "markdown_document", ".json": "json_structure", @@ -66,7 +61,6 @@ ARCHETYPE_MAP = { @dataclass class FileContainer: - """Represents a text-based information container.""" filepath: Path ext: str size: int @@ -75,50 +69,53 @@ class FileContainer: def should_skip(filepath: Path) -> Tuple[bool, str]: - """Check if file should be skipped.""" path_str = str(filepath) - - # Skip 4-Infrastructure/config/metadata files + config_files = { - "package.json", "package-lock.json", "tsconfig.json", "devcontainer.json", - "settings.json", "launch.json", "tasks.json", ".stylelintrc.json", - "biome.json", "lake-manifest.json", "pyrightconfig.json", - "manifest.json", "requirements.txt", "app.json", "acme.json", - ".vscode", ".devcontainer" + "package.json", + "package-lock.json", + "tsconfig.json", + "devcontainer.json", + "settings.json", + "launch.json", + "tasks.json", + ".stylelintrc.json", + "biome.json", + "lake-manifest.json", + "pyrightconfig.json", + "manifest.json", + "requirements.txt", + "app.json", + "acme.json", + ".vscode", + ".devcontainer", } - + if filepath.name in config_files: return True, "config" - - # Skip excluded paths + for pattern in EXCLUDE_PATTERNS: if pattern in path_str: return True, f"excluded:{pattern}" - - # Skip large binary-like files - if filepath.stat().st_size > 100_000_000: # > 100MB + + if filepath.stat().st_size > 100_000_000: return True, "too_large" - + return False, "" def compute_sha256(filepath: Path) -> str: - """Compute SHA256 hash of file.""" sha256 = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for chunk in iter(lambda: f.read(8192), b""): - sha256.update(chunk) - return f"sha256:{sha256.hexdigest()}" - except Exception as e: - return f"sha256:error:{str(e)}" + with open(filepath, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + sha256.update(chunk) + return f"sha256:{sha256.hexdigest()}" def infer_domain(filepath: Path, content: str = "") -> str: - """Infer domain from filename and path.""" name = filepath.stem.upper() path = str(filepath).upper() - + domain_hints = { "LEAN|SEMANTIC|FORMALIZATION": "formalization", "MATH|EQUATION|MODEL": "mathematics", @@ -132,38 +129,34 @@ def infer_domain(filepath: Path, content: str = "") -> str: "DATA|DATASET|CSV|TSV": "data_science", "CONFIG|SETTING": "infrastructure", } - + for patterns, domain in domain_hints.items(): if any(p in name or p in path for p in patterns.split("|")): return domain - - # Default based on extension - if filepath.suffix == ".csv" or filepath.suffix == ".tsv": + + if filepath.suffix in {".csv", ".tsv"}: return "data_science" - elif filepath.suffix == ".json": + if filepath.suffix == ".json": return "specification" return "unknown" def infer_tier(filepath: Path) -> str: - """Infer tier from path.""" path = str(filepath).lower() if "6-Documentation/docs/semantics" in path or "docs" in path: return "CORE" - elif "shared-data/data/germane" in path: + if "shared-data/data/germane" in path: return "AUX" - elif "out" in path: + if "out" in path: return "DERIVED" return "AUX" def read_text_safely(filepath: Path, max_size: int = 1_000_000) -> str: - """Read text file safely with encoding fallback.""" encodings = ["utf-8", "utf-8-sig", "latin-1", "ascii", "cp1252"] - + size = filepath.stat().st_size if size > max_size: - # Read first and last chunks with open(filepath, "rb") as f: start = f.read(500) f.seek(max(0, size - 500)) @@ -172,275 +165,210 @@ def read_text_safely(filepath: Path, max_size: int = 1_000_000) -> str: else: with open(filepath, "rb") as f: content_bytes = f.read() - + for encoding in encodings: try: return content_bytes.decode(encoding, errors="replace") except Exception: continue - + return "" def extract_summary(content: str, max_chars: int = 250) -> str: - """Extract summary from content.""" lines = content.split("\n") summary_lines = [] - + for line in lines: stripped = line.strip() if stripped and not stripped.startswith("#") and not stripped.startswith("{"): summary_lines.append(stripped) if len(" ".join(summary_lines)) >= max_chars: break - + summary = " ".join(summary_lines)[:max_chars] return summary or "" -def csv_to_dict_list(filepath: Path, limit_rows: int = 100) -> List[Dict]: - """Read CSV file into list of dicts.""" - try: - with open(filepath, "r", encoding="utf-8", errors="replace") as f: - reader = csv.DictReader(f) - return list(islice(reader, limit_rows)) - except Exception: - return [] - - def compute_genome(filepath: Path, content: str = "") -> Dict[str, int]: - """Compute 6D genome signature.""" try: size = filepath.stat().st_size lines = len(content.split("\n")) if content else 10 - + mu = (lines % 256) // 32 rho = min(7, (size // 1024) % 8) c = 4 m = 4 ne = min(7, len(filepath.stem) // 10) sig = 0 if filepath.suffix != ".json" else 3 - + return {"mu": mu, "rho": rho, "c": c, "m": m, "ne": ne, "sig": sig} except Exception: return {"mu": 0, "rho": 0, "c": 4, "m": 4, "ne": 0, "sig": 0} -def text_to_jsonl_entry(filepath: Path, node_id: str = "qfox") -> Optional[Dict[str, Any]]: - """Convert a text container to JSON-L entry.""" - - try: - should_skip_file, reason = should_skip(filepath) - if should_skip_file: - return None - - # Read content - content = read_text_safely(filepath) - file_hash = compute_sha256(filepath) - file_stat = filepath.stat() - mtime_unix = file_stat.st_mtime - file_size = file_stat.st_size - - # Compute metadata - domain = infer_domain(filepath, content) - tier = infer_tier(filepath) - genome = compute_genome(filepath, content) - - # Create pkg identifier - rel_path = filepath.relative_to(WORKSPACE_ROOT) - pkg = f"ene/text/{filepath.suffix[1:]}/{rel_path.stem}".replace("\\", "/") - version = datetime.fromtimestamp(mtime_unix, tz=timezone.utc).isoformat() - - concept_anchor = { - "domain": domain, - "concept": filepath.stem.lower().replace(" ", "_").replace("-", "_").replace(".", "_"), - "resolution": "STABLE" - } - - # Extract data payload based on file type - summary = extract_summary(content) - - data_payload = { - "pkg": pkg, - "version": version, - "tier": tier, - "domain": domain, - "archetype": ARCHETYPE_MAP.get(filepath.suffix, "text_document"), - "concept_anchor": concept_anchor, - "file_path": str(rel_path).replace("\\", "/"), - "file_ext": filepath.suffix, - "file_hash": file_hash, - "byte_count": file_size, - "line_count": len(content.split("\n")), - "summary": summary, - } - - # Add format-specific metadata - if filepath.suffix == ".json": - try: - obj = json.loads(content) - data_payload["json_keys"] = list(obj.keys() if isinstance(obj, dict) else []) - except Exception: - data_payload["json_keys"] = [] - - elif filepath.suffix in {".csv", ".tsv"}: - try: - with open(filepath, "r", encoding="utf-8", errors="replace") as f: - reader = csv.DictReader(f) - first_row = next(reader, None) - if first_row: - data_payload["columns"] = list(first_row.keys()) - except Exception: - pass - - # Provenance - provenance = { - "node": node_id, - "lake_seed": "text_converter", - "tailscale_ip": "127.0.0.1", - "attestation_hash": file_hash, - "prev_id": None - } - - # Bind - bind = { - "lawful": True, - "cost": 0x00010000, - "invariant": "documentConsistency", - "class": "informational_bind" - } - - # Full JSON-L entry - entry = { - "t": mtime_unix, - "src": "ene", - "id": f"ene:{pkg}:{version}", - "op": "upsert", - "data": data_payload, - "genome": genome, - "bind": bind, - "provenance": provenance - } - - return entry - - except Exception as e: - print(f" ⚠️ Error processing {filepath}: {e}", file=sys.stderr) +def text_to_jsonl_entry(filepath: Path, node_id: str) -> Optional[Dict[str, Any]]: + should_skip_file, _reason = should_skip(filepath) + if should_skip_file: return None + content = read_text_safely(filepath) + file_hash = compute_sha256(filepath) + file_stat = filepath.stat() + mtime_unix = file_stat.st_mtime + file_size = file_stat.st_size + + domain = infer_domain(filepath, content) + tier = infer_tier(filepath) + genome = compute_genome(filepath, content) + + rel_path = filepath.relative_to(WORKSPACE_ROOT) + pkg = f"ene/text/{filepath.suffix[1:]}/{rel_path.stem}".replace("\\", "/") + version = datetime.fromtimestamp(mtime_unix, tz=timezone.utc).isoformat() + + concept_anchor = { + "domain": domain, + "concept": filepath.stem.lower().replace(" ", "_").replace("-", "_").replace(".", "_"), + "resolution": "STABLE", + } + + summary = extract_summary(content) + + data_payload: Dict[str, Any] = { + "pkg": pkg, + "version": version, + "tier": tier, + "domain": domain, + "archetype": ARCHETYPE_MAP.get(filepath.suffix, "text_document"), + "concept_anchor": concept_anchor, + "file_path": str(rel_path).replace("\\", "/"), + "file_ext": filepath.suffix, + "file_hash": file_hash, + "byte_count": file_size, + "line_count": len(content.split("\n")), + "summary": summary, + } + + if filepath.suffix == ".json": + try: + obj = json.loads(content) + data_payload["json_keys"] = list(obj.keys() if isinstance(obj, dict) else []) + except Exception: + data_payload["json_keys"] = [] + + if filepath.suffix in {".csv", ".tsv"}: + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + reader = csv.DictReader(f) + first_row = next(reader, None) + if first_row: + data_payload["columns"] = list(first_row.keys()) + except Exception: + pass + + provenance = { + "node": node_id, + "lake_seed": env_default("ENE_LAKE_SEED", "text_converter"), + "tailscale_ip": env_default("ENE_TAILSCALE_IP", "127.0.0.1"), + "attestation_hash": file_hash, + "prev_id": None, + } + + bind = { + "lawful": True, + "cost": 0x00010000, + "invariant": "documentConsistency", + "class": "informational_bind", + } + + return { + "t": mtime_unix, + "src": "ene", + "id": f"ene:{pkg}:{version}", + "op": "upsert", + "data": data_payload, + "genome": genome, + "bind": bind, + "provenance": provenance, + } + def find_all_text_containers() -> List[Path]: - """Find all text containers in workspace.""" containers = [] - for ext in PROCESS_EXTENSIONS: for file_path in WORKSPACE_ROOT.rglob(f"*{ext}"): if file_path.is_file(): containers.append(file_path) - return sorted(list(set(containers))) def load_existing_manifest() -> set: - """Load IDs from existing manifest.""" existing_ids = set() if MANIFEST_PATH.exists(): - try: - with open(MANIFEST_PATH, "r") as f: - for line in f: - line = line.strip() - if line: - try: - entry = json.loads(line) - existing_ids.add(entry.get("id", "")) - except json.JSONDecodeError: - pass - except Exception as e: - print(f"Warning: Could not read existing manifest: {e}") + with open(MANIFEST_PATH, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + existing_ids.add(entry.get("id", "")) + except json.JSONDecodeError: + continue return existing_ids -def convert_all_text_containers(output_file: Optional[str] = None) -> Tuple[int, int, int, int]: - """ - Convert all text containers to JSON-L. - - Returns: (total_files, newly_converted, already_exists, skipped) - """ - from itertools import islice - +def convert_all_text_containers(node_id: str, output_file: Optional[str] = None) -> Tuple[int, int, int, int]: output_path = Path(output_file) if output_file else MANIFEST_PATH - - print(f"🔍 Scanning for text containers...") + containers = find_all_text_containers() - print(f"✅ Found {len(containers)} text containers") - existing_ids = load_existing_manifest() - print(f"📋 Manifest already has {len(existing_ids)} entries") - print() - + converted = 0 skipped = 0 already_exists = 0 - - ext_stats = {} - + with open(output_path, "a") as manifest_f: - for i, container in enumerate(containers, 1): - ext = container.suffix - ext_stats[ext] = ext_stats.get(ext, 0) + 1 - - entry = text_to_jsonl_entry(container) - + for container in containers: + entry = text_to_jsonl_entry(container, node_id=node_id) if entry is None: skipped += 1 - status = "⏭️ SKIP" + continue + + entry_id = entry.get("id", "") + if entry_id in existing_ids: + already_exists += 1 else: - entry_id = entry.get("id", "") - if entry_id in existing_ids: - already_exists += 1 - status = "⏪ DUP" - else: - manifest_f.write(json.dumps(entry) + "\n") - manifest_f.flush() - converted += 1 - status = "✅ CONV" - - rel_path = container.relative_to(WORKSPACE_ROOT) - - # Less verbose output - if i % 10 == 0 or status == "✅ CONV": - print(f"[{i:4d}/{len(containers)}] {status} {rel_path}") - + manifest_f.write(json.dumps(entry) + "\n") + manifest_f.flush() + converted += 1 + return len(containers), converted, already_exists, skipped -def main(): - """Main entry point.""" - output_file = None - if len(sys.argv) > 2 and sys.argv[1] == "--output-file": - output_file = sys.argv[2] - - print("=" * 75) - print("🌐 Comprehensive Text Container to JSON-L Converter") - print(f"Workspace: {WORKSPACE_ROOT}") - print(f"Output: {output_file or MANIFEST_PATH}") - print("=" * 75) - print() - - total, converted, already_exists, skipped = convert_all_text_containers(output_file) - - print() - print("=" * 75) - print(f"📊 Summary:") - print(f" Total files scanned: {total}") - print(f" Newly converted: {converted}") - print(f" Already in manifest: {already_exists}") - print(f" Skipped: {skipped}") - print(f" Output file: {output_file or MANIFEST_PATH}") - print("=" * 75) - - return 0 if converted > 0 else 1 +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output-file", default=None) + parser.add_argument("--node-id", default=env_default("ENE_NODE_ID", "qfox")) + args = parser.parse_args() + + total, converted, already_exists, skipped = convert_all_text_containers( + node_id=args.node_id, output_file=args.output_file + ) + print( + json.dumps( + { + "total": total, + "converted": converted, + "already_exists": already_exists, + "skipped": skipped, + "node_id": args.node_id, + }, + indent=2, + ) + ) + return 0 if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main())