mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
446 lines
14 KiB
Python
446 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Comprehensive Text Container to JSON-L Converter
|
|
|
|
Converts all information-bearing text containers (MD, JSON, CSV, TSV, TXT) to
|
|
JSON-L format compatible with UNIFIED_JSONL_SCHEMA.md.
|
|
|
|
Handles:
|
|
- .md files (Markdown documents)
|
|
- .json files (JSON structures)
|
|
- .jsonl files (already JSON-L, wrap as documents)
|
|
- .csv files (tabular data)
|
|
- .tsv files (tabular data)
|
|
- .txt files (plain text documents)
|
|
|
|
Excludes:
|
|
- Tool/library files (node_modules, .lake, tools/search, .git)
|
|
- Config files (setup.json, package.json, etc.)
|
|
- vendored dependencies
|
|
|
|
"""
|
|
|
|
import json
|
|
import csv
|
|
import os
|
|
import sys
|
|
import hashlib
|
|
import time
|
|
from pathlib import Path
|
|
from datetime import datetime, timezone
|
|
from typing import Dict, Any, List, Optional, Tuple
|
|
from dataclasses import dataclass
|
|
|
|
# Configuration
|
|
WORKSPACE_ROOT = Path("/home/allaun/Documents/Research Stack")
|
|
MANIFEST_PATH = WORKSPACE_ROOT / "data" / "manifest.jsonl"
|
|
|
|
# Directories to skip
|
|
EXCLUDE_PATTERNS = [
|
|
".git",
|
|
"node_modules",
|
|
".lake",
|
|
"tools/search",
|
|
".vscode",
|
|
".devcontainer",
|
|
"__pycache__",
|
|
".pytest_cache",
|
|
"venv",
|
|
"hutter_venv",
|
|
"venv_",
|
|
]
|
|
|
|
# File types to process
|
|
PROCESS_EXTENSIONS = {".md", ".json", ".jsonl", ".csv", ".tsv", ".txt"}
|
|
|
|
# Archetype mapping
|
|
ARCHETYPE_MAP = {
|
|
".md": "markdown_document",
|
|
".json": "json_structure",
|
|
".jsonl": "jsonl_dataset",
|
|
".csv": "csv_table",
|
|
".tsv": "tsv_table",
|
|
".txt": "text_document",
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class FileContainer:
|
|
"""Represents a text-based information container."""
|
|
filepath: Path
|
|
ext: str
|
|
size: int
|
|
mtime: float
|
|
hash: str
|
|
|
|
|
|
def should_skip(filepath: Path) -> Tuple[bool, str]:
|
|
"""Check if file should be skipped."""
|
|
path_str = str(filepath)
|
|
|
|
# Skip 4-Infrastructure/config/metadata files
|
|
config_files = {
|
|
"package.json", "package-lock.json", "tsconfig.json", "devcontainer.json",
|
|
"settings.json", "launch.json", "tasks.json", ".stylelintrc.json",
|
|
"biome.json", "lake-manifest.json", "pyrightconfig.json",
|
|
"manifest.json", "requirements.txt", "app.json", "acme.json",
|
|
".vscode", ".devcontainer"
|
|
}
|
|
|
|
if filepath.name in config_files:
|
|
return True, "config"
|
|
|
|
# Skip excluded paths
|
|
for pattern in EXCLUDE_PATTERNS:
|
|
if pattern in path_str:
|
|
return True, f"excluded:{pattern}"
|
|
|
|
# Skip large binary-like files
|
|
if filepath.stat().st_size > 100_000_000: # > 100MB
|
|
return True, "too_large"
|
|
|
|
return False, ""
|
|
|
|
|
|
def compute_sha256(filepath: Path) -> str:
|
|
"""Compute SHA256 hash of file."""
|
|
sha256 = hashlib.sha256()
|
|
try:
|
|
with open(filepath, "rb") as f:
|
|
for chunk in iter(lambda: f.read(8192), b""):
|
|
sha256.update(chunk)
|
|
return f"sha256:{sha256.hexdigest()}"
|
|
except Exception as e:
|
|
return f"sha256:error:{str(e)}"
|
|
|
|
|
|
def infer_domain(filepath: Path, content: str = "") -> str:
|
|
"""Infer domain from filename and path."""
|
|
name = filepath.stem.upper()
|
|
path = str(filepath).upper()
|
|
|
|
domain_hints = {
|
|
"LEAN|SEMANTIC|FORMALIZATION": "formalization",
|
|
"MATH|EQUATION|MODEL": "mathematics",
|
|
"ENE|SUBSTRATE": "ene",
|
|
"HUTTER|COMPRESS|KOLMOGOROV": "compression",
|
|
"PHYSICS|FIELD|QUANTUM": "physics",
|
|
"GENOME|RGFLOW|SWARM|TOPOLOGY": "orchestration",
|
|
"NOTION|LINEAR|ISSUE|TASK": "project_management",
|
|
"BRAIN|MANIFOLD": "topology",
|
|
"SPEC|SPECIFICATION|SCHEMA": "specification",
|
|
"DATA|DATASET|CSV|TSV": "data_science",
|
|
"CONFIG|SETTING": "infrastructure",
|
|
}
|
|
|
|
for patterns, domain in domain_hints.items():
|
|
if any(p in name or p in path for p in patterns.split("|")):
|
|
return domain
|
|
|
|
# Default based on extension
|
|
if filepath.suffix == ".csv" or filepath.suffix == ".tsv":
|
|
return "data_science"
|
|
elif filepath.suffix == ".json":
|
|
return "specification"
|
|
return "unknown"
|
|
|
|
|
|
def infer_tier(filepath: Path) -> str:
|
|
"""Infer tier from path."""
|
|
path = str(filepath).lower()
|
|
if "6-Documentation/docs/semantics" in path or "docs" in path:
|
|
return "CORE"
|
|
elif "shared-data/data/germane" in path:
|
|
return "AUX"
|
|
elif "out" in path:
|
|
return "DERIVED"
|
|
return "AUX"
|
|
|
|
|
|
def read_text_safely(filepath: Path, max_size: int = 1_000_000) -> str:
|
|
"""Read text file safely with encoding fallback."""
|
|
encodings = ["utf-8", "utf-8-sig", "latin-1", "ascii", "cp1252"]
|
|
|
|
size = filepath.stat().st_size
|
|
if size > max_size:
|
|
# Read first and last chunks
|
|
with open(filepath, "rb") as f:
|
|
start = f.read(500)
|
|
f.seek(max(0, size - 500))
|
|
end = f.read(500)
|
|
content_bytes = start + b"\n...\n" + end
|
|
else:
|
|
with open(filepath, "rb") as f:
|
|
content_bytes = f.read()
|
|
|
|
for encoding in encodings:
|
|
try:
|
|
return content_bytes.decode(encoding, errors="replace")
|
|
except Exception:
|
|
continue
|
|
|
|
return "<unable to decode>"
|
|
|
|
|
|
def extract_summary(content: str, max_chars: int = 250) -> str:
|
|
"""Extract summary from content."""
|
|
lines = content.split("\n")
|
|
summary_lines = []
|
|
|
|
for line in lines:
|
|
stripped = line.strip()
|
|
if stripped and not stripped.startswith("#") and not stripped.startswith("{"):
|
|
summary_lines.append(stripped)
|
|
if len(" ".join(summary_lines)) >= max_chars:
|
|
break
|
|
|
|
summary = " ".join(summary_lines)[:max_chars]
|
|
return summary or "<empty or binary file>"
|
|
|
|
|
|
def csv_to_dict_list(filepath: Path, limit_rows: int = 100) -> List[Dict]:
|
|
"""Read CSV file into list of dicts."""
|
|
try:
|
|
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
|
|
reader = csv.DictReader(f)
|
|
return list(islice(reader, limit_rows))
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def compute_genome(filepath: Path, content: str = "") -> Dict[str, int]:
|
|
"""Compute 6D genome signature."""
|
|
try:
|
|
size = filepath.stat().st_size
|
|
lines = len(content.split("\n")) if content else 10
|
|
|
|
mu = (lines % 256) // 32
|
|
rho = min(7, (size // 1024) % 8)
|
|
c = 4
|
|
m = 4
|
|
ne = min(7, len(filepath.stem) // 10)
|
|
sig = 0 if filepath.suffix != ".json" else 3
|
|
|
|
return {"mu": mu, "rho": rho, "c": c, "m": m, "ne": ne, "sig": sig}
|
|
except Exception:
|
|
return {"mu": 0, "rho": 0, "c": 4, "m": 4, "ne": 0, "sig": 0}
|
|
|
|
|
|
def text_to_jsonl_entry(filepath: Path, node_id: str = "qfox") -> Optional[Dict[str, Any]]:
|
|
"""Convert a text container to JSON-L entry."""
|
|
|
|
try:
|
|
should_skip_file, reason = should_skip(filepath)
|
|
if should_skip_file:
|
|
return None
|
|
|
|
# Read content
|
|
content = read_text_safely(filepath)
|
|
file_hash = compute_sha256(filepath)
|
|
file_stat = filepath.stat()
|
|
mtime_unix = file_stat.st_mtime
|
|
file_size = file_stat.st_size
|
|
|
|
# Compute metadata
|
|
domain = infer_domain(filepath, content)
|
|
tier = infer_tier(filepath)
|
|
genome = compute_genome(filepath, content)
|
|
|
|
# Create pkg identifier
|
|
rel_path = filepath.relative_to(WORKSPACE_ROOT)
|
|
pkg = f"ene/text/{filepath.suffix[1:]}/{rel_path.stem}".replace("\\", "/")
|
|
version = datetime.fromtimestamp(mtime_unix, tz=timezone.utc).isoformat()
|
|
|
|
concept_anchor = {
|
|
"domain": domain,
|
|
"concept": filepath.stem.lower().replace(" ", "_").replace("-", "_").replace(".", "_"),
|
|
"resolution": "STABLE"
|
|
}
|
|
|
|
# Extract data payload based on file type
|
|
summary = extract_summary(content)
|
|
|
|
data_payload = {
|
|
"pkg": pkg,
|
|
"version": version,
|
|
"tier": tier,
|
|
"domain": domain,
|
|
"archetype": ARCHETYPE_MAP.get(filepath.suffix, "text_document"),
|
|
"concept_anchor": concept_anchor,
|
|
"file_path": str(rel_path).replace("\\", "/"),
|
|
"file_ext": filepath.suffix,
|
|
"file_hash": file_hash,
|
|
"byte_count": file_size,
|
|
"line_count": len(content.split("\n")),
|
|
"summary": summary,
|
|
}
|
|
|
|
# Add format-specific metadata
|
|
if filepath.suffix == ".json":
|
|
try:
|
|
obj = json.loads(content)
|
|
data_payload["json_keys"] = list(obj.keys() if isinstance(obj, dict) else [])
|
|
except Exception:
|
|
data_payload["json_keys"] = []
|
|
|
|
elif filepath.suffix in {".csv", ".tsv"}:
|
|
try:
|
|
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
|
|
reader = csv.DictReader(f)
|
|
first_row = next(reader, None)
|
|
if first_row:
|
|
data_payload["columns"] = list(first_row.keys())
|
|
except Exception:
|
|
pass
|
|
|
|
# Provenance
|
|
provenance = {
|
|
"node": node_id,
|
|
"lake_seed": "text_converter",
|
|
"tailscale_ip": "127.0.0.1",
|
|
"attestation_hash": file_hash,
|
|
"prev_id": None
|
|
}
|
|
|
|
# Bind
|
|
bind = {
|
|
"lawful": True,
|
|
"cost": 0x00010000,
|
|
"invariant": "documentConsistency",
|
|
"class": "informational_bind"
|
|
}
|
|
|
|
# Full JSON-L entry
|
|
entry = {
|
|
"t": mtime_unix,
|
|
"src": "ene",
|
|
"id": f"ene:{pkg}:{version}",
|
|
"op": "upsert",
|
|
"data": data_payload,
|
|
"genome": genome,
|
|
"bind": bind,
|
|
"provenance": provenance
|
|
}
|
|
|
|
return entry
|
|
|
|
except Exception as e:
|
|
print(f" ⚠️ Error processing {filepath}: {e}", file=sys.stderr)
|
|
return None
|
|
|
|
|
|
def find_all_text_containers() -> List[Path]:
|
|
"""Find all text containers in workspace."""
|
|
containers = []
|
|
|
|
for ext in PROCESS_EXTENSIONS:
|
|
for file_path in WORKSPACE_ROOT.rglob(f"*{ext}"):
|
|
if file_path.is_file():
|
|
containers.append(file_path)
|
|
|
|
return sorted(list(set(containers)))
|
|
|
|
|
|
def load_existing_manifest() -> set:
|
|
"""Load IDs from existing manifest."""
|
|
existing_ids = set()
|
|
if MANIFEST_PATH.exists():
|
|
try:
|
|
with open(MANIFEST_PATH, "r") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
try:
|
|
entry = json.loads(line)
|
|
existing_ids.add(entry.get("id", ""))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
except Exception as e:
|
|
print(f"Warning: Could not read existing manifest: {e}")
|
|
return existing_ids
|
|
|
|
|
|
def convert_all_text_containers(output_file: Optional[str] = None) -> Tuple[int, int, int, int]:
|
|
"""
|
|
Convert all text containers to JSON-L.
|
|
|
|
Returns: (total_files, newly_converted, already_exists, skipped)
|
|
"""
|
|
from itertools import islice
|
|
|
|
output_path = Path(output_file) if output_file else MANIFEST_PATH
|
|
|
|
print(f"🔍 Scanning for text containers...")
|
|
containers = find_all_text_containers()
|
|
print(f"✅ Found {len(containers)} text containers")
|
|
|
|
existing_ids = load_existing_manifest()
|
|
print(f"📋 Manifest already has {len(existing_ids)} entries")
|
|
print()
|
|
|
|
converted = 0
|
|
skipped = 0
|
|
already_exists = 0
|
|
|
|
ext_stats = {}
|
|
|
|
with open(output_path, "a") as manifest_f:
|
|
for i, container in enumerate(containers, 1):
|
|
ext = container.suffix
|
|
ext_stats[ext] = ext_stats.get(ext, 0) + 1
|
|
|
|
entry = text_to_jsonl_entry(container)
|
|
|
|
if entry is None:
|
|
skipped += 1
|
|
status = "⏭️ SKIP"
|
|
else:
|
|
entry_id = entry.get("id", "")
|
|
if entry_id in existing_ids:
|
|
already_exists += 1
|
|
status = "⏪ DUP"
|
|
else:
|
|
manifest_f.write(json.dumps(entry) + "\n")
|
|
manifest_f.flush()
|
|
converted += 1
|
|
status = "✅ CONV"
|
|
|
|
rel_path = container.relative_to(WORKSPACE_ROOT)
|
|
|
|
# Less verbose output
|
|
if i % 10 == 0 or status == "✅ CONV":
|
|
print(f"[{i:4d}/{len(containers)}] {status} {rel_path}")
|
|
|
|
return len(containers), converted, already_exists, skipped
|
|
|
|
|
|
def main():
|
|
"""Main entry point."""
|
|
output_file = None
|
|
if len(sys.argv) > 2 and sys.argv[1] == "--output-file":
|
|
output_file = sys.argv[2]
|
|
|
|
print("=" * 75)
|
|
print("🌐 Comprehensive Text Container to JSON-L Converter")
|
|
print(f"Workspace: {WORKSPACE_ROOT}")
|
|
print(f"Output: {output_file or MANIFEST_PATH}")
|
|
print("=" * 75)
|
|
print()
|
|
|
|
total, converted, already_exists, skipped = convert_all_text_containers(output_file)
|
|
|
|
print()
|
|
print("=" * 75)
|
|
print(f"📊 Summary:")
|
|
print(f" Total files scanned: {total}")
|
|
print(f" Newly converted: {converted}")
|
|
print(f" Already in manifest: {already_exists}")
|
|
print(f" Skipped: {skipped}")
|
|
print(f" Output file: {output_file or MANIFEST_PATH}")
|
|
print("=" * 75)
|
|
|
|
return 0 if converted > 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|