mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-08-07 09:25:47 +00:00
273 lines
9.6 KiB
Python
273 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Topological Storage Delta GCL Integration
|
|
|
|
Integrates Delta GCL compression with topological storage manifests.
|
|
Compresses metadata before storing to Google Drive topological storage.
|
|
|
|
Features:
|
|
- Delta GCL compression for topological manifests
|
|
- Previous state tracking for delta encoding
|
|
- Integration with Rclone for storage operations
|
|
- Compression statistics tracking
|
|
|
|
Per AGENTS.md: This is a shim layer - logic in Lean, only JSON marshaling here.
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
# Add project root to Python path
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
import json
|
|
import hashlib
|
|
from typing import Dict, Any, List, Optional
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
|
|
from infra.delta_gcl_compression_service import DeltaGCLCompressionService, CompressionResult
|
|
|
|
|
|
@dataclass
|
|
class TopologicalManifest:
|
|
"""Topological storage manifest."""
|
|
manifest_id: str
|
|
version: int
|
|
timestamp: float
|
|
topology_type: str # "manifold", "connectome", "resource", etc.
|
|
data: Dict[str, Any]
|
|
compression_metadata: Optional[Dict[str, Any]] = None
|
|
|
|
|
|
@dataclass
|
|
class StoredManifest:
|
|
"""Manifest stored in topological storage with compression."""
|
|
manifest_id: str
|
|
version: int
|
|
compressed_data: str # Delta GCL compressed
|
|
original_size: int
|
|
compressed_size: int
|
|
stored_at: float
|
|
storage_path: str
|
|
|
|
|
|
class TopologicalStorageDeltaGCL:
|
|
"""
|
|
Delta GCL compression for topological storage manifests.
|
|
|
|
Compresses manifests before storing to Google Drive topological storage.
|
|
Maintains previous state for delta encoding.
|
|
"""
|
|
|
|
def __init__(self, storage_path: str = "gdrive:topological_storage/manifests"):
|
|
self.storage_path = storage_path
|
|
self.compression_service = DeltaGCLCompressionService()
|
|
self.previous_manifests: Dict[str, TopologicalManifest] = {}
|
|
self.stored_manifests: Dict[str, StoredManifest] = {}
|
|
|
|
def compress_manifest(self, manifest: TopologicalManifest) -> CompressionResult:
|
|
"""Compress manifest using Delta GCL."""
|
|
# Convert to PTOS format for compression
|
|
ptos_manifest = {
|
|
"layer": manifest.topology_type,
|
|
"domain": "TOPOLOGICAL",
|
|
"tier": "STORAGE",
|
|
"condition": "ACTIVE",
|
|
"metadata": manifest.data
|
|
}
|
|
|
|
# Check if we have previous version for delta encoding
|
|
previous = None
|
|
use_delta = False
|
|
if manifest.manifest_id in self.previous_manifests:
|
|
previous = self.previous_manifests[manifest.manifest_id]
|
|
use_delta = True
|
|
|
|
# Compress
|
|
result = self.compression_service.compress_manifest(
|
|
ptos_manifest,
|
|
f"topo_{manifest.manifest_id}",
|
|
use_delta=use_delta
|
|
)
|
|
|
|
# Update previous manifest
|
|
self.previous_manifests[manifest.manifest_id] = manifest
|
|
|
|
return result
|
|
|
|
def store_manifest(self, manifest: TopologicalManifest,
|
|
storage_path: Optional[str] = None) -> StoredManifest:
|
|
"""
|
|
Store manifest to topological storage with Delta GCL compression.
|
|
|
|
In production, this would call Rclone to upload to Google Drive.
|
|
For now, simulates storage.
|
|
"""
|
|
# Compress manifest
|
|
compression_result = self.compress_manifest(manifest)
|
|
|
|
# Calculate sizes
|
|
original_size = len(json.dumps(manifest.data))
|
|
compressed_size = len(compression_result.delta_gcl)
|
|
|
|
# Determine storage path
|
|
path = storage_path or self.storage_path
|
|
full_path = f"{path}/{manifest.manifest_id}_v{manifest.version}.delta_gcl"
|
|
|
|
# Create stored manifest record
|
|
stored = StoredManifest(
|
|
manifest_id=manifest.manifest_id,
|
|
version=manifest.version,
|
|
compressed_data=compression_result.delta_gcl,
|
|
original_size=original_size,
|
|
compressed_size=compressed_size,
|
|
stored_at=datetime.now().timestamp(),
|
|
storage_path=full_path
|
|
)
|
|
|
|
# Store in local cache (in production: upload via Rclone)
|
|
self.stored_manifests[manifest.manifest_id] = stored
|
|
|
|
# Add compression metadata to manifest
|
|
manifest.compression_metadata = {
|
|
"original_size": original_size,
|
|
"compressed_size": compressed_size,
|
|
"reduction": original_size - compressed_size,
|
|
"reduction_percent": compression_result.stats["reduction_percent"],
|
|
"compression_method": "delta_gcl",
|
|
"use_delta": compression_result.stats["use_delta"]
|
|
}
|
|
|
|
return stored
|
|
|
|
def retrieve_manifest(self, manifest_id: str) -> Optional[TopologicalManifest]:
|
|
"""
|
|
Retrieve manifest from topological storage.
|
|
|
|
In production, this would call Rclone to download from Google Drive.
|
|
For now, retrieves from local cache.
|
|
"""
|
|
if manifest_id not in self.stored_manifests:
|
|
return None
|
|
|
|
stored = self.stored_manifests[manifest_id]
|
|
|
|
# In production: decompress using Delta GCL
|
|
# For now: return placeholder indicating compression
|
|
return TopologicalManifest(
|
|
manifest_id=stored.manifest_id,
|
|
version=stored.version,
|
|
timestamp=stored.stored_at,
|
|
topology_type="retrieved",
|
|
data={"compressed": True, "path": stored.storage_path},
|
|
compression_metadata={
|
|
"original_size": stored.original_size,
|
|
"compressed_size": stored.compressed_size,
|
|
"reduction": stored.original_size - stored.compressed_size
|
|
}
|
|
)
|
|
|
|
def get_compression_stats(self) -> Dict[str, Any]:
|
|
"""Get aggregate compression statistics."""
|
|
if not self.stored_manifests:
|
|
return {"total_manifests": 0}
|
|
|
|
total_original = sum(m.original_size for m in self.stored_manifests.values())
|
|
total_compressed = sum(m.compressed_size for m in self.stored_manifests.values())
|
|
total_reduction = total_original - total_compressed
|
|
avg_reduction = total_reduction / len(self.stored_manifests) if self.stored_manifests else 0
|
|
|
|
return {
|
|
"total_manifests": len(self.stored_manifests),
|
|
"total_original_size": total_original,
|
|
"total_compressed_size": total_compressed,
|
|
"total_reduction": total_reduction,
|
|
"average_reduction_percent": (avg_reduction / total_original * 100) if total_original > 0 else 0
|
|
}
|
|
|
|
def list_manifests(self) -> List[str]:
|
|
"""List all stored manifest IDs."""
|
|
return list(self.stored_manifests.keys())
|
|
|
|
|
|
# Singleton instance
|
|
_storage_instance: Optional[TopologicalStorageDeltaGCL] = None
|
|
|
|
|
|
def get_topological_storage() -> TopologicalStorageDeltaGCL:
|
|
"""Get singleton topological storage instance."""
|
|
global _storage_instance
|
|
if _storage_instance is None:
|
|
_storage_instance = TopologicalStorageDeltaGCL()
|
|
return _storage_instance
|
|
|
|
|
|
def store_topological_manifest(manifest: TopologicalManifest) -> StoredManifest:
|
|
"""Convenience function to store manifest."""
|
|
storage = get_topological_storage()
|
|
return storage.store_manifest(manifest)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Test topological storage with Delta GCL
|
|
print("=" * 70)
|
|
print("TOPOLOGICAL STORAGE DELTA GCL INTEGRATION")
|
|
print("=" * 70)
|
|
|
|
storage = TopologicalStorageDeltaGCL()
|
|
|
|
# Create test manifest
|
|
manifest1 = TopologicalManifest(
|
|
manifest_id="manifold_001",
|
|
version=1,
|
|
timestamp=datetime.now().timestamp(),
|
|
topology_type="manifold",
|
|
data={
|
|
"vertices": 1000,
|
|
"edges": 2500,
|
|
"dimension": 14,
|
|
"embedding": "high_dim"
|
|
}
|
|
)
|
|
|
|
print("\n[1] Storing first manifest (full encoding)...")
|
|
stored1 = storage.store_manifest(manifest1)
|
|
print(f" Manifest ID: {stored1.manifest_id}")
|
|
print(f" Storage Path: {stored1.storage_path}")
|
|
print(f" Compression: {stored1.original_size} → {stored1.compressed_size} bytes")
|
|
print(f" Reduction: {stored1.original_size - stored1.compressed_size} bytes")
|
|
|
|
# Create second version (delta encoding)
|
|
manifest2 = TopologicalManifest(
|
|
manifest_id="manifold_001",
|
|
version=2,
|
|
timestamp=datetime.now().timestamp(),
|
|
topology_type="manifold",
|
|
data={
|
|
"vertices": 1005, # Small change
|
|
"edges": 2510,
|
|
"dimension": 14,
|
|
"embedding": "high_dim"
|
|
}
|
|
)
|
|
|
|
print("\n[2] Storing second manifest (delta encoding)...")
|
|
stored2 = storage.store_manifest(manifest2)
|
|
print(f" Manifest ID: {stored2.manifest_id}")
|
|
print(f" Version: {stored2.version}")
|
|
print(f" Compression: {stored2.original_size} → {stored2.compressed_size} bytes")
|
|
print(f" Reduction: {stored2.original_size - stored2.compressed_size} bytes")
|
|
|
|
# Get aggregate stats
|
|
print("\n[3] Aggregate compression statistics...")
|
|
stats = storage.get_compression_stats()
|
|
print(f" Total Manifests: {stats['total_manifests']}")
|
|
print(f" Total Original: {stats['total_original_size']} bytes")
|
|
print(f" Total Compressed: {stats['total_compressed_size']} bytes")
|
|
print(f" Total Reduction: {stats['total_reduction']} bytes")
|
|
print(f" Avg Reduction: {stats['average_reduction_percent']:.2f}%")
|
|
|
|
print("\n" + "=" * 70)
|
|
print("TOPOLOGICAL STORAGE DELTA GCL OPERATIONAL")
|
|
print("=" * 70)
|