mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
208 lines
7.9 KiB
Python
208 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Analyze waveform differences from user's compression approaches.
|
|
"""
|
|
|
|
import os
|
|
import numpy as np
|
|
import json
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent
|
|
|
|
# Configuration
|
|
ENWIK9_PATH = REPO_ROOT / "hutter_bind_implementation" / "enwik9"
|
|
OUTPUT_DIR = REPO_ROOT / "compression_comparison"
|
|
USER_COMPRESSION_DIR = REPO_ROOT
|
|
WINDOW_SIZE = 1024
|
|
|
|
def extract_waveform(data, window_size=WINDOW_SIZE):
|
|
"""Extract waveform from data using sliding window."""
|
|
waveform = []
|
|
n_windows = len(data) // window_size
|
|
|
|
for i in range(n_windows):
|
|
window = data[i*window_size:(i+1)*window_size]
|
|
energy = sum(window)
|
|
waveform.append(energy)
|
|
|
|
return np.array(waveform, dtype=np.float64)
|
|
|
|
def load_user_compressed_data(compressed_path, metadata_path):
|
|
"""Load and reconstruct data from user's compressed format."""
|
|
# Read compressed data
|
|
with open(compressed_path, 'rb') as f:
|
|
compressed_data = f.read()
|
|
|
|
# Read metadata to understand format
|
|
with open(metadata_path, 'r') as f:
|
|
metadata = json.load(f)
|
|
|
|
print(f"Metadata: {metadata}")
|
|
|
|
# Try to reconstruct based on metadata
|
|
# This depends on the specific compression format
|
|
|
|
return compressed_data, metadata
|
|
|
|
def main():
|
|
"""Analyze user compression waveform differences."""
|
|
print("=" * 60)
|
|
print("User Compression Waveform Analysis")
|
|
print("=" * 60)
|
|
|
|
# Load original waveform
|
|
print("\nLoading original waveform...")
|
|
with open(ENWIK9_PATH, 'rb') as f:
|
|
original_data = f.read()
|
|
|
|
original_waveform = extract_waveform(original_data)
|
|
print(f"Original waveform length: {len(original_waveform)} points")
|
|
|
|
# Load original waveform from numpy if exists
|
|
original_waveform_path = OUTPUT_DIR / "waveform_original.npy"
|
|
if original_waveform_path.exists():
|
|
original_waveform = np.load(original_waveform_path)
|
|
print(f"Loaded from: {original_waveform_path}")
|
|
|
|
# User compression approaches
|
|
user_approaches = {
|
|
'zero_spike_uvmap': {
|
|
'compressed': USER_COMPRESSION_DIR / 'enwik9_zero_spike_uvmap_compressed.bin',
|
|
'metadata': USER_COMPRESSION_DIR / 'enwik9_zero_spike_uvmap_metadata.txt'
|
|
},
|
|
'sparse_zero_spike': {
|
|
'compressed': USER_COMPRESSION_DIR / 'enwik9_sparse_zero_spike_compressed.bin',
|
|
'metadata': USER_COMPRESSION_DIR / 'enwik9_sparse_zero_spike_metadata.txt'
|
|
},
|
|
'hybrid_dyson_swarm': {
|
|
'compressed': USER_COMPRESSION_DIR / 'enwik9_hybrid_dyson_swarm_compressed.bin',
|
|
'metadata': USER_COMPRESSION_DIR / 'enwik9_hybrid_dyson_swarm_metadata.txt'
|
|
},
|
|
'irreducible_surface': {
|
|
'compressed': USER_COMPRESSION_DIR / 'enwik9_irreducible_surface_compressed.bin',
|
|
'metadata': USER_COMPRESSION_DIR / 'enwik9_irreducible_surface_metadata.txt'
|
|
}
|
|
}
|
|
|
|
# Analyze each approach
|
|
results = {}
|
|
|
|
for approach_name, paths in user_approaches.items():
|
|
print(f"\n{'=' * 60}")
|
|
print(f"Analyzing: {approach_name.upper()}")
|
|
print('=' * 60)
|
|
|
|
if not paths['compressed'].exists():
|
|
print(f"✗ Compressed file not found: {paths['compressed']}")
|
|
continue
|
|
|
|
if not paths['metadata'].exists():
|
|
print(f"✗ Metadata file not found: {paths['metadata']}")
|
|
continue
|
|
|
|
try:
|
|
# Read compressed data
|
|
with open(paths['compressed'], 'rb') as f:
|
|
compressed_data = f.read()
|
|
|
|
compressed_size = len(compressed_data)
|
|
print(f"Compressed size: {compressed_size:,} bytes ({compressed_size / 1024**3:.2f} GB)")
|
|
|
|
# Read metadata
|
|
with open(paths['metadata'], 'r') as f:
|
|
metadata_content = f.read()
|
|
|
|
print(f"Metadata preview (first 500 chars):")
|
|
print(metadata_content[:500])
|
|
|
|
# Try to parse as JSON
|
|
try:
|
|
metadata = json.loads(metadata_content)
|
|
print(f"\nParsed metadata keys: {metadata.keys()}")
|
|
|
|
# Check if there's reconstructed data info
|
|
if 'reconstructed_size' in metadata:
|
|
print(f"Reconstructed size: {metadata['reconstructed_size']:,} bytes")
|
|
|
|
if 'compression_ratio' in metadata:
|
|
print(f"Compression ratio: {metadata['compression_ratio']:.2f}x")
|
|
|
|
except json.JSONDecodeError:
|
|
print("Metadata is not JSON format")
|
|
|
|
# For now, we can't reconstruct without the decompression code
|
|
# But we can analyze the compressed data structure
|
|
print(f"\nCompressed data statistics:")
|
|
print(f" Min byte: {min(compressed_data)}")
|
|
print(f" Max byte: {max(compressed_data)}")
|
|
print(f" Unique bytes: {len(set(compressed_data))}")
|
|
|
|
# Try to extract waveform from compressed data directly
|
|
# (this won't match original, but shows what the compressed representation looks like)
|
|
compressed_waveform = extract_waveform(compressed_data)
|
|
|
|
# Compare to original (will be very different)
|
|
waveform_diff = np.abs(original_waveform[:len(compressed_waveform)] - compressed_waveform)
|
|
max_diff = np.max(waveform_diff)
|
|
mean_diff = np.mean(waveform_diff)
|
|
|
|
print(f"\nCompressed waveform (direct from compressed data):")
|
|
print(f" Length: {len(compressed_waveform)} points")
|
|
print(f" Max diff from original: {max_diff:.2f}")
|
|
print(f" Mean diff from original: {mean_diff:.2f}")
|
|
|
|
results[approach_name] = {
|
|
'compressed_size': compressed_size,
|
|
'metadata': metadata_content,
|
|
'compressed_waveform': compressed_waveform,
|
|
'max_diff': max_diff,
|
|
'mean_diff': mean_diff
|
|
}
|
|
|
|
except Exception as e:
|
|
print(f"✗ Error: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
# Save results
|
|
results_path = os.path.join(OUTPUT_DIR, "user_compression_analysis.json")
|
|
with open(results_path, 'w') as f:
|
|
# Convert numpy arrays to lists for JSON serialization
|
|
json_results = {}
|
|
for key, value in results.items():
|
|
json_results[key] = {
|
|
'compressed_size': value['compressed_size'],
|
|
'metadata_preview': value['metadata'][:1000],
|
|
'max_diff': float(value['max_diff']),
|
|
'mean_diff': float(value['mean_diff'])
|
|
}
|
|
json.dump(json_results, f, indent=2)
|
|
|
|
print(f"\nResults saved to: {results_path}")
|
|
|
|
# Summary
|
|
print("\n" + "=" * 60)
|
|
print("SUMMARY")
|
|
print("=" * 60)
|
|
print(f"{'Approach':<25} {'Size (GB)':<12} {'Max Diff':<12} {'Mean Diff':<12}")
|
|
print("-" * 60)
|
|
|
|
for approach_name, result in results.items():
|
|
size_gb = result['compressed_size'] / 1024**3
|
|
print(f"{approach_name:<25} {size_gb:<12.3f} {result['max_diff']:<12.2f} {result['mean_diff']:<12.2f}")
|
|
|
|
print("\n" + "=" * 60)
|
|
print("IMPORTANT NOTE")
|
|
print("=" * 60)
|
|
print("To properly compare waveforms, we need to:")
|
|
print("1. Decompress/reconstruct the data using your decompression code")
|
|
print("2. Extract waveform from the RECONSTRUCTED data")
|
|
print("3. Compare reconstructed waveform to original waveform")
|
|
print("\nCurrently, we're comparing the compressed binary directly to original,")
|
|
print("which shows structural differences but not the actual compression effect.")
|
|
print("\nPlease provide the decompression/reconstruction code for each approach")
|
|
print("so we can properly analyze the waveform differences.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|