mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
60 lines
2.2 KiB
Python
60 lines
2.2 KiB
Python
import numpy as np
|
|
try:
|
|
from datasets import Dataset
|
|
except ImportError:
|
|
class Dataset:
|
|
@staticmethod
|
|
def from_list(data_list): return data_list
|
|
@staticmethod
|
|
def from_dict(data_dict): return data_dict
|
|
|
|
from sovereign_shm_bridge import SovereignSHMBridge
|
|
|
|
class SovereignTopologicalSieve:
|
|
"""
|
|
Kaiser-Squires E/B Mode Sieve for Dataset Filtering.
|
|
Uses the WGPU/Rust engine via SHM to identify the 'Invariant Backbone'.
|
|
"""
|
|
def __init__(self, bridge: SovereignSHMBridge = None, threshold: float = 0.5):
|
|
self.bridge = bridge or SovereignSHMBridge()
|
|
self.threshold = threshold
|
|
|
|
def _get_collapsed_verdict(self, data_chunk: bytes) -> bool:
|
|
"""
|
|
1D Collapse: Converts high-dimensional E-mode map to a scalar verdict.
|
|
Follows the DefaultCollapser logic:
|
|
bow_wave_intensity → collision_val → net_value
|
|
"""
|
|
self.bridge.write_dataset_chunk(data_chunk)
|
|
# In a real sync, we'd trigger the internal_module_api socket here
|
|
e_mode_map = self.bridge.read_signal_map()
|
|
|
|
if len(e_mode_map) == 0:
|
|
return False
|
|
|
|
# 1D Collapse Calculation
|
|
bow_wave_intensity = np.mean(e_mode_map)
|
|
collision_val = np.round(bow_wave_intensity * 10000)
|
|
net_value = collision_val / 10000.0
|
|
|
|
# print(f" [DEBUG] Collapse: Intensity={bow_wave_intensity:.4f}, NetValue={net_value:.4f}")
|
|
|
|
# Judge (PAUSE) mechanism
|
|
return net_value > self.threshold
|
|
|
|
def filter_text_dataset(self, dataset: Dataset, text_column: str = "text") -> Dataset:
|
|
"""
|
|
Applies the sieve to a standard datasets.Dataset.
|
|
Uses the 1D Collapse verdict (Judge PAUSE) to gate entire chunks.
|
|
"""
|
|
filtered_records = []
|
|
for record in dataset:
|
|
raw_text = record[text_column]
|
|
raw_bytes = raw_text.encode("utf-8")
|
|
|
|
# Apply 1D Collapse Verdict
|
|
if self._get_collapsed_verdict(raw_bytes):
|
|
filtered_records.append(record)
|
|
# else: Skip high-entropy noise (Judge PAUSE)
|
|
|
|
return Dataset.from_list(filtered_records)
|