Research-Stack/4-Infrastructure/infra/swarm_ene_middleware.py

#!/usr/bin/env python3
"""
Swarm API-ENE Middleware

Middleware layer that hooks the Research/Swarm API into the ENE database
for caching, audit logging, and secure data storage. Intercepts API calls,
checks ENE cache first, and stores results for future use.

Features:
- Query result caching in ENE database
- Semantic vector-based retrieval
- Access control integration
- Audit logging for all operations
- Automatic cache invalidation on updates
"""

import hashlib
import json
import sqlite3
import sys
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional, Any

# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent.parent))

from infra.ene_api import ENEAPIHook, AccessLevel


@dataclass
class SwarmQueryCache:
    """Cached swarm query result"""
    query_hash: str
    subjects: List[str]
    keywords: Optional[str]
    formal_status: Optional[str]
    results: List[Dict[str, Any]]
    count: int
    confidence: float
    timestamp: int
    semantic_vector: List[float]
    ttl: int = 3600  # 1 hour default


class SwarmENEMiddleware:
    """Middleware for Swarm API integration with ENE"""
    
    def __init__(self, db_path: str = "/home/allaun/Documents/Research Stack/data/substrate_index.db"):
        self.db_path = db_path
        self.ene_api = ENEAPIHook()
        self._init_middleware_tables()
    
    def _init_middleware_tables(self):
        """Initialize middleware tables in ENE database"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        # Query cache table
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS swarm_query_cache (
                query_hash TEXT PRIMARY KEY,
                subjects TEXT NOT NULL,
                keywords TEXT,
                formal_status TEXT,
                results TEXT NOT NULL,
                count INTEGER NOT NULL,
                confidence REAL NOT NULL,
                semantic_vector TEXT NOT NULL,
                created_at INTEGER NOT NULL,
                ttl INTEGER NOT NULL,
                hit_count INTEGER DEFAULT 0
            )
        """)
        
        # API operation audit log
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS swarm_api_audit (
                id TEXT PRIMARY KEY,
                operation TEXT NOT NULL,
                query_hash TEXT,
                parameters TEXT NOT NULL,
                result_cached BOOLEAN NOT NULL,
                result_count INTEGER,
                execution_time_ms REAL NOT NULL,
                created_at INTEGER NOT NULL
            )
        """)
        
        # Semantic index for queries
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS swarm_semantic_index (
                id TEXT PRIMARY KEY,
                query_hash TEXT NOT NULL,
                semantic_vector TEXT NOT NULL,
                domain TEXT NOT NULL,
                created_at INTEGER NOT NULL,
                FOREIGN KEY (query_hash) REFERENCES swarm_query_cache(query_hash)
            )
        """)
        
        conn.commit()
        conn.close()
    
    def _compute_query_hash(self, subjects: List[str], keywords: Optional[str], 
                           formal_status: Optional[str]) -> str:
        """Compute hash for query cache key"""
        query_str = json.dumps({
            "subjects": sorted(subjects),
            "keywords": keywords,
            "formal_status": formal_status
        }, sort_keys=True)
        return hashlib.sha256(query_str.encode()).hexdigest()
    
    def _derive_semantic_vector(self, subjects: List[str], keywords: Optional[str]) -> List[float]:
        """Derive 14D semantic vector from query parameters"""
        # Simple heuristic: distribute subjects across semantic axes
        # In production, this would use actual semantic embeddings
        vector = [0.0] * 14
        
        for i, subject in enumerate(subjects):
            axis = i % 14
            # Hash subject to get a value in [0,1]
            h = hashlib.md5(subject.encode()).hexdigest()
            value = int(h[:8], 16) / 0xFFFFFFFF
            vector[axis] = value
        
        if keywords:
            # Add keyword influence to first axis
            h = hashlib.md5(keywords.encode()).hexdigest()
            value = int(h[:8], 16) / 0xFFFFFFFF
            vector[0] = (vector[0] + value) / 2
        
        return vector
    
    def check_cache(self, subjects: List[str], keywords: Optional[str], 
                   formal_status: Optional[str]) -> Optional[SwarmQueryCache]:
        """Check if query result is cached"""
        try:
            query_hash = self._compute_query_hash(subjects, keywords, formal_status)
            conn = sqlite3.connect(self.db_path)
            cursor = conn.cursor()
            
            cursor.execute("""
                SELECT subjects, keywords, formal_status, results, count, confidence, 
                       semantic_vector, created_at, ttl, hit_count
                FROM swarm_query_cache
                WHERE query_hash = ?
            """, (query_hash,))
            
            row = cursor.fetchone()
            
            if row:
                # Check if cache entry is still valid
                now = int(time.time())
                created_at = row[7]
                ttl = row[8]
                
                if now - created_at < ttl:
                    # Increment hit count
                    cursor.execute("""
                        UPDATE swarm_query_cache
                        SET hit_count = hit_count + 1
                        WHERE query_hash = ?
                    """, (query_hash,))
                    conn.commit()
                    conn.close()
                    
                    return SwarmQueryCache(
                        query_hash=query_hash,
                        subjects=json.loads(row[0]),
                        keywords=row[1],
                        formal_status=row[2],
                        results=json.loads(row[3]),
                        count=row[4],
                        confidence=row[5],
                        semantic_vector=json.loads(row[6]),
                        timestamp=created_at,
                        ttl=ttl
                    )
                else:
                    # Cache expired, delete it
                    cursor.execute("DELETE FROM swarm_query_cache WHERE query_hash = ?", (query_hash,))
                    conn.commit()
                    conn.close()
                    return None
            
            conn.close()
            return None
        except Exception as e:
            print(f"Error checking cache: {e}")
            return None
    
    def store_cache(self, subjects: List[str], keywords: Optional[str], formal_status: Optional[str],
                   results: List[Dict[str, Any]], count: int, confidence: float, 
                   ttl: int = 3600) -> bool:
        """Store query result in cache"""
        try:
            query_hash = self._compute_query_hash(subjects, keywords, formal_status)
            semantic_vector = self._derive_semantic_vector(subjects, keywords)
            now = int(time.time())
            
            conn = sqlite3.connect(self.db_path)
            cursor = conn.cursor()
            
            cursor.execute("""
                INSERT OR REPLACE INTO swarm_query_cache
                (query_hash, subjects, keywords, formal_status, results, count, confidence, 
                 semantic_vector, created_at, ttl)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                query_hash,
                json.dumps(subjects),
                keywords,
                formal_status,
                json.dumps(results),
                count,
                confidence,
                json.dumps(semantic_vector),
                now,
                ttl
            ))
            
            # Also store in semantic index
            domain = subjects[0] if subjects else "general"
            cursor.execute("""
                INSERT OR REPLACE INTO swarm_semantic_index
                (id, query_hash, semantic_vector, domain, created_at)
                VALUES (?, ?, ?, ?, ?)
            """, (
                f"semantic_{query_hash}",
                query_hash,
                json.dumps(semantic_vector),
                domain,
                now
            ))
            
            conn.commit()
            conn.close()
            
            # Store sensitive results in ENE secure storage
            if results:
                self.ene_api.store_sensitive_data(
                    pkg=f"swarm/query/{query_hash}",
                    payload=json.dumps(results),
                    classification=AccessLevel.INTERNAL,
                    semantic_vector=semantic_vector
                )
            
            return True
        except Exception as e:
            print(f"Error storing cache: {e}")
            return False
    
    def log_operation(self, operation: str, parameters: Dict, query_hash: Optional[str],
                    result_cached: bool, result_count: Optional[int], execution_time_ms: float):
        """Log API operation to audit trail"""
        try:
            conn = sqlite3.connect(self.db_path)
            cursor = conn.cursor()
            
            operation_id = f"op_{operation}_{int(time.time() * 1000000)}"
            
            cursor.execute("""
                INSERT INTO swarm_api_audit
                (id, operation, query_hash, parameters, result_cached, result_count, execution_time_ms, created_at)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                operation_id,
                operation,
                query_hash,
                json.dumps(parameters),
                result_cached,
                result_count,
                execution_time_ms,
                int(time.time())
            ))
            
            conn.commit()
            conn.close()
        except Exception as e:
            print(f"Error logging operation: {e}")
    
    def invalidate_cache(self, subjects: Optional[List[str]] = None):
        """Invalidate cache entries matching criteria"""
        try:
            conn = sqlite3.connect(self.db_path)
            cursor = conn.cursor()
            
            if subjects:
                # Invalidate specific subject queries
                for subject in subjects:
                    cursor.execute("""
                        DELETE FROM swarm_query_cache
                        WHERE subjects LIKE ?
                    """, (f"%{subject}%",))
            else:
                # Invalidate all cache
                cursor.execute("DELETE FROM swarm_query_cache")
            
            conn.commit()
            conn.close()
            return True
        except Exception as e:
            print(f"Error invalidating cache: {e}")
            return False
    
    def get_cache_statistics(self) -> Dict[str, Any]:
        """Get middleware cache statistics"""
        try:
            conn = sqlite3.connect(self.db_path)
            cursor = conn.cursor()
            
            # Query cache stats
            cursor.execute("SELECT COUNT(*), SUM(hit_count) FROM swarm_query_cache")
            cache_count, cache_hits = cursor.fetchone()
            
            # Audit log stats
            cursor.execute("SELECT COUNT(*) FROM swarm_api_audit")
            audit_count = cursor.fetchone()[0]
            
            # Cache hit rate
            cursor.execute("""
                SELECT 
                    COUNT(CASE WHEN hit_count > 0 THEN 1 END) * 100.0 / COUNT(*) 
                FROM swarm_query_cache
            """)
            hit_rate = cursor.fetchone()[0] or 0
            
            conn.close()
            
            return {
                "cached_queries": cache_count or 0,
                "cache_hits": cache_hits or 0,
                "audit_log_entries": audit_count or 0,
                "cache_hit_rate": round(hit_rate, 2)
            }
        except Exception as e:
            print(f"Error getting statistics: {e}")
            return {}
    
    def semantic_search(self, query_vector: List[float], threshold: float = 0.7) -> List[str]:
        """Find similar queries using semantic vector similarity"""
        try:
            conn = sqlite3.connect(self.db_path)
            cursor = conn.cursor()
            
            cursor.execute("SELECT query_hash, semantic_vector FROM swarm_semantic_index")
            rows = cursor.fetchall()
            
            similar_queries = []
            for query_hash, vector_json in rows:
                cached_vector = json.loads(vector_json)
                # Compute cosine similarity
                similarity = self._cosine_similarity(query_vector, cached_vector)
                if similarity >= threshold:
                    similar_queries.append((query_hash, similarity))
            
            conn.close()
            
            # Sort by similarity descending
            similar_queries.sort(key=lambda x: x[1], reverse=True)
            return [q[0] for q in similar_queries]
        except Exception as e:
            print(f"Error in semantic search: {e}")
            return []
    
    def _cosine_similarity(self, v1: List[float], v2: List[float]) -> float:
        """Compute cosine similarity between two vectors"""
        dot_product = sum(a * b for a, b in zip(v1, v2))
        norm1 = sum(a * a for a in v1) ** 0.5
        norm2 = sum(b * b for b in v2) ** 0.5
        if norm1 == 0 or norm2 == 0:
            return 0
        return dot_product / (norm1 * norm2)


# Example usage
if __name__ == "__main__":
    middleware = SwarmENEMiddleware()
    
    # Test cache check (should miss)
    print("Checking cache (should miss)...")
    cached = middleware.check_cache(
        subjects=["geometry", "topology"],
        keywords="manifold",
        formal_status="proven"
    )
    print(f"Cached result: {cached}")
    
    # Store a cache entry
    print("Storing cache entry...")
    middleware.store_cache(
        subjects=["geometry", "topology"],
        keywords="manifold",
        formal_status="proven",
        results=[{"id": 1, "name": "Test Result"}],
        count=1,
        confidence=0.95,
        ttl=3600
    )
    
    # Check cache again (should hit)
    print("Checking cache again (should hit)...")
    cached = middleware.check_cache(
        subjects=["geometry", "topology"],
        keywords="manifold",
        formal_status="proven"
    )
    print(f"Cached result: {cached}")
    
    # Log an operation
    print("Logging operation...")
    middleware.log_operation(
        operation="query",
        parameters={"subjects": ["geometry"], "limit": 10},
        query_hash=None,
        result_cached=False,
        result_count=5,
        execution_time_ms=123.45
    )
    
    # Get statistics
    print("Cache statistics:")
    stats = middleware.get_cache_statistics()
    print(json.dumps(stats, indent=2))