mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
550 lines
No EOL
18 KiB
Python
550 lines
No EOL
18 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Cloud Runtime System — Central Orchestration Layer
|
|
|
|
Implements the cloud architecture from the diagram:
|
|
- Agent Runtime: Central execution environment
|
|
- Session State: Session management and persistence
|
|
- File System: Unified storage interface
|
|
- MCP Integration: Model Context Protocol orchestration
|
|
- Tools: Unified tool registry and execution
|
|
|
|
This ties together existing infrastructure components into a coherent cloud system.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import sqlite3
|
|
import time
|
|
import uuid
|
|
import hashlib
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Any, Callable
|
|
from dataclasses import dataclass, field, asdict
|
|
from datetime import datetime, timedelta
|
|
from enum import Enum
|
|
import subprocess
|
|
import sys
|
|
|
|
# Add paths to existing infrastructure
|
|
ROOT = Path(__file__).parent.parent.parent
|
|
sys.path.insert(0, str(ROOT / "4-Infrastructure" / "infra"))
|
|
sys.path.insert(0, str(ROOT / "5-Applications" / "scripts"))
|
|
|
|
|
|
class SessionState(Enum):
|
|
"""Session lifecycle states."""
|
|
INITIALIZING = "initializing"
|
|
ACTIVE = "active"
|
|
SUSPENDED = "suspended"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
|
|
|
|
@dataclass
|
|
class Session:
|
|
"""Cloud session representation."""
|
|
session_id: str
|
|
created_at: float
|
|
updated_at: float
|
|
state: SessionState
|
|
workspace: str
|
|
agent_id: Optional[str] = None
|
|
context: Dict[str, Any] = field(default_factory=dict)
|
|
file_system_state: Dict[str, Any] = field(default_factory=dict)
|
|
mcp_connections: Dict[str, bool] = field(default_factory=dict)
|
|
tool_registry: List[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class AgentRuntime:
|
|
"""Agent runtime configuration and state."""
|
|
runtime_id: str
|
|
agent_type: str
|
|
model: str
|
|
capabilities: List[str]
|
|
status: str = "idle"
|
|
current_session: Optional[str] = None
|
|
metrics: Dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
class CloudFileSystem:
|
|
"""
|
|
Unified file system interface for cloud runtime.
|
|
|
|
Provides consistent file operations across local storage,
|
|
cloud storage, and MCP-mounted file systems.
|
|
"""
|
|
|
|
def __init__(self, base_path: str = "/home/allaun/Documents/Research Stack"):
|
|
self.base_path = Path(base_path)
|
|
self.shared_data = self.base_path / "shared-data"
|
|
self.mount_points: Dict[str, Path] = {}
|
|
|
|
def register_mount(self, name: str, path: str):
|
|
"""Register a file system mount point."""
|
|
self.mount_points[name] = Path(path)
|
|
|
|
def read_file(self, path: str, mount: str = "local") -> str:
|
|
"""Read file from specified mount."""
|
|
if mount == "local":
|
|
file_path = self.base_path / path
|
|
elif mount in self.mount_points:
|
|
file_path = self.mount_points[mount] / path
|
|
else:
|
|
raise ValueError(f"Unknown mount: {mount}")
|
|
|
|
return file_path.read_text()
|
|
|
|
def write_file(self, path: str, content: str, mount: str = "local"):
|
|
"""Write file to specified mount."""
|
|
if mount == "local":
|
|
file_path = self.base_path / path
|
|
elif mount in self.mount_points:
|
|
file_path = self.mount_points[mount] / path
|
|
else:
|
|
raise ValueError(f"Unknown mount: {mount}")
|
|
|
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
file_path.write_text(content)
|
|
|
|
def list_files(self, path: str = "", mount: str = "local") -> List[str]:
|
|
"""List files in directory."""
|
|
if mount == "local":
|
|
dir_path = self.base_path / path
|
|
elif mount in self.mount_points:
|
|
dir_path = self.mount_points[mount] / path
|
|
else:
|
|
raise ValueError(f"Unknown mount: {mount}")
|
|
|
|
if not dir_path.exists():
|
|
return []
|
|
|
|
return [f.name for f in dir_path.iterdir() if f.is_file()]
|
|
|
|
|
|
class MCPOrchestrator:
|
|
"""
|
|
MCP (Model Context Protocol) orchestrator.
|
|
|
|
Manages connections to multiple MCP servers and provides
|
|
unified interface for tool execution across all servers.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.servers: Dict[str, Any] = {}
|
|
self.server_status: Dict[str, bool] = {}
|
|
self.tool_registry: Dict[str, Dict[str, Any]] = {}
|
|
|
|
async def initialize_server(self, server_name: str, config: Dict[str, Any]):
|
|
"""Initialize an MCP server connection."""
|
|
try:
|
|
# This would connect to actual MCP servers
|
|
# For now, we'll simulate the connection
|
|
self.servers[server_name] = config
|
|
self.server_status[server_name] = True
|
|
print(f"[MCP] Initialized server: {server_name}")
|
|
return True
|
|
except Exception as e:
|
|
print(f"[MCP] Failed to initialize {server_name}: {e}")
|
|
self.server_status[server_name] = False
|
|
return False
|
|
|
|
async def call_tool(self, server_name: str, tool_name: str, arguments: Dict[str, Any]) -> Any:
|
|
"""Execute a tool on a specific MCP server."""
|
|
if server_name not in self.servers or not self.server_status[server_name]:
|
|
raise ValueError(f"Server {server_name} not available")
|
|
|
|
# Simulate tool execution
|
|
print(f"[MCP] Calling {tool_name} on {server_name} with args: {arguments}")
|
|
return {"ok": True, "result": f"Executed {tool_name}"}
|
|
|
|
def get_available_tools(self, server_name: str) -> List[str]:
|
|
"""Get available tools from a server."""
|
|
if server_name not in self.servers:
|
|
return []
|
|
return ["tool1", "tool2", "tool3"] # Placeholder
|
|
|
|
|
|
class ToolRegistry:
|
|
"""
|
|
Unified tool registry.
|
|
|
|
Manages all available tools from MCP servers, local scripts,
|
|
and cloud functions in a single registry.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.tools: Dict[str, Dict[str, Any]] = {}
|
|
self.categories: Dict[str, List[str]] = {
|
|
"file": [],
|
|
"mcp": [],
|
|
"cloud": [],
|
|
"local": []
|
|
}
|
|
|
|
def register_tool(self, tool_id: str, tool_def: Dict[str, Any], category: str = "local"):
|
|
"""Register a tool in the registry."""
|
|
self.tools[tool_id] = tool_def
|
|
if category not in self.categories:
|
|
self.categories[category] = []
|
|
self.categories[category].append(tool_id)
|
|
|
|
def get_tool(self, tool_id: str) -> Optional[Dict[str, Any]]:
|
|
"""Get tool definition."""
|
|
return self.tools.get(tool_id)
|
|
|
|
def list_tools(self, category: Optional[str] = None) -> List[str]:
|
|
"""List tools by category."""
|
|
if category:
|
|
return self.categories.get(category, [])
|
|
return list(self.tools.keys())
|
|
|
|
def execute_tool(self, tool_id: str, arguments: Dict[str, Any]) -> Any:
|
|
"""Execute a tool by ID."""
|
|
tool = self.get_tool(tool_id)
|
|
if not tool:
|
|
raise ValueError(f"Tool {tool_id} not found")
|
|
|
|
# Execute based on tool type
|
|
if tool.get("type") == "mcp":
|
|
# Would call MCP orchestrator
|
|
return {"ok": True, "result": f"Executed MCP tool {tool_id}"}
|
|
elif tool.get("type") == "local":
|
|
# Execute local script
|
|
return self._execute_local_tool(tool, arguments)
|
|
else:
|
|
return {"ok": False, "error": f"Unknown tool type: {tool.get('type')}"}
|
|
|
|
def _execute_local_tool(self, tool: Dict[str, Any], arguments: Dict[str, Any]) -> Any:
|
|
"""Execute a local tool script."""
|
|
script_path = tool.get("path")
|
|
if not script_path:
|
|
return {"ok": False, "error": "No script path"}
|
|
|
|
try:
|
|
# Execute script
|
|
result = subprocess.run(
|
|
["python3", script_path],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30
|
|
)
|
|
return {"ok": True, "output": result.stdout, "error": result.stderr}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)}
|
|
|
|
|
|
class CloudRuntime:
|
|
"""
|
|
Central Cloud Runtime — Main Orchestrator
|
|
|
|
Implements the complete cloud architecture from the diagram:
|
|
- Manages sessions and their state
|
|
- Coordinates agent runtimes
|
|
- Orchestrates MCP connections
|
|
- Provides unified tool interface
|
|
- Manages file system operations
|
|
"""
|
|
|
|
def __init__(self, db_path: str = "/home/allaun/Documents/Research Stack/shared-data/data/cloud_runtime.db"):
|
|
self.db_path = db_path
|
|
self.sessions: Dict[str, Session] = {}
|
|
self.agent_runtimes: Dict[str, AgentRuntime] = {}
|
|
self.file_system = CloudFileSystem()
|
|
self.mcp_orchestrator = MCPOrchestrator()
|
|
self.tool_registry = ToolRegistry()
|
|
|
|
self._init_database()
|
|
self._load_sessions()
|
|
self._init_agent_runtimes()
|
|
self._register_tools()
|
|
|
|
def _init_database(self):
|
|
"""Initialize runtime database."""
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS sessions (
|
|
session_id TEXT PRIMARY KEY,
|
|
created_at REAL,
|
|
updated_at REAL,
|
|
state TEXT,
|
|
workspace TEXT,
|
|
agent_id TEXT,
|
|
context TEXT,
|
|
file_system_state TEXT,
|
|
mcp_connections TEXT,
|
|
tool_registry TEXT
|
|
)
|
|
""")
|
|
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS agent_runtimes (
|
|
runtime_id TEXT PRIMARY KEY,
|
|
agent_type TEXT,
|
|
model TEXT,
|
|
capabilities TEXT,
|
|
status TEXT,
|
|
current_session TEXT,
|
|
metrics TEXT
|
|
)
|
|
""")
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
def _load_sessions(self):
|
|
"""Load sessions from database."""
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute("SELECT * FROM sessions")
|
|
for row in cursor.fetchall():
|
|
session = Session(
|
|
session_id=row[0],
|
|
created_at=row[1],
|
|
updated_at=row[2],
|
|
state=SessionState(row[3]),
|
|
workspace=row[4],
|
|
agent_id=row[5],
|
|
context=json.loads(row[6]) if row[6] else {},
|
|
file_system_state=json.loads(row[7]) if row[7] else {},
|
|
mcp_connections=json.loads(row[8]) if row[8] else {},
|
|
tool_registry=json.loads(row[9]) if row[9] else []
|
|
)
|
|
self.sessions[session.session_id] = session
|
|
|
|
conn.close()
|
|
|
|
def _init_agent_runtimes(self):
|
|
"""Initialize default agent runtimes."""
|
|
# Devin CLI agent
|
|
self.agent_runtimes["devin-cli"] = AgentRuntime(
|
|
runtime_id="devin-cli",
|
|
agent_type="devin-cli",
|
|
model="swe-1-6",
|
|
capabilities=["code_editing", "file_operations", "terminal"],
|
|
status="idle"
|
|
)
|
|
|
|
# Devin Cloud agent
|
|
self.agent_runtimes["devin-cloud"] = AgentRuntime(
|
|
runtime_id="devin-cloud",
|
|
agent_type="devin-cloud",
|
|
model="adaptive",
|
|
capabilities=["cloud_execution", "mcp_tools", "session_management"],
|
|
status="idle"
|
|
)
|
|
|
|
async def _init_mcp_servers(self):
|
|
"""Initialize MCP servers from configuration."""
|
|
# Initialize known MCP servers
|
|
mcp_configs = {
|
|
"ene": {"type": "stdio", "command": "python3", "args": ["mcp_ene_atlas.py"]},
|
|
"sovereign-research-stack": {"type": "stdio", "command": "python3", "args": ["mcp_server.py"]},
|
|
"sqlite": {"type": "stdio", "command": "uvx", "args": ["mcp-server-sqlite"]},
|
|
}
|
|
|
|
for server_name, config in mcp_configs.items():
|
|
await self.mcp_orchestrator.initialize_server(server_name, config)
|
|
|
|
def _register_tools(self):
|
|
"""Register available tools from various sources."""
|
|
# Register file system tools
|
|
self.tool_registry.register_tool("fs.read", {
|
|
"name": "Read File",
|
|
"description": "Read file from file system",
|
|
"type": "local",
|
|
"parameters": {"path": "string", "mount": "string"}
|
|
}, "file")
|
|
|
|
self.tool_registry.register_tool("fs.write", {
|
|
"name": "Write File",
|
|
"description": "Write file to file system",
|
|
"type": "local",
|
|
"parameters": {"path": "string", "content": "string", "mount": "string"}
|
|
}, "file")
|
|
|
|
# Register MCP tools
|
|
self.tool_registry.register_tool("mcp.ene.lookup", {
|
|
"name": "ENE Atlas Lookup",
|
|
"description": "Lookup memory atoms in ENE Atlas",
|
|
"type": "mcp",
|
|
"server": "ene"
|
|
}, "mcp")
|
|
|
|
async def create_session(self, workspace: str, agent_type: str = "devin-cli") -> str:
|
|
"""Create a new cloud session."""
|
|
session_id = str(uuid.uuid4())
|
|
now = time.time()
|
|
|
|
session = Session(
|
|
session_id=session_id,
|
|
created_at=now,
|
|
updated_at=now,
|
|
state=SessionState.INITIALIZING,
|
|
workspace=workspace,
|
|
agent_id=agent_type
|
|
)
|
|
|
|
self.sessions[session_id] = session
|
|
self._save_session(session)
|
|
|
|
# Initialize session
|
|
await self._initialize_session(session_id)
|
|
|
|
return session_id
|
|
|
|
async def _initialize_session(self, session_id: str):
|
|
"""Initialize a session with required resources."""
|
|
session = self.sessions[session_id]
|
|
|
|
# Set up MCP connections
|
|
for server_name in self.mcp_orchestrator.servers:
|
|
session.mcp_connections[server_name] = self.mcp_orchestrator.server_status[server_name]
|
|
|
|
# Initialize file system state
|
|
session.file_system_state = {
|
|
"mounts": list(self.file_system.mount_points.keys()),
|
|
"workspace_files": self.file_system.list_files(mount="local")
|
|
}
|
|
|
|
# Register available tools
|
|
session.tool_registry = self.tool_registry.list_tools()
|
|
|
|
# Update session state
|
|
session.state = SessionState.ACTIVE
|
|
session.updated_at = time.time()
|
|
self._save_session(session)
|
|
|
|
def _save_session(self, session: Session):
|
|
"""Save session to database."""
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute("""
|
|
INSERT OR REPLACE INTO sessions
|
|
(session_id, created_at, updated_at, state, workspace, agent_id,
|
|
context, file_system_state, mcp_connections, tool_registry)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
session.session_id,
|
|
session.created_at,
|
|
session.updated_at,
|
|
session.state.value,
|
|
session.workspace,
|
|
session.agent_id,
|
|
json.dumps(session.context),
|
|
json.dumps(session.file_system_state),
|
|
json.dumps(session.mcp_connections),
|
|
json.dumps(session.tool_registry)
|
|
))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
async def execute_tool(self, session_id: str, tool_id: str, arguments: Dict[str, Any]) -> Any:
|
|
"""Execute a tool within a session."""
|
|
if session_id not in self.sessions:
|
|
raise ValueError(f"Session {session_id} not found")
|
|
|
|
session = self.sessions[session_id]
|
|
session.updated_at = time.time()
|
|
|
|
# Execute tool
|
|
result = self.tool_registry.execute_tool(tool_id, arguments)
|
|
|
|
# Update session context
|
|
session.context[f"last_tool_{tool_id}"] = {
|
|
"executed_at": time.time(),
|
|
"arguments": arguments,
|
|
"result": result
|
|
}
|
|
|
|
self._save_session(session)
|
|
return result
|
|
|
|
def get_session_status(self, session_id: str) -> Dict[str, Any]:
|
|
"""Get session status and state."""
|
|
if session_id not in self.sessions:
|
|
raise ValueError(f"Session {session_id} not found")
|
|
|
|
session = self.sessions[session_id]
|
|
return {
|
|
"session_id": session.session_id,
|
|
"state": session.state.value,
|
|
"workspace": session.workspace,
|
|
"agent_id": session.agent_id,
|
|
"created_at": session.created_at,
|
|
"updated_at": session.updated_at,
|
|
"mcp_connections": session.mcp_connections,
|
|
"available_tools": len(session.tool_registry)
|
|
}
|
|
|
|
def get_runtime_status(self) -> Dict[str, Any]:
|
|
"""Get overall runtime status."""
|
|
return {
|
|
"sessions": {
|
|
"total": len(self.sessions),
|
|
"active": sum(1 for s in self.sessions.values() if s.state == SessionState.ACTIVE),
|
|
"by_state": {state.value: sum(1 for s in self.sessions.values() if s.state == state)
|
|
for state in SessionState}
|
|
},
|
|
"agent_runtimes": {
|
|
runtime_id: {
|
|
"type": runtime.agent_type,
|
|
"model": runtime.model,
|
|
"status": runtime.status,
|
|
"current_session": runtime.current_session
|
|
}
|
|
for runtime_id, runtime in self.agent_runtimes.items()
|
|
},
|
|
"mcp_servers": {
|
|
"total": len(self.mcp_orchestrator.servers),
|
|
"connected": sum(1 for s in self.mcp_orchestrator.server_status.values() if s),
|
|
"status": self.mcp_orchestrator.server_status
|
|
},
|
|
"tools": {
|
|
"total": len(self.tool_registry.tools),
|
|
"by_category": {cat: len(tools) for cat, tools in self.tool_registry.categories.items()}
|
|
},
|
|
"file_system": {
|
|
"base_path": str(self.file_system.base_path),
|
|
"mounts": list(self.file_system.mount_points.keys())
|
|
}
|
|
}
|
|
|
|
|
|
async def main():
|
|
"""Main entry point for cloud runtime."""
|
|
runtime = CloudRuntime()
|
|
|
|
print("=== Cloud Runtime System ===")
|
|
print("Initializing cloud infrastructure...")
|
|
|
|
# Initialize MCP servers
|
|
print("\nInitializing MCP servers...")
|
|
await runtime._init_mcp_servers()
|
|
|
|
# Create a test session
|
|
print("\nCreating test session...")
|
|
session_id = await runtime.create_session("/home/allaun/Documents/Research Stack", "devin-cli")
|
|
print(f"Created session: {session_id}")
|
|
|
|
# Get session status
|
|
print("\nSession status:")
|
|
status = runtime.get_session_status(session_id)
|
|
print(json.dumps(status, indent=2))
|
|
|
|
# Get runtime status
|
|
print("\nRuntime status:")
|
|
runtime_status = runtime.get_runtime_status()
|
|
print(json.dumps(runtime_status, indent=2))
|
|
|
|
print("\n=== Cloud Runtime Initialized Successfully ===")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |