mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
456 lines
18 KiB
Python
456 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Catalog local and pulled MCP surfaces as gated Research Stack adapters.
|
|
|
|
This script inventories MCP-like surfaces without starting them. It snapshots
|
|
external repositories, records local MCP entrypoints, ranks useful surfaces for
|
|
the OpenClaw/metaprobe bus, and emits receipt + curriculum + wiki artifacts.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import tomllib
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
SHIM = REPO / "4-Infrastructure" / "shim"
|
|
WIKI = REPO / "6-Documentation" / "tiddlywiki-local" / "wiki" / "tiddlers"
|
|
MCP_EXTERNAL = REPO / "5-Applications" / "tools-scripts" / "external" / "mcp"
|
|
|
|
|
|
def sha256_text(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def file_hash(path: Path) -> str | None:
|
|
if not path.exists():
|
|
return None
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def read_text(path: Path, limit: int = 16000) -> str:
|
|
if not path.exists():
|
|
return ""
|
|
return path.read_text(encoding="utf-8", errors="replace")[:limit]
|
|
|
|
|
|
def run_git(path: Path, *args: str) -> str:
|
|
proc = subprocess.run(
|
|
["git", "-C", str(path), *args],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
check=False,
|
|
)
|
|
if proc.returncode != 0:
|
|
return ""
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def package_summary(path: Path) -> dict[str, Any]:
|
|
package_path = path / "package.json"
|
|
if package_path.exists():
|
|
try:
|
|
data = json.loads(package_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
data = {}
|
|
return {
|
|
"name": data.get("name"),
|
|
"version": data.get("version"),
|
|
"description": data.get("description"),
|
|
"license": data.get("license"),
|
|
"scripts": sorted((data.get("scripts") or {}).keys())[:40],
|
|
}
|
|
pyproject_path = path / "pyproject.toml"
|
|
if pyproject_path.exists():
|
|
try:
|
|
data = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
data = {}
|
|
project = data.get("project", {})
|
|
return {
|
|
"name": project.get("name"),
|
|
"version": project.get("version"),
|
|
"description": project.get("description"),
|
|
"license": project.get("license"),
|
|
"scripts": sorted((project.get("scripts") or {}).keys())[:40],
|
|
}
|
|
return {}
|
|
|
|
|
|
def repo_surface(name: str, path: Path, role: str, priority: int, gate: str) -> dict[str, Any]:
|
|
readme = read_text(path / "README.md")
|
|
return {
|
|
"id": name,
|
|
"kind": "external_snapshot",
|
|
"path": str(path.relative_to(REPO)),
|
|
"remote": run_git(path, "remote", "get-url", "origin"),
|
|
"commit": run_git(path, "rev-parse", "HEAD"),
|
|
"working_tree_clean": run_git(path, "status", "--short") == "",
|
|
"package": package_summary(path),
|
|
"role": role,
|
|
"priority": priority,
|
|
"gate": gate,
|
|
"source_fingerprint": sha256_text(readme[:8000] + run_git(path, "rev-parse", "HEAD")),
|
|
}
|
|
|
|
|
|
def local_surface(surface_id: str, path: Path, role: str, priority: int, gate: str, smoke: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
text = read_text(path)
|
|
return {
|
|
"id": surface_id,
|
|
"kind": "local_surface",
|
|
"path": str(path.relative_to(REPO)),
|
|
"source_hash": file_hash(path),
|
|
"role": role,
|
|
"priority": priority,
|
|
"gate": gate,
|
|
"smoke": smoke or {},
|
|
"source_fingerprint": sha256_text(text[:8000] + str(file_hash(path))),
|
|
}
|
|
|
|
|
|
def sciencehub_report() -> dict[str, Any]:
|
|
script = REPO / "scripts" / "sciencehub_mcp.py"
|
|
if not script.exists():
|
|
return {"available": False}
|
|
proc = subprocess.run(
|
|
["python", str(script), "--report"],
|
|
cwd=str(REPO),
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
metrics: dict[str, Any] = {"available": proc.returncode == 0, "returncode": proc.returncode}
|
|
for line in proc.stdout.splitlines():
|
|
if ":" not in line:
|
|
continue
|
|
key, value = line.split(":", 1)
|
|
key = key.strip().lower().replace(" ", "_")
|
|
value = value.strip()
|
|
if value.isdigit():
|
|
metrics[key] = int(value)
|
|
if proc.stderr.strip():
|
|
metrics["stderr_tail"] = proc.stderr[-1000:]
|
|
return metrics
|
|
|
|
|
|
def build_catalog() -> dict[str, Any]:
|
|
external = [
|
|
repo_surface(
|
|
"modelcontextprotocol_servers",
|
|
MCP_EXTERNAL / "modelcontextprotocol-servers",
|
|
"reference servers for filesystem, git, memory, sequential thinking, fetch, time, and protocol testing",
|
|
95,
|
|
"inactive snapshot only; enable individual servers with explicit allowlists and per-tool receipts",
|
|
),
|
|
repo_surface(
|
|
"modelcontextprotocol_registry",
|
|
MCP_EXTERNAL / "modelcontextprotocol-registry",
|
|
"official registry/API substrate for discovering published MCP servers",
|
|
90,
|
|
"use for catalog discovery only; do not auto-install registry results without security scoring",
|
|
),
|
|
repo_surface(
|
|
"github_mcp_server",
|
|
MCP_EXTERNAL / "github-mcp-server",
|
|
"GitHub issue, PR, repository, workflow, and code intelligence MCP surface",
|
|
88,
|
|
"prefer existing GitHub connector first; use this as pinned implementation/reference until auth and scope filters are explicit",
|
|
),
|
|
repo_surface(
|
|
"awesome_mcp_servers",
|
|
MCP_EXTERNAL / "awesome-mcp-servers",
|
|
"broad community discovery list for future MCP candidates",
|
|
65,
|
|
"discovery only; every candidate must be separately pinned and audited before use",
|
|
),
|
|
repo_surface(
|
|
"kobsidian",
|
|
MCP_EXTERNAL / "kobsidian",
|
|
"filesystem-first Obsidian/Markdown vault server for wiki notes, links, tags, tasks, and LLM-wiki operations",
|
|
91,
|
|
"read-only/wiki-index mode first; write tools disabled until vault path allowlist and TiddlyWiki/ENE receipts exist",
|
|
),
|
|
repo_surface(
|
|
"jupyter_mcp_server",
|
|
MCP_EXTERNAL / "jupyter-mcp-server",
|
|
"Jupyter notebook control surface for iterative Python/math prototyping with notebook context",
|
|
89,
|
|
"sandbox kernel only; no arbitrary host shell; every execution needs notebook path and output hash",
|
|
),
|
|
repo_surface(
|
|
"mcp_python_repl",
|
|
MCP_EXTERNAL / "mcp-python-repl",
|
|
"minimal Python REPL MCP surface for quick computation probes",
|
|
83,
|
|
"use only in sandboxed scratch directory with timeout and no network/secrets",
|
|
),
|
|
repo_surface(
|
|
"mcp_wolfram_alpha",
|
|
MCP_EXTERNAL / "mcp-wolfram-alpha",
|
|
"Wolfram Alpha query surface for external math/facts cross-checking",
|
|
79,
|
|
"requires API credential pointer; use for standard-equation cross-checks, not custom theorem promotion",
|
|
),
|
|
repo_surface(
|
|
"arxiv_mcp_server",
|
|
MCP_EXTERNAL / "arxiv-mcp-server",
|
|
"arXiv search/download surface with Semantic Scholar citation/reference traversal",
|
|
92,
|
|
"research retrieval only; downloaded papers need source hashes and citation receipts before curriculum use",
|
|
),
|
|
repo_surface(
|
|
"sci_hub_mcp_server",
|
|
MCP_EXTERNAL / "sci-hub-mcp-server",
|
|
"Sci-Hub-oriented academic paper MCP surface; useful only as a metadata/search-pattern reference",
|
|
25,
|
|
"HOLD: do not enable PDF download or paywall bypass. Use lawful/local sources first: ScienceHub, Zotero, arXiv, Semantic Scholar, publisher open access, or user-provided PDFs.",
|
|
),
|
|
]
|
|
local = [
|
|
local_surface(
|
|
"sciencehub_mcp",
|
|
REPO / "scripts" / "sciencehub_mcp.py",
|
|
"sovereign research surface for local PDFs, Zotero, arXiv, paper review, and topic fetches",
|
|
96,
|
|
"safe as CLI smoke first; MCP mode requires dependency check and source/receipt outputs",
|
|
sciencehub_report(),
|
|
),
|
|
local_surface(
|
|
"substack_connector_mcp",
|
|
REPO / "plugins" / "substack-connector" / "scripts" / "substack_mcp_server.py",
|
|
"MCP-style Substack publication/update surface",
|
|
72,
|
|
"requires local auth env and no secret echo; publish actions must be explicit",
|
|
),
|
|
local_surface(
|
|
"tardygrada_mcp",
|
|
REPO / "2-Search-Space" / "tardygrada" / "README.md",
|
|
"claim/proof-carrying language that compiles programs to MCP servers",
|
|
82,
|
|
"treat as proof/claim lab; run tests before exposing to shared bus",
|
|
),
|
|
local_surface(
|
|
"claw_mcp_tool_pool",
|
|
REPO / "1-Distributed-Systems" / "agents" / "claw" / "src" / "tools.py",
|
|
"local agent tool-pool mirror with MCP include/deny toggles",
|
|
70,
|
|
"use as compatibility/reference layer only; deny-prefix controls must remain available",
|
|
),
|
|
]
|
|
selected = sorted(external + local, key=lambda item: item["priority"], reverse=True)
|
|
return {
|
|
"schema": "mcp_surface_catalog_receipt_v1",
|
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
"claim_boundary": "MCP surfaces are cataloged as inactive adapters. They are not trusted or enabled until scoped auth, allowlists, sandboxing, and metaprobe receipts pass.",
|
|
"external_snapshots": external,
|
|
"local_surfaces": local,
|
|
"selected_surfaces": selected,
|
|
"bus_rules": [
|
|
"Prefer local read-only or receipt-producing tools before write-capable tools.",
|
|
"Run every MCP surface first in CLI/dry-run mode where possible.",
|
|
"Require source path, source hash, tool name, arguments hash, output hash, lawful flag, and claim boundary in every bus receipt.",
|
|
"Do not auto-install from registries or awesome lists; pin a commit and audit first.",
|
|
"Never pass secrets through model-visible memory; store only credential-store pointers and receipt hashes.",
|
|
"Do not use MCP retrieval surfaces to bypass copyright or access controls; route paper retrieval through lawful/local sources.",
|
|
],
|
|
"openclaw_bridge": {
|
|
"role": "OpenClaw can route MCP surfaces as bus adapters after loopback/sandbox/pairing receipts.",
|
|
"first_candidates": ["sciencehub_mcp", "modelcontextprotocol_servers:git", "modelcontextprotocol_servers:memory", "github_mcp_server"],
|
|
"research_candidates": ["arxiv_mcp_server", "jupyter_mcp_server", "kobsidian"],
|
|
"hold_candidates": ["sci_hub_mcp_server"],
|
|
"defer": ["filesystem write tools", "browser automation", "remote unauthenticated servers", "public inbound channels"],
|
|
},
|
|
"lawful": True,
|
|
}
|
|
|
|
|
|
def curriculum_records(catalog: dict[str, Any]) -> list[dict[str, Any]]:
|
|
system = "You are an MCP surface router. Return compact JSON and keep MCP tools behind receipt gates."
|
|
records = []
|
|
for item in catalog["selected_surfaces"]:
|
|
prompt = {
|
|
"task": "route_mcp_surface",
|
|
"surface_id": item["id"],
|
|
"kind": item["kind"],
|
|
"role": item["role"],
|
|
"priority": item["priority"],
|
|
"gate": item["gate"],
|
|
"claim_boundary": catalog["claim_boundary"],
|
|
}
|
|
answer = {
|
|
"selected": item["priority"] >= 80,
|
|
"use_as": "mcp_bus_surface_prior",
|
|
"surface_id": item["id"],
|
|
"source_path": item["path"],
|
|
"source_hash": item.get("source_hash") or item.get("source_fingerprint"),
|
|
"route_rule": item["gate"],
|
|
"claim_boundary": catalog["claim_boundary"],
|
|
"receipt_rule": "Use only after dry-run or scoped live receipt; never treat MCP output as trusted without source/output hashes.",
|
|
}
|
|
records.append(
|
|
{
|
|
"messages": [
|
|
{"role": "system", "content": system},
|
|
{"role": "user", "content": json.dumps(prompt, ensure_ascii=False)},
|
|
{"role": "assistant", "content": json.dumps(answer, ensure_ascii=False)},
|
|
]
|
|
}
|
|
)
|
|
return records
|
|
|
|
|
|
def write_config(catalog: dict[str, Any], path: Path) -> None:
|
|
config = {
|
|
"$schema": "research_stack_mcp_surface_config_example_v1",
|
|
"claim_boundary": catalog["claim_boundary"],
|
|
"servers": {
|
|
"sciencehub": {
|
|
"command": "python",
|
|
"args": ["scripts/sciencehub_mcp.py"],
|
|
"mode": "stdio",
|
|
"status": "candidate_cli_smoked",
|
|
},
|
|
"substack_connector": {
|
|
"command": "python",
|
|
"args": ["plugins/substack-connector/scripts/substack_mcp_server.py"],
|
|
"mode": "stdio",
|
|
"status": "candidate_requires_auth",
|
|
},
|
|
"mcp_reference_git": {
|
|
"command": "uvx",
|
|
"args": ["mcp-server-git", "--repository", str(REPO)],
|
|
"mode": "stdio",
|
|
"status": "candidate_not_enabled",
|
|
},
|
|
"mcp_reference_memory": {
|
|
"command": "npx",
|
|
"args": ["-y", "@modelcontextprotocol/server-memory"],
|
|
"mode": "stdio",
|
|
"status": "candidate_not_enabled",
|
|
},
|
|
"kobsidian": {
|
|
"command": "npx",
|
|
"args": ["-y", "kobsidian"],
|
|
"mode": "stdio",
|
|
"status": "candidate_not_enabled_requires_vault_allowlist",
|
|
},
|
|
"jupyter_mcp": {
|
|
"command": "python",
|
|
"args": ["-m", "jupyter_mcp_server"],
|
|
"mode": "stdio",
|
|
"status": "candidate_not_enabled_requires_sandbox_kernel",
|
|
},
|
|
"python_repl": {
|
|
"command": "python",
|
|
"args": ["-m", "mcp_python"],
|
|
"mode": "stdio",
|
|
"status": "candidate_not_enabled_requires_sandbox",
|
|
},
|
|
"arxiv": {
|
|
"command": "python",
|
|
"args": ["-m", "arxiv_mcp_server"],
|
|
"mode": "stdio",
|
|
"status": "candidate_not_enabled_research_retrieval",
|
|
},
|
|
"wolfram_alpha": {
|
|
"command": "python",
|
|
"args": ["-m", "mcp_wolfram_alpha"],
|
|
"mode": "stdio",
|
|
"status": "candidate_not_enabled_requires_api_credential_pointer",
|
|
},
|
|
"sci_hub": {
|
|
"command": "python",
|
|
"args": ["sci_hub_server.py"],
|
|
"mode": "stdio",
|
|
"status": "hold_not_enabled_copyright_risk_metadata_reference_only",
|
|
},
|
|
},
|
|
"required_receipt_fields": [
|
|
"surface_id",
|
|
"tool_name",
|
|
"arguments_hash",
|
|
"output_hash",
|
|
"source_path",
|
|
"lawful",
|
|
"claim_boundary",
|
|
],
|
|
}
|
|
path.write_text(json.dumps(config, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
|
|
|
|
def write_wiki(catalog: dict[str, Any], path: Path) -> None:
|
|
lines = [
|
|
"created: 20260507000000000",
|
|
"modified: 20260507000000000",
|
|
"tags: ResearchStack MCP OpenClaw Metaprobe AgentBus",
|
|
"title: MCP Surface Catalog",
|
|
"type: text/vnd.tiddlywiki",
|
|
"",
|
|
"! MCP Surface Catalog",
|
|
"",
|
|
"This catalog pulls and inventories MCP surfaces that can help the OpenClaw/metaprobe shared bus.",
|
|
"",
|
|
"Durable source: `4-Infrastructure/shim/mcp_surface_catalog.py`",
|
|
"",
|
|
"Receipt: `4-Infrastructure/shim/mcp_surface_catalog_receipt.json`",
|
|
"",
|
|
"Curriculum: `4-Infrastructure/shim/mcp_surface_catalog_curriculum.jsonl`",
|
|
"",
|
|
"Config skeleton: `4-Infrastructure/shim/mcp_surface_config.example.json`",
|
|
"",
|
|
"!! Claim Boundary",
|
|
"",
|
|
catalog["claim_boundary"],
|
|
"",
|
|
"!! Selected Surfaces",
|
|
"",
|
|
]
|
|
for item in catalog["selected_surfaces"]:
|
|
lines.append(f"* `{item['id']}` ({item['kind']}, priority {item['priority']}): {item['role']}. Gate: {item['gate']}")
|
|
lines.extend(["", "!! Bus Rules", ""])
|
|
for rule in catalog["bus_rules"]:
|
|
lines.append(f"* {rule}")
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"!! Links",
|
|
"",
|
|
"* [[OpenClaw Shared Bus Surface]]",
|
|
"* [[Physics Math LLM Metaprobe Audit]]",
|
|
"* [[Custom Equation Awareness Manifest]]",
|
|
]
|
|
)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--receipt", type=Path, default=SHIM / "mcp_surface_catalog_receipt.json")
|
|
parser.add_argument("--curriculum", type=Path, default=SHIM / "mcp_surface_catalog_curriculum.jsonl")
|
|
parser.add_argument("--config", type=Path, default=SHIM / "mcp_surface_config.example.json")
|
|
parser.add_argument("--wiki", type=Path, default=WIKI / "MCP Surface Catalog.tid")
|
|
args = parser.parse_args()
|
|
catalog = build_catalog()
|
|
args.receipt.write_text(json.dumps(catalog, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
with args.curriculum.open("w", encoding="utf-8") as handle:
|
|
for record in curriculum_records(catalog):
|
|
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
write_config(catalog, args.config)
|
|
write_wiki(catalog, args.wiki)
|
|
print(json.dumps(catalog, indent=2, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|