diff --git a/4-Infrastructure/AGENTS.md b/4-Infrastructure/AGENTS.md index b82381e0..b98946bb 100644 --- a/4-Infrastructure/AGENTS.md +++ b/4-Infrastructure/AGENTS.md @@ -313,7 +313,7 @@ python3 4-Infrastructure/storage/storage_agent.py --loop --interval 900 ## Current Stack-Solidification Anchors -- `4-Infrastructure/shim/mcp_drive_gccl_compress.py` — MCP-only Google Drive folder compressor: downloads a Drive folder via `@piotr-agier/google-drive-mcp`, GCCL-delta-compresses contents into a `.tar.zst` archive with JSON manifest, uploads archive + manifest back to Drive, optionally trashes the original. No rclone. +- `4-Infrastructure/shim/mcp_drive_gccl_compress.py` — MCP-only Google Drive folder compressor: downloads a Drive folder via `@piotr-agier/google-drive-mcp`, GCCL-delta-compresses contents into a `.tar.zst` archive with JSON manifest, uploads archive + manifest back to Drive, optionally trashes the original. Optimized to skip existing local files and stream large files (>100MB) directly to prevent OOM. No rclone. - `4-Infrastructure/shim/rrc_arxiv_kernel_refine.py` — RRC arXiv kernel refinement: title+abstract keyword search against arxiv_papers for unmatched equations - `4-Infrastructure/infra/lean_lsp_mcp_wrapper.py` — Python wrapper for `lean-lsp-mcp` to fix the schema of `lean_diagnostic_messages` for compatibility with strict validators like Moonshot/Kimi API. - `4-Infrastructure/shim/stack_solidification_audit.py` diff --git a/4-Infrastructure/shim/mcp_drive_gccl_compress.py b/4-Infrastructure/shim/mcp_drive_gccl_compress.py index 19e5a6fc..78fe10e8 100644 --- a/4-Infrastructure/shim/mcp_drive_gccl_compress.py +++ b/4-Infrastructure/shim/mcp_drive_gccl_compress.py @@ -205,8 +205,22 @@ def gccl_compress_file( src_path: Path, reference_path: Optional[Path] = None, threshold_q16: int = 32768, + max_gccl_size: int = 100 * 1024 * 1024, ) -> Tuple[bytes, Dict[str, Any]]: - """Compress a single file with GCCL. Optionally use reference for delta.""" + """Compress a single file with GCCL. Optionally use reference for delta. + Files larger than max_gccl_size are stored raw to avoid memory issues.""" + original_size = src_path.stat().st_size + + if original_size > max_gccl_size: + return b"", { + "mode": "raw-oversize", + "reference": str(reference_path) if reference_path else None, + "original_size": original_size, + "compressed_size": original_size, + "ratio": 1.0, + "decision": "OVERSIZE", + } + data = src_path.read_bytes() reference = reference_path.read_bytes() if reference_path else None @@ -265,8 +279,11 @@ def _download_folder_recursive( if item["type"] == "folder": _download_folder_recursive(client, item["id"], local_path, files, rename_map) else: - print(f" downloading {safe_name} ...") - client.download_file(item["id"], local_path) + if local_path.exists(): + print(f" [skip] {safe_name} already exists locally") + else: + print(f" downloading {safe_name} ...") + client.download_file(item["id"], local_path) files.append({ "drive_id": item["id"], "name": safe_name, @@ -299,6 +316,7 @@ def compress_folder( local_root: Path, archive_path: Path, threshold_q16: int = 32768, + max_gccl_size: int = 100 * 1024 * 1024, ) -> Dict[str, Any]: """GCCL-compress downloaded files and pack into a .tar.zst archive.""" refs = build_reference_index(files, local_root) @@ -316,13 +334,16 @@ def compress_folder( rel = src.relative_to(local_root) ref = refs.get(f["local_path"]) - compressed_bytes, meta = gccl_compress_file(src, ref, threshold_q16) + compressed_bytes, meta = gccl_compress_file(src, ref, threshold_q16, max_gccl_size) - arc_name = f"{rel}.gccl" if meta["mode"] != "raw" else str(rel) - info = tarfile.TarInfo(name=arc_name) - info.size = len(compressed_bytes) - info.mtime = time.time() - tar.addfile(info, io.BytesIO(compressed_bytes)) + arc_name = f"{rel}.gccl" if meta["mode"] not in ("raw", "raw-oversize") else str(rel) + if meta["mode"] in ("raw", "raw-oversize"): + tar.add(src, arcname=arc_name) + else: + info = tarfile.TarInfo(name=arc_name) + info.size = len(compressed_bytes) + info.mtime = time.time() + tar.addfile(info, io.BytesIO(compressed_bytes)) manifest["files"].append({ "path": str(rel), @@ -367,6 +388,7 @@ def main() -> int: parser.add_argument("--upload-parent", help="Drive folder ID to upload archive into") parser.add_argument("--trash-original", action="store_true", help="Move original folder to trash after upload") parser.add_argument("--threshold-q16", type=int, default=32768, help="GCCL threshold in Q16.16 (default 32768 = 0.5)") + parser.add_argument("--max-gccl-size", type=int, default=100*1024*1024, help="Max bytes for GCCL compression; larger files stored raw (default 100 MiB)") parser.add_argument("--dry-run", action="store_true", help="Download and compress but do not upload or trash") args = parser.parse_args() @@ -393,7 +415,7 @@ def main() -> int: return 1 print(f"[*] GCCL-compressing into {archive_path}") - manifest = compress_folder(files, local_clone, archive_path, args.threshold_q16) + manifest = compress_folder(files, local_clone, archive_path, args.threshold_q16, args.max_gccl_size) print(f"[+] Archive: {archive_path} ({archive_path.stat().st_size} bytes)") print(f"[+] Manifest: {manifest_path}") print(f"[+] SI compression ratio: {manifest['summary']['si_compression_ratio']:.2f}")