From 81b62532611f996d3b57ebbbc8b9b721f0bdaf82 Mon Sep 17 00:00:00 2001 From: allaun Date: Sun, 21 Jun 2026 20:59:24 -0500 Subject: [PATCH] fix(infra): optimize drive compression memory usage and download skip Check file sizes before loading bytes into memory and stream large files directly into the tarball to prevent OOM errors on Steam Deck. Skip downloading files that already exist locally. Build: 0 errors (py_compile) --- 4-Infrastructure/AGENTS.md | 2 +- .../shim/mcp_drive_gccl_compress.py | 42 ++++++++++++++----- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/4-Infrastructure/AGENTS.md b/4-Infrastructure/AGENTS.md index b82381e0..b98946bb 100644 --- a/4-Infrastructure/AGENTS.md +++ b/4-Infrastructure/AGENTS.md @@ -313,7 +313,7 @@ python3 4-Infrastructure/storage/storage_agent.py --loop --interval 900 ## Current Stack-Solidification Anchors -- `4-Infrastructure/shim/mcp_drive_gccl_compress.py` — MCP-only Google Drive folder compressor: downloads a Drive folder via `@piotr-agier/google-drive-mcp`, GCCL-delta-compresses contents into a `.tar.zst` archive with JSON manifest, uploads archive + manifest back to Drive, optionally trashes the original. No rclone. +- `4-Infrastructure/shim/mcp_drive_gccl_compress.py` — MCP-only Google Drive folder compressor: downloads a Drive folder via `@piotr-agier/google-drive-mcp`, GCCL-delta-compresses contents into a `.tar.zst` archive with JSON manifest, uploads archive + manifest back to Drive, optionally trashes the original. Optimized to skip existing local files and stream large files (>100MB) directly to prevent OOM. No rclone. - `4-Infrastructure/shim/rrc_arxiv_kernel_refine.py` — RRC arXiv kernel refinement: title+abstract keyword search against arxiv_papers for unmatched equations - `4-Infrastructure/infra/lean_lsp_mcp_wrapper.py` — Python wrapper for `lean-lsp-mcp` to fix the schema of `lean_diagnostic_messages` for compatibility with strict validators like Moonshot/Kimi API. - `4-Infrastructure/shim/stack_solidification_audit.py` diff --git a/4-Infrastructure/shim/mcp_drive_gccl_compress.py b/4-Infrastructure/shim/mcp_drive_gccl_compress.py index 19e5a6fc..78fe10e8 100644 --- a/4-Infrastructure/shim/mcp_drive_gccl_compress.py +++ b/4-Infrastructure/shim/mcp_drive_gccl_compress.py @@ -205,8 +205,22 @@ def gccl_compress_file( src_path: Path, reference_path: Optional[Path] = None, threshold_q16: int = 32768, + max_gccl_size: int = 100 * 1024 * 1024, ) -> Tuple[bytes, Dict[str, Any]]: - """Compress a single file with GCCL. Optionally use reference for delta.""" + """Compress a single file with GCCL. Optionally use reference for delta. + Files larger than max_gccl_size are stored raw to avoid memory issues.""" + original_size = src_path.stat().st_size + + if original_size > max_gccl_size: + return b"", { + "mode": "raw-oversize", + "reference": str(reference_path) if reference_path else None, + "original_size": original_size, + "compressed_size": original_size, + "ratio": 1.0, + "decision": "OVERSIZE", + } + data = src_path.read_bytes() reference = reference_path.read_bytes() if reference_path else None @@ -265,8 +279,11 @@ def _download_folder_recursive( if item["type"] == "folder": _download_folder_recursive(client, item["id"], local_path, files, rename_map) else: - print(f" downloading {safe_name} ...") - client.download_file(item["id"], local_path) + if local_path.exists(): + print(f" [skip] {safe_name} already exists locally") + else: + print(f" downloading {safe_name} ...") + client.download_file(item["id"], local_path) files.append({ "drive_id": item["id"], "name": safe_name, @@ -299,6 +316,7 @@ def compress_folder( local_root: Path, archive_path: Path, threshold_q16: int = 32768, + max_gccl_size: int = 100 * 1024 * 1024, ) -> Dict[str, Any]: """GCCL-compress downloaded files and pack into a .tar.zst archive.""" refs = build_reference_index(files, local_root) @@ -316,13 +334,16 @@ def compress_folder( rel = src.relative_to(local_root) ref = refs.get(f["local_path"]) - compressed_bytes, meta = gccl_compress_file(src, ref, threshold_q16) + compressed_bytes, meta = gccl_compress_file(src, ref, threshold_q16, max_gccl_size) - arc_name = f"{rel}.gccl" if meta["mode"] != "raw" else str(rel) - info = tarfile.TarInfo(name=arc_name) - info.size = len(compressed_bytes) - info.mtime = time.time() - tar.addfile(info, io.BytesIO(compressed_bytes)) + arc_name = f"{rel}.gccl" if meta["mode"] not in ("raw", "raw-oversize") else str(rel) + if meta["mode"] in ("raw", "raw-oversize"): + tar.add(src, arcname=arc_name) + else: + info = tarfile.TarInfo(name=arc_name) + info.size = len(compressed_bytes) + info.mtime = time.time() + tar.addfile(info, io.BytesIO(compressed_bytes)) manifest["files"].append({ "path": str(rel), @@ -367,6 +388,7 @@ def main() -> int: parser.add_argument("--upload-parent", help="Drive folder ID to upload archive into") parser.add_argument("--trash-original", action="store_true", help="Move original folder to trash after upload") parser.add_argument("--threshold-q16", type=int, default=32768, help="GCCL threshold in Q16.16 (default 32768 = 0.5)") + parser.add_argument("--max-gccl-size", type=int, default=100*1024*1024, help="Max bytes for GCCL compression; larger files stored raw (default 100 MiB)") parser.add_argument("--dry-run", action="store_true", help="Download and compress but do not upload or trash") args = parser.parse_args() @@ -393,7 +415,7 @@ def main() -> int: return 1 print(f"[*] GCCL-compressing into {archive_path}") - manifest = compress_folder(files, local_clone, archive_path, args.threshold_q16) + manifest = compress_folder(files, local_clone, archive_path, args.threshold_q16, args.max_gccl_size) print(f"[+] Archive: {archive_path} ({archive_path.stat().st_size} bytes)") print(f"[+] Manifest: {manifest_path}") print(f"[+] SI compression ratio: {manifest['summary']['si_compression_ratio']:.2f}")