allaun/Research-Stack
1
0
Fork 0
mirror of https://github.com/allaunthefox/Research-Stack.git synced 2026-07-31 03:05:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(infra): make lake_build_ingest.py duplicate-safe

Browse source 
Sets content_hash to NULL for build_log packages so the partial unique

index ene_pkg_hash_idx does not prevent multiple ingestions of the same

build output. Each build session now gets its own package row. Also makes

receipt/ingest_event inserts idempotent with ON CONFLICT DO NOTHING.
This commit is contained in:
allaun 2026-06-22 02:48:51 -05:00
parent 779f4fe413
commit 1a44f890c8
1 changed files with 12 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
4-Infrastructure/shim/lake_build_ingest.py
View file

@ -184,10 +184,13 @@ def _insert_session(target: str, git: dict[str, str], build: dict[str, Any], ela
def _insert_package(session_id: str, target: str, git: dict[str, str], build: dict[str, Any]) -> str: def _insert_package(session_id: str, target: str, git: dict[str, str], build: dict[str, Any]) -> str:
"""Insert a package row representing the build artifact and return its id.""" """Insert a package row representing the build artifact and return its id.
pkg_id = hashlib.sha256(f"lake-build:{git['commit']}:{target}".encode()).hexdigest()[:32]
Build logs get content_hash = NULL so the partial unique index
ene_pkg_hash_idx (WHERE content_hash IS NOT NULL) does not fire.
Each build session therefore gets its own package row."""
pkg_id = hashlib.sha256(f"lake-build:{session_id}:{git['commit']}:{target}".encode()).hexdigest()[:32]
content = build["raw_output"] content = build["raw_output"]
content_hash = hashlib.sha256(content.encode()).hexdigest()
title = f"lake build {target}" if target else "lake build" title = f"lake build {target}" if target else "lake build"
sql = f""" sql = f"""
INSERT INTO ene.packages INSERT INTO ene.packages
@ -196,20 +199,13 @@ def _insert_package(session_id: str, target: str, git: dict[str, str], build: di
verification_status, promotion_state, domain, archetype, notes) verification_status, promotion_state, domain, archetype, notes)
VALUES VALUES
('{pkg_id}', 'build_log', '{title}', '{_escape_sql(content)}', ('{pkg_id}', 'build_log', '{title}', '{_escape_sql(content)}',
'{content_hash}', 'METAL', NULL, 'METAL',
'["lake", "build", "lean"]'::jsonb, '["lake", "build", "lean"]'::jsonb,
'4-Infrastructure/shim/lake_build_ingest.py', '4-Infrastructure/shim/lake_build_ingest.py',
'{_now()}', '{_now()}', '{session_id}', '{_now()}', '{_now()}', '{session_id}',
'{{"git_commit": "{git['commit']}", "git_branch": "{git['branch']}", "dirty": {str(git['dirty']).lower()}}}'::jsonb, '{{"git_commit": "{git['commit']}", "git_branch": "{git['branch']}", "dirty": {str(git['dirty']).lower()}}}'::jsonb,
'{build['status']}', 'held', 'core-formalism', 'build_artifact', '{build['status']}', 'held', 'core-formalism', 'build_artifact',
'Build output captured from lake build {target}') 'Build output captured from lake build {target}');
ON CONFLICT (pkg) DO UPDATE SET
content = EXCLUDED.content,
content_hash = EXCLUDED.content_hash,
updated_at = EXCLUDED.updated_at,
session_id = EXCLUDED.session_id,
verification_status = EXCLUDED.verification_status,
notes = EXCLUDED.notes;
""" """
_ssh_psql(sql) _ssh_psql(sql)
return pkg_id return pkg_id
@ -230,7 +226,8 @@ def _insert_receipt(pkg_id: str, session_id: str, build: dict[str, Any]) -> None
VALUES VALUES
('{pkg_id}', 'lake_build', '{receipt_hash}', '{input_hash}', ('{pkg_id}', 'lake_build', '{receipt_hash}', '{input_hash}',
'{receipt_hash}', 'lake build', 'opencode', '{build['status']}', '{receipt_hash}', 'lake build', 'opencode', '{build['status']}',
'{residual}'::jsonb); '{residual}'::jsonb)
ON CONFLICT (package_id, receipt_type, receipt_hash) DO NOTHING;
""" """
_ssh_psql(sql) _ssh_psql(sql)
@ -246,7 +243,8 @@ def _insert_ingest_event(pkg_id: str, session_id: str) -> None:
('{pkg_id}', 'file://4-Infrastructure/shim/lake_build_ingest.py', ('{pkg_id}', 'file://4-Infrastructure/shim/lake_build_ingest.py',
'build_log_ingest', '', 'lake_build_ingest_v1', '1.0.0', 'build_log_ingest', '', 'lake_build_ingest_v1', '1.0.0',
'{{"session_id": "{session_id}"}}'::jsonb, '[]'::jsonb, '{{"session_id": "{session_id}"}}'::jsonb, '[]'::jsonb,
'[]'::jsonb, '[]'::jsonb, '[]'::jsonb); '[]'::jsonb, '[]'::jsonb, '[]'::jsonb)
ON CONFLICT DO NOTHING;
""" """
_ssh_psql(sql) _ssh_psql(sql)