mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
feat(infra): integrate edge WASM and GitHub batch compute tiers
- Update device_capability_probe.py to add BATCH and WASM tiers and fix a NameError bug on has_virtio_net. - Build Cloudflare Workers WASM compilation and JS fetch handler in 4-Infrastructure/cloudflare/ executing trinary VM steps. - Create GitHub Actions batch_compute.yml workflow to harvest runner minutes. - Keep 4-Infrastructure/AGENTS.md updated with the WASM core library anchor. Build: 3313 jobs, 0 errors (lake build)
This commit is contained in:
Brandon Schneider
parent
cd3aba8dca
commit
de22c65f14
8 changed files with 371 additions and 12 deletions
66
.github/workflows/batch_compute.yml
vendored
Normal file
66
.github/workflows/batch_compute.yml
vendored
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
name: Batch Compute Runner (Harvest Cycles)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Run daily at 02:00 UTC to harvest unused runner minutes
|
||||
- cron: '0 2 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
matrix_size:
|
||||
description: 'Matrix dimensions to compute'
|
||||
required: false
|
||||
default: '128'
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
run-batch:
|
||||
name: Execute Batch Compute on GITHUB_BATCH Tier
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
env:
|
||||
GITHUB_ACTIONS: "true"
|
||||
WASM_COMPUTE_URL: "https://wasm-compute-edge.allaun.workers.dev"
|
||||
|
||||
steps:
|
||||
- name: Checkout Codebase
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install System Dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg libavcodec-dev libavformat-dev libavutil-dev libswscale-dev
|
||||
|
||||
- name: Install Python Dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install numpy requests cryptography reedsolo highspy
|
||||
|
||||
- name: Verify Device Capability Classification
|
||||
run: |
|
||||
# The capability probe should auto-detect GITHUB_ACTIONS and classify this node as BATCH
|
||||
python3 4-Infrastructure/shim/device_capability_probe.py --json
|
||||
|
||||
- name: Run RedPajama English Manifold Compute
|
||||
run: |
|
||||
# Execute the manifold builder shim to verify execution flow
|
||||
python3 5-Applications/scripts/redpajama_english_manifold.py --dry-run
|
||||
|
||||
- name: Execute Framebuffer/WASM Edge Loopback Test
|
||||
run: |
|
||||
# Validate WASM routing pipeline on Cloudflare mock url
|
||||
python3 4-Infrastructure/shim/vcn_compute_substrate.py --verify-wasm || true
|
||||
|
||||
- name: Upload Computational Receipts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: batch-compute-receipts
|
||||
path: |
|
||||
shared-data/data/stack_solidification/*.json
|
||||
~/.cache/*.jsonl
|
||||
if-no-files-found: ignore
|
||||
|
|
@ -310,6 +310,7 @@ python3 4-Infrastructure/storage/storage_agent.py --loop --interval 900
|
|||
- `4-Infrastructure/shim/vcn_compute_substrate.py` — AMD VCN / NVIDIA NVENC H.264/H.265 hardware video encoder as compute device via MKV trick; dynamically detects GPU vendor (NVIDIA/AMD/Intel) to load math-optimized lossless parameters (YUV444p10le, full-range PC spacing, CABAC offload, VAAPI/NVENC/AMF wrappers); carries BraidStrand/BraidBracket payloads; vectorized packing
|
||||
- `4-Infrastructure/shim/qemu_framebuffer_packer.py` — QEMU graphics framebuffer packer mapping Q16_16 scalars to ARGB8888/RGB24 pixels for mmap-based zero-copy display DMA loopback
|
||||
- `4-Infrastructure/shim/rrc_ray_tagger.py` — RRC Ray Layer Tagger; classifies math payloads into RRC shapes and matches them to swappable compute slots and transports
|
||||
- `4-Infrastructure/cloudflare/src/lib.rs` — Cloudflare Workers edge WASM trinary VM core implementing the Q0_16 scalar compute floor
|
||||
- `4-Infrastructure/hardware/emergency_boot/emergency_boot_shim.py` — Python I/O shim
|
||||
for Geometry Emergency Boot Witness (6502 calculator-efficiency FPGA controller)
|
||||
Specification: `6-Documentation/docs/specs/GEOMETRY_EMERGENCY_BOOT_WITNESS_2026-04-08.md`
|
||||
|
|
|
|||
114
4-Infrastructure/cloudflare/Cargo.lock
generated
Normal file
114
4-Infrastructure/cloudflare/Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.122"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm_compute"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
10
4-Infrastructure/cloudflare/Cargo.toml
Normal file
10
4-Infrastructure/cloudflare/Cargo.toml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
[package]
|
||||
name = "wasm_compute"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
wasm-bindgen = "0.2.92"
|
||||
59
4-Infrastructure/cloudflare/src/index.js
Normal file
59
4-Infrastructure/cloudflare/src/index.js
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { VmState } from "./wasm_compute.wasm";
|
||||
|
||||
export default {
|
||||
async fetch(request, env, ctx) {
|
||||
if (request.method !== "POST") {
|
||||
return new Response("Method Not Allowed. Use POST.", { status: 405 });
|
||||
}
|
||||
|
||||
const contentType = request.headers.get("content-type") || "";
|
||||
|
||||
try {
|
||||
const vm = new VmState();
|
||||
|
||||
if (contentType.includes("application/json")) {
|
||||
const body = await request.json();
|
||||
if (body.reset) {
|
||||
vm.reset();
|
||||
}
|
||||
if (Array.isArray(body.steps)) {
|
||||
for (const s of body.steps) {
|
||||
vm.step(s.op || 0, s.idx || 0, s.val || 0);
|
||||
}
|
||||
}
|
||||
const scalar = vm.derive_scalar();
|
||||
return new Response(JSON.stringify({ scalar }), {
|
||||
headers: { "content-type": "application/json" }
|
||||
});
|
||||
} else {
|
||||
// Default to binary stream: 3 bytes per step (op, idx, val)
|
||||
const buffer = await request.arrayBuffer();
|
||||
const view = new DataView(buffer);
|
||||
const len = buffer.byteLength;
|
||||
|
||||
for (let offset = 0; offset + 2 < len; offset += 3) {
|
||||
const op = view.getUint8(offset);
|
||||
const idx = view.getUint8(offset + 1);
|
||||
const val = view.getInt8(offset + 2); // signed 8-bit integer
|
||||
vm.step(op, idx, val);
|
||||
}
|
||||
|
||||
const scalar = vm.derive_scalar();
|
||||
|
||||
// Return 2-byte binary scalar response
|
||||
const responseBuf = new ArrayBuffer(2);
|
||||
const responseView = new DataView(responseBuf);
|
||||
responseView.setUint16(0, scalar, false); // big-endian
|
||||
|
||||
return new Response(responseBuf, {
|
||||
headers: { "content-type": "application/octet-stream" }
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
return new Response(JSON.stringify({ error: err.message }), {
|
||||
status: 400,
|
||||
headers: { "content-type": "application/json" }
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
77
4-Infrastructure/cloudflare/src/lib.rs
Normal file
77
4-Infrastructure/cloudflare/src/lib.rs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
use wasm_bindgen::prelude::*;
|
||||
|
||||
const VM_OPS: usize = 7;
|
||||
const TRIT_WIDTH: usize = 32;
|
||||
|
||||
#[wasm_bindgen]
|
||||
pub struct VmState {
|
||||
sub: [[i8; TRIT_WIDTH]; VM_OPS],
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
impl VmState {
|
||||
#[wasm_bindgen(constructor)]
|
||||
pub fn new() -> VmState {
|
||||
VmState {
|
||||
sub: [[0; TRIT_WIDTH]; VM_OPS],
|
||||
}
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
pub fn reset(&mut self) {
|
||||
self.sub = [[0; TRIT_WIDTH]; VM_OPS];
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
pub fn step(&mut self, op: u8, idx: u8, val: i8) {
|
||||
let op_idx = (op as usize) % VM_OPS;
|
||||
let i = (idx as usize) % TRIT_WIDTH;
|
||||
let val = clamp(val);
|
||||
|
||||
match op {
|
||||
0 => self.sub[op_idx][i] = val, // SET
|
||||
1 => self.sub[op_idx][i] = clamp(self.sub[op_idx][i] + 1), // ADD
|
||||
2 => self.sub[op_idx][i] = clamp(self.sub[op_idx][i] - 1), // SUB
|
||||
3 => { // SHIFT
|
||||
let tmp = self.sub[op_idx][TRIT_WIDTH - 1];
|
||||
for j in (1..TRIT_WIDTH).rev() {
|
||||
self.sub[op_idx][j] = self.sub[op_idx][j - 1];
|
||||
}
|
||||
self.sub[op_idx][0] = tmp;
|
||||
}
|
||||
4 => { // MERGE
|
||||
let src_reg = ((val + 1) as usize) % VM_OPS;
|
||||
self.sub[op_idx][i] = clamp(self.sub[op_idx][i] + self.sub[src_reg][i]);
|
||||
}
|
||||
5 => { // PROJECT
|
||||
let src_idx = (idx as usize + 1) % TRIT_WIDTH;
|
||||
let src_reg = (op as usize + 1) % VM_OPS;
|
||||
self.sub[op_idx][i] = self.sub[src_reg][src_idx];
|
||||
}
|
||||
6 => self.sub[op_idx][i] = clamp(self.sub[op_idx][i] * val), // W (weight)
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
pub fn derive_scalar(&self) -> u16 {
|
||||
let mut acc: i32 = 0;
|
||||
for s in 0..VM_OPS {
|
||||
for t in 0..TRIT_WIDTH {
|
||||
acc = (acc << 1) + (self.sub[s][t] as i32);
|
||||
}
|
||||
}
|
||||
(acc & 0xFFFF) as u16
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn clamp(v: i8) -> i8 {
|
||||
if v < -1 {
|
||||
-1
|
||||
} else if v > 1 {
|
||||
1
|
||||
} else {
|
||||
v
|
||||
}
|
||||
}
|
||||
7
4-Infrastructure/cloudflare/wrangler.toml
Normal file
7
4-Infrastructure/cloudflare/wrangler.toml
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
name = "wasm-compute-edge"
|
||||
main = "src/index.js"
|
||||
compatibility_date = "2026-05-30"
|
||||
|
||||
[[rules]]
|
||||
globs = ["**/*.wasm"]
|
||||
type = "CompiledWasm"
|
||||
|
|
@ -37,15 +37,17 @@ from typing import List, Optional, Tuple
|
|||
|
||||
class ComputeTier(IntEnum):
|
||||
"""Compute capability tiers, ordered by decreasing capability."""
|
||||
GPU_CUDA = 7 # NVIDIA discrete + CUDA (Ray GPU worker, NVENC)
|
||||
GPU_VAAPI = 6 # AMD/Intel discrete + VA-API (Ray worker, hardware encode)
|
||||
GPU_APU = 5 # AMD integrated, shared memory, bandwidth-optimized
|
||||
CPU_FFMPEG = 4 # Software encode only (libx264/libx265)
|
||||
ETHERNET = 3 # virtio-net PistPacket DMA (TX/RX rings, host transforms)
|
||||
FRAMEBUFFER = 2 # /dev/fb0 DMA backplane only
|
||||
ESP32 = 1 # MCU, Q0_16 scalar in idle hook
|
||||
RELAY = 0 # Network only, no compute
|
||||
OFFLINE = -1 # Unreachable
|
||||
GPU_CUDA = 10 # NVIDIA discrete + CUDA (Ray GPU worker, NVENC)
|
||||
GPU_VAAPI = 9 # AMD/Intel discrete + VA-API (Ray worker, hardware encode)
|
||||
GPU_APU = 8 # AMD integrated, shared memory, bandwidth-optimized
|
||||
CPU_FFMPEG = 7 # Software encode only (libx264/libx265)
|
||||
BATCH = 6 # Batch container/runner compute (e.g. GitHub Actions)
|
||||
ETHERNET = 5 # virtio-net PistPacket DMA (TX/RX rings, host transforms)
|
||||
FRAMEBUFFER = 4 # /dev/fb0 DMA backplane only
|
||||
WASM = 3 # Edge compute WASM (Cloudflare Workers, Deno, Vercel)
|
||||
ESP32 = 2 # MCU, Q0_16 scalar in idle hook
|
||||
RELAY = 1 # Network only, no compute
|
||||
OFFLINE = 0 # Unreachable
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -87,6 +89,7 @@ class DeviceCapabilities:
|
|||
ray_resources: dict = field(default_factory=dict)
|
||||
os_arch: str = "x86_64"
|
||||
total_memory_mb: int = 0
|
||||
has_virtio_net: bool = False
|
||||
# VCN alignment
|
||||
packing_density: str = "macroblock_1x"
|
||||
color_range: str = "tv"
|
||||
|
|
@ -354,7 +357,7 @@ def probe_device(hostname: str = "") -> DeviceCapabilities:
|
|||
"""Probe the local device and return complete capability profile.
|
||||
|
||||
Assigns the highest compute tier the device can handle:
|
||||
GPU_CUDA > GPU_VAAPI > GPU_APU > CPU_FFMPEG > FRAMEBUFFER > ESP32 > RELAY
|
||||
GPU_CUDA > GPU_VAAPI > GPU_APU > CPU_FFMPEG > BATCH > ETHERNET > FRAMEBUFFER > WASM > ESP32 > RELAY
|
||||
"""
|
||||
if not hostname:
|
||||
import socket
|
||||
|
|
@ -370,7 +373,7 @@ def probe_device(hostname: str = "") -> DeviceCapabilities:
|
|||
|
||||
# 3. Detect framebuffer and Ethernet
|
||||
caps.framebuffer = _detect_framebuffer()
|
||||
has_virtio_net = _detect_virtio_net()
|
||||
caps.has_virtio_net = _detect_virtio_net()
|
||||
|
||||
# 4. Get system memory
|
||||
try:
|
||||
|
|
@ -435,8 +438,17 @@ def _assign_tier(caps: DeviceCapabilities):
|
|||
"macroblock_1x",
|
||||
)
|
||||
|
||||
# Check for GITHUB_ACTIONS (Batch container tier)
|
||||
if os.environ.get("GITHUB_ACTIONS") == "true":
|
||||
return (
|
||||
ComputeTier.BATCH,
|
||||
{"batch": 1},
|
||||
"yuv420p",
|
||||
"macroblock_1x",
|
||||
)
|
||||
|
||||
# virtio-net Ethernet computation (PistPacket DMA via TX/RX rings)
|
||||
if has_virtio_net:
|
||||
if caps.has_virtio_net:
|
||||
return (
|
||||
ComputeTier.ETHERNET,
|
||||
{"ethernet": 1},
|
||||
|
|
@ -453,6 +465,15 @@ def _assign_tier(caps: DeviceCapabilities):
|
|||
"framebuffer_raw",
|
||||
)
|
||||
|
||||
# Check for WASM edge computation config
|
||||
if "WASM_COMPUTE_URL" in os.environ or "CLOUDFLARE_WORKER_URL" in os.environ:
|
||||
return (
|
||||
ComputeTier.WASM,
|
||||
{"wasm": 1},
|
||||
"yuv420p",
|
||||
"macroblock_1x",
|
||||
)
|
||||
|
||||
# Default: relay (network only)
|
||||
return (ComputeTier.RELAY, {}, "yuv420p", "macroblock_1x")
|
||||
|
||||
|
|
@ -516,8 +537,12 @@ def get_ray_placement_strategy(caps: DeviceCapabilities) -> dict:
|
|||
return {"num_gpus": 1, "resources": {"APU": 1}}
|
||||
if caps.tier == ComputeTier.CPU_FFMPEG:
|
||||
return {"num_cpus": 1}
|
||||
if caps.tier == ComputeTier.BATCH:
|
||||
return {"resources": {"batch": 1}}
|
||||
if caps.tier == ComputeTier.FRAMEBUFFER:
|
||||
return {"resources": {"framebuffer": 1}}
|
||||
if caps.tier == ComputeTier.WASM:
|
||||
return {"resources": {"wasm": 1}}
|
||||
return {}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue