diff --git a/5-Applications/caddy-edge/.gitignore b/5-Applications/caddy-edge/.gitignore new file mode 100644 index 00000000..90f42c63 --- /dev/null +++ b/5-Applications/caddy-edge/.gitignore @@ -0,0 +1,2 @@ +caddy +caddy.tar.gz diff --git a/5-Applications/caddy-edge/Caddyfile b/5-Applications/caddy-edge/Caddyfile new file mode 100644 index 00000000..37166426 --- /dev/null +++ b/5-Applications/caddy-edge/Caddyfile @@ -0,0 +1,29 @@ +{ + email allaun@researchstack.info + default_sni researchstack.info + acme_ca https://acme-staging-v02.api.letsencrypt.org/directory +} + +(porkbun_tls) { + tls { + dns porkbun { + api_key {$PORKBUN_API_KEY} + api_secret_key {$PORKBUN_SECRET_KEY} + } + } +} + +researchstack.info { + import porkbun_tls + reverse_proxy 100.102.173.61:30080 +} + +auth.researchstack.info { + import porkbun_tls + reverse_proxy 100.102.173.61:30080 +} + +*.researchstack.info { + import porkbun_tls + redir https://researchstack.info{uri} 301 +} diff --git a/5-Applications/caddy-edge/Containerfile b/5-Applications/caddy-edge/Containerfile new file mode 100644 index 00000000..5d9617d0 --- /dev/null +++ b/5-Applications/caddy-edge/Containerfile @@ -0,0 +1,6 @@ +FROM docker.io/library/caddy:2.10-builder AS builder +RUN xcaddy build --with github.com/caddy-dns/porkbun + +FROM docker.io/library/caddy:2.10 +COPY --from=builder /usr/bin/caddy /usr/bin/caddy +EXPOSE 80 443 2019 diff --git a/5-Applications/caddy-edge/caddy.json b/5-Applications/caddy-edge/caddy.json new file mode 100644 index 00000000..f141cf31 --- /dev/null +++ b/5-Applications/caddy-edge/caddy.json @@ -0,0 +1,87 @@ +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [":443"], + "routes": [ + { + "match": [{"host": ["researchstack.info"]}], + "handle": [ + { + "handler": "reverse_proxy", + "upstreams": [{"dial": "100.102.173.61:30080"}] + } + ], + "terminal": true + }, + { + "match": [{"host": ["auth.researchstack.info"]}], + "handle": [ + { + "handler": "reverse_proxy", + "upstreams": [{"dial": "100.102.173.61:30080"}] + } + ], + "terminal": true + }, + { + "match": [{"host": ["*.researchstack.info"]}], + "handle": [ + { + "handler": "static_response", + "status_code": 301, + "headers": { + "Location": ["https://researchstack.info{http.request.uri}"] + } + } + ], + "terminal": true + } + ] + }, + "srv_redirect": { + "listen": [":80"], + "routes": [ + { + "handle": [ + { + "handler": "static_response", + "status_code": 301, + "headers": { + "Location": ["https://{http.request.host}{http.request.uri}"] + } + } + ] + } + ] + } + } + }, + "tls": { + "automation": { + "policies": [ + { + "subjects": ["researchstack.info", "*.researchstack.info"], + "issuers": [ + { + "module": "acme", + "ca": "https://acme-v02.api.letsencrypt.org/directory", + "email": "allaun@researchstack.info", + "challenges": { + "dns": { + "provider": { + "name": "porkbun", + "api_key": "{env.PORKBUN_API_KEY}", + "api_secret_key": "{env.PORKBUN_SECRET_KEY}" + } + } + } + } + ] + } + ] + } + } + } +} diff --git a/5-Applications/caddy-edge/k8s/caddy.yaml b/5-Applications/caddy-edge/k8s/caddy.yaml new file mode 100644 index 00000000..b1d24818 --- /dev/null +++ b/5-Applications/caddy-edge/k8s/caddy.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: edge +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: caddy-data-v2 + namespace: edge +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: caddy-edge + namespace: edge + labels: + app: caddy-edge +spec: + replicas: 1 + selector: + matchLabels: + app: caddy-edge + template: + metadata: + labels: + app: caddy-edge + spec: + hostNetwork: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/hostname: 361395-1 + containers: + - name: caddy + image: localhost/caddy-edge:latest + imagePullPolicy: Never + command: ["caddy"] + args: ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + env: + - name: PORKBUN_API_KEY + valueFrom: + secretKeyRef: + name: porkbun-credentials + key: api-key + - name: PORKBUN_SECRET_KEY + valueFrom: + secretKeyRef: + name: porkbun-credentials + key: api-secret-key + volumeMounts: + - name: caddy-config + mountPath: /etc/caddy + readOnly: true + - name: caddy-data + mountPath: /data + - name: caddy-config-storage + mountPath: /config + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "250m" + readinessProbe: + tcpSocket: + port: 443 + initialDelaySeconds: 15 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 443 + initialDelaySeconds: 20 + periodSeconds: 30 + volumes: + - name: caddy-config + configMap: + name: caddy-caddyfile + - name: caddy-data + persistentVolumeClaim: + claimName: caddy-data-v2 + - name: caddy-config-storage + emptyDir: {} diff --git a/5-Applications/webrtc-bridge/Containerfile b/5-Applications/webrtc-bridge/Containerfile new file mode 100644 index 00000000..41886d85 --- /dev/null +++ b/5-Applications/webrtc-bridge/Containerfile @@ -0,0 +1,13 @@ +FROM docker.io/library/golang:1.23-alpine AS builder +WORKDIR /build +COPY go.mod ./ +RUN go mod download || true +COPY signaling-server.go . +RUN go mod tidy && CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /webrtc-bridge . + +FROM docker.io/library/alpine:3.20 +RUN apk add --no-cache ca-certificates +COPY --from=builder /webrtc-bridge /usr/local/bin/webrtc-bridge +COPY client/ /app/client/ +EXPOSE 8080 +ENTRYPOINT ["webrtc-bridge"] diff --git a/5-Applications/webrtc-bridge/README.md b/5-Applications/webrtc-bridge/README.md new file mode 100644 index 00000000..469955c2 --- /dev/null +++ b/5-Applications/webrtc-bridge/README.md @@ -0,0 +1,136 @@ +# WebRTC Bridge + +Bypass Tailscale DERP relay latency for the Research Stack k3s cluster. + +## Problem + +Tailscale Funnel on node `361395-1` exposes Traefik at +`https://361395-1.tail4e7094.ts.net`, but Funnel traffic traverses DERP +relays, adding ~129 ms of latency per hop. For interactive workloads +(dashboards, shell, streaming) this is unacceptable. + +## Solution + +A lightweight WebRTC bridge that establishes a direct peer-to-peer data +channel between the browser client and the cluster node, bypassing DERP +entirely. After a one-time signaling exchange over WebSocket, all HTTP +traffic flows over the WebRTC data channel with sub-10 ms latency when +both peers have direct connectivity. + +## Architecture + +``` +┌──────────────┐ WebSocket (signaling) ┌───────────────────┐ +│ │ ◄──────────────────────── │ │ +│ Browser │ SDP offer/answer │ Signaling Server │ +│ Client │ ICE candidates │ (Go, port 8080) │ +│ │ │ on 361395-1 │ +│ │ WebRTC Data Channel │ │ +│ │ ◄════════════════════════► │ HTTP proxy │ +│ │ (peer-to-peer) │ → Traefik:30080 │ +└──────────────┘ └───────────────────┘ + │ │ + │ Direct P2P (UDP) │ + │ STUN for NAT traversal │ + │ No DERP relay! │ + ▼ ▼ + Client NAT Tailscale mesh + (STUN resolves) (direct route) +``` + +### Components + +| Component | Language | Port | Description | +|-----------|----------|------|-------------| +| `signaling-server.go` | Go | 8080 | WebSocket signaling + static file server | +| `client/index.html` | HTML/JS | — | Browser-side WebRTC client | +| `Containerfile` | — | — | Multi-stage Go build | +| `k8s/bridge.yaml` | YAML | — | k3s deployment manifests | + +### Signaling Flow + +1. Client opens WebSocket to `wss://361395-1.tail4e7094.ts.net/webrtc/ws` +2. Client creates `RTCPeerConnection` with public STUN servers +3. Client creates SDP offer, sends via WebSocket +4. Server receives offer, creates answer, sends back +5. ICE candidates exchanged over WebSocket +6. WebRTC data channel opens — signaling WebSocket may close +7. Client sends HTTP requests as JSON over the data channel +8. Server proxies requests to Traefik, returns responses + +### Data Channel Protocol + +Messages on the data channel are JSON-encoded HTTP request/response pairs: + +**Request (client → server):** +```json +{ + "id": "req-uuid", + "method": "GET", + "path": "/api/v1/status", + "headers": {"Accept": "application/json"}, + "body": "" +} +``` + +**Response (server → client):** +```json +{ + "id": "req-uuid", + "status": 200, + "headers": {"Content-Type": "application/json"}, + "body": "{\"ok\": true}" +} +``` + +### NAT Traversal + +- Uses public STUN servers (`stun:stun.l.google.com:19302`, + `stun:stun1.l.google.com:19302`) +- The server runs on `361395-1` with `hostNetwork: true`, so its + Tailscale IP `100.110.163.82` is directly reachable +- If STUN fails to resolve a direct path, the connection falls back to + Tailscale's own NAT traversal (which is better than DERP for most cases) +- A TURN server can be added later if needed + +## Deployment + +### Build the container image + +```bash +cd 5-Applications/webrtc-bridge +podman build -t localhost/webrtc-bridge:latest . +``` + +### Import into k3s + +```bash +podman save localhost/webrtc-bridge:latest | ssh 361395-1 k3s ctr images import - +``` + +### Apply manifests + +```bash +kubectl apply -f k8s/bridge.yaml +``` + +### Verify + +```bash +kubectl -n edge get pods -l app=webrtc-bridge +curl -s https://361395-1.tail4e7094.ts.net/webrtc/health +``` + +## Access + +Open `https://361395-1.tail4e7094.ts.net/webrtc/` in a browser. The client +will automatically connect via the signaling server, establish a WebRTC data +channel, and begin proxying HTTP requests. + +## Future Work + +- [ ] Add TURN server for symmetric NAT scenarios +- [ ] Multiplex multiple HTTP requests over a single data channel +- [ ] Add connection quality metrics (RTT, packet loss) +- [ ] Support WebSocket proxying through the data channel +- [ ] Add authentication (Tailscale identity headers) diff --git a/5-Applications/webrtc-bridge/client/index.html b/5-Applications/webrtc-bridge/client/index.html new file mode 100644 index 00000000..311dd7f9 --- /dev/null +++ b/5-Applications/webrtc-bridge/client/index.html @@ -0,0 +1,434 @@ + + + + + + WebRTC Bridge — Research Stack + + + +

⚡ WebRTC Bridge

+

Direct P2P to Research Stack — bypasses DERP relay

+ +
+
+
+
Disconnected
+
Not connected to signaling server
+
+
+ +
+
+
—
+
RTT (ms)
+
+
+
0
+
Requests
+
+
+
0
+
Bytes
+
+
+
—
+
ICE Type
+
+
+ +
+
+ + + +
+
+ +
+ + + + diff --git a/5-Applications/webrtc-bridge/go.sum b/5-Applications/webrtc-bridge/go.sum new file mode 100644 index 00000000..39256323 --- /dev/null +++ b/5-Applications/webrtc-bridge/go.sum @@ -0,0 +1,3 @@ +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/pion/webrtc/v4 v4.0.14 h1:nyds/sFRR+HvmWoBa6wrL46sSfpArE0qR883MBW96lg= +github.com/pion/webrtc/v4 v4.0.14/go.mod h1:R3+qTnQTS03UzwDarYecgioNf7DYgTsldxnCXB821Kk= diff --git a/5-Applications/webrtc-bridge/k8s/bridge.yaml b/5-Applications/webrtc-bridge/k8s/bridge.yaml new file mode 100644 index 00000000..77a75d2f --- /dev/null +++ b/5-Applications/webrtc-bridge/k8s/bridge.yaml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: edge +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: webrtc-bridge + namespace: edge + labels: + app: webrtc-bridge +spec: + replicas: 1 + selector: + matchLabels: + app: webrtc-bridge + template: + metadata: + labels: + app: webrtc-bridge + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/hostname: 361395-1 + containers: + - name: webrtc-bridge + image: localhost/webrtc-bridge:latest + imagePullPolicy: Never + ports: + - containerPort: 8080 + name: http + env: + - name: PORT + value: "8080" + - name: PROXY_TARGET + value: "http://100.102.173.61:30080" + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "250m" + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: webrtc-bridge + namespace: edge +spec: + selector: + app: webrtc-bridge + ports: + - port: 80 + targetPort: 8080 + name: http + type: ClusterIP +--- +# Traefik IngressRoute to expose under /webrtc path on the Funnel hostname. +# Traefik is already configured for 361395-1.tail4e7094.ts.net via Tailscale Funnel. +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: webrtc-bridge + namespace: edge +spec: + entryPoints: + - web + routes: + - match: Host(`361395-1.tail4e7094.ts.net`) && PathPrefix(`/webrtc`) + kind: Rule + services: + - name: webrtc-bridge + port: 80 + middlewares: + - name: webrtc-strip-prefix +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: webrtc-strip-prefix + namespace: edge +spec: + stripPrefix: + prefixes: + - /webrtc diff --git a/5-Applications/webrtc-bridge/signaling-server.go b/5-Applications/webrtc-bridge/signaling-server.go new file mode 100644 index 00000000..a585c924 --- /dev/null +++ b/5-Applications/webrtc-bridge/signaling-server.go @@ -0,0 +1,287 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "sync" + "time" + + "github.com/gorilla/websocket" + "github.com/pion/webrtc/v4" +) + +// Signaling message types exchanged over WebSocket. +type SignalMessage struct { + Type string `json:"type"` // "offer", "answer", "candidate" + SDP *webrtc.SessionDescription `json:"sdp,omitempty"` + Candidate *webrtc.ICECandidateInit `json:"candidate,omitempty"` +} + +// HTTP request forwarded over the WebRTC data channel. +type HTTPRequest struct { + ID string `json:"id"` + Method string `json:"method"` + Path string `json:"path"` + Headers map[string]string `json:"headers"` + Body string `json:"body"` +} + +// HTTP response returned over the WebRTC data channel. +type HTTPResponse struct { + ID string `json:"id"` + Status int `json:"status"` + Headers map[string]string `json:"headers"` + Body string `json:"body"` +} + +var ( + upgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, + } + // Target for proxied HTTP requests — Traefik ingress. + proxyTarget = envOr("PROXY_TARGET", "http://100.102.173.61:30080") +) + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func main() { + port := envOr("PORT", "8080") + + http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"status":"ok","service":"webrtc-bridge"}`)) + }) + + http.HandleFunc("/ws", handleWebSocket) + + // Serve the static client files. + http.Handle("/", http.FileServer(http.Dir("/app/client"))) + + log.Printf("WebRTC bridge signaling server starting on :%s (proxy target: %s)", port, proxyTarget) + if err := http.ListenAndServe(":"+port, nil); err != nil { + log.Fatalf("server failed: %v", err) + } +} + +func handleWebSocket(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Printf("WebSocket upgrade failed: %v", err) + return + } + defer conn.Close() + + log.Printf("Client connected: %s", r.RemoteAddr) + + // Create a new RTCPeerConnection for this client. + config := webrtc.Configuration{ + ICEServers: []webrtc.ICEServer{ + {URLs: []string{"stun:stun.l.google.com:19302"}}, + {URLs: []string{"stun:stun1.l.google.com:19302"}}, + {URLs: []string{"stun:stun2.l.google.com:19302"}}, + }, + } + + peerConnection, err := webrtc.NewPeerConnection(config) + if err != nil { + log.Printf("Failed to create PeerConnection: %v", err) + return + } + defer peerConnection.Close() + + var dcOnce sync.Once + // Handle incoming data channels from the client. + peerConnection.OnDataChannel(func(dc *webrtc.DataChannel) { + dcOnce.Do(func() { + setupDataChannel(dc) + }) + }) + + // Send ICE candidates to the client. + peerConnection.OnICECandidate(func(c *webrtc.ICECandidate) { + if c == nil { + return + } + candidate := c.ToJSON() + msg := SignalMessage{ + Type: "candidate", + Candidate: &candidate, + } + conn.WriteJSON(msg) + }) + + // Log connection state changes. + peerConnection.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { + log.Printf("PeerConnection state: %s", state.String()) + }) + + // Read signaling messages from the WebSocket. + for { + _, raw, err := conn.ReadMessage() + if err != nil { + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + log.Printf("Client disconnected: %s", r.RemoteAddr) + } else { + log.Printf("WebSocket read error: %v", err) + } + return + } + + var msg SignalMessage + if err := json.Unmarshal(raw, &msg); err != nil { + log.Printf("Invalid signaling message: %v", err) + continue + } + + switch msg.Type { + case "offer": + if msg.SDP == nil { + log.Printf("Offer missing SDP") + continue + } + if err := peerConnection.SetRemoteDescription(*msg.SDP); err != nil { + log.Printf("SetRemoteDescription failed: %v", err) + continue + } + + answer, err := peerConnection.CreateAnswer(nil) + if err != nil { + log.Printf("CreateAnswer failed: %v", err) + continue + } + if err := peerConnection.SetLocalDescription(answer); err != nil { + log.Printf("SetLocalDescription failed: %v", err) + continue + } + + conn.WriteJSON(SignalMessage{ + Type: "answer", + SDP: &answer, + }) + log.Printf("Sent SDP answer to %s", r.RemoteAddr) + + case "candidate": + if msg.Candidate == nil { + continue + } + if err := peerConnection.AddICECandidate(*msg.Candidate); err != nil { + log.Printf("AddICECandidate failed: %v", err) + } + } + } +} + +func setupDataChannel(dc *webrtc.DataChannel) { + dc.OnOpen(func() { + log.Printf("Data channel opened: %s (id=%d)", dc.Label(), *dc.ID()) + }) + + dc.OnClose(func() { + log.Printf("Data channel closed: %s", dc.Label()) + }) + + dc.OnMessage(func(msg webrtc.DataChannelMessage) { + var req HTTPRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + log.Printf("Invalid data channel message: %v", err) + return + } + + go func() { + resp := proxyRequest(&req) + data, _ := json.Marshal(resp) + dc.SendText(string(data)) + }() + }) +} + +func proxyRequest(req *HTTPRequest) *HTTPResponse { + start := time.Now() + + path := req.Path + if path == "" { + path = "/" + } + url := proxyTarget + path + + var body io.Reader + if req.Body != "" { + body = &readCloser{[]byte(req.Body)} + } + + httpReq, err := http.NewRequest(req.Method, url, body) + if err != nil { + return &HTTPResponse{ + ID: req.ID, + Status: 502, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: fmt.Sprintf(`{"error":"bad request: %s"}`, err.Error()), + } + } + + for k, v := range req.Headers { + httpReq.Header.Set(k, v) + } + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(httpReq) + if err != nil { + return &HTTPResponse{ + ID: req.ID, + Status: 502, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: fmt.Sprintf(`{"error":"proxy error: %s"}`, err.Error()), + } + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return &HTTPResponse{ + ID: req.ID, + Status: 502, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: fmt.Sprintf(`{"error":"read body: %s"}`, err.Error()), + } + } + + headers := make(map[string]string) + for k := range resp.Header { + headers[k] = resp.Header.Get(k) + } + + elapsed := time.Since(start) + log.Printf("Proxy %s %s → %d (%s)", req.Method, req.Path, resp.Status, elapsed) + + return &HTTPResponse{ + ID: req.ID, + Status: resp.StatusCode, + Headers: headers, + Body: string(respBody), + } +} + +// readCloser wraps a byte slice as an io.ReadCloser. +type readCloser struct { + data []byte +} + +func (r *readCloser) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, io.EOF + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil +} + +func (r *readCloser) Close() error { return nil }