diff --git a/.gitignore b/.gitignore index ae79292..cdd3c69 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,10 @@ .idea/ .vscode/ +# Python (examples/ Python brains are not in CI; ignore bytecode) +__pycache__/ +*.pyc + # Local worktrees (per using-git-worktrees skill; isolation, not content) .worktrees/ worktrees/ diff --git a/LEARNING.md b/LEARNING.md index 3295157..b680e43 100644 --- a/LEARNING.md +++ b/LEARNING.md @@ -79,6 +79,33 @@ can read in `cargo doc --open` plus the source file itself. — `TapHandle(())` compiles `Option` to a single `bool`; no runtime cost for the type-system marker. +### Slice 3 — OpenAI Realtime brain + +- **`async-trait` patterns / async fns in trait objects** → + `crates/rutster/src/tool_registry.rs` — the `Tool` trait's `async fn call` + and how `async-trait` lowers it to `Pin>` (the same + boxing the slice-1 `pipe` widening used, now applied to trait methods). +- **OpenAI Realtime adapter + event translation (pure-function layer)** → + `crates/rutster-brain-realtime/src/translator.rs` — the §4.2 event mapping + table as pure `fn`s: tap `audio_in` ⇄ OpenAI `input_audio_buffer.append`, + `response.audio.delta` ⇄ `audio_out`, `function_call_arguments.done` ⇄ + `function_call`. Testable without a network (the pump is in a separate file). +- **Tap protocol additive extension + forward-compat via `#[serde(other)]`** → + `crates/rutster-tap/src/protocol.rs` — how slice-3 adds `speech_started`, + `speech_stopped`, `function_call`, `function_call_output`, `tools.update` + without breaking slice-2 brains: unknown variants deserialize to a catch-all + arm instead of erroring (forward-compat by construction). +- **Side-channel mpsc for FOB-boundary dispatch** → + `crates/rutster/src/session_map.rs` — how `drive_all_sessions` drains a + function-call mpsc side channel into the tap WS writer without blocking the + 20 ms media loop. The pattern: hot loop polls `try_recv`; cold path spawns. +- **HTTP request builder for WS subprotocol handshake** → + `crates/rutster-brain-realtime/src/openai_client.rs` — why we hand-roll the + WS upgrade `Request` instead of letting `connect_async` build it: to set + `Authorization` + `OpenAI-Beta` headers on the WS handshake (tungstenite + doesn't expose subprotocol/auth headers via the simple `connect_async(url)` + entry point — see `openai_headers` / `openai_realtime_url`). + ## How to read 1. `cargo doc --open` — every module has a `//!` doc comment; the doc diff --git a/README.md b/README.md index e009220..82bb084 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,38 @@ cargo run Open → click "Start call" → grant mic → hear yourself echo. Full walkthrough + troubleshooting: **[`docs/QUICKSTART.md`](docs/QUICKSTART.md)**. +### Slice 3 dev loop — OpenAI Realtime brain + +The dev loop *without* real OpenAI credentials (no API key required): + +``` +cargo run -p rutster-brain-realtime --features=mock # brain on :8082 +cargo run # core on :8080 +``` + +Open → click "Start call" → speak → hear the +mock-brain reply within ~250 ms (the mock echoes audio back, no real OpenAI +RTT; this exercises the full brain→core audio round-trip + the new +function_call dispatch path). + +With real OpenAI Realtime: + +``` +export OPENAI_API_KEY=sk-... # or OPENAI_API_KEY_FILE=/var/secrets/openai +cargo run -p rutster-brain-realtime +cargo run +``` + +Speak → end-to-end speech-to-speech with OpenAI Realtime within ~700 ms +(slice-1's 200 ms + tap round-trip + OpenAI latency + 100 ms playout buffer). + +For the foreign-language brain demo (Python, not in CI): + +``` +pip install -r examples/openai_realtime_brain/requirements.txt +OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py +``` + > **Status:** Slice 1 (WebRTC media loopback) is the active build target. The workspace is > landing task-by-task on the `slice-1-webrtc-loopback` branch. Design: > [`docs/superpowers/specs/2026-06-28-slice-1-webrtc-loopback-design.md`](docs/superpowers/specs/2026-06-28-slice-1-webrtc-loopback-design.md). diff --git a/examples/openai_realtime_brain/README.md b/examples/openai_realtime_brain/README.md new file mode 100644 index 0000000..e11bba5 --- /dev/null +++ b/examples/openai_realtime_brain/README.md @@ -0,0 +1,52 @@ +# OpenAI Realtime reference brain — Python (slice-3 spec §7.5) + +A Python implementation of the slice-3 OpenAI Realtime brain — the canonical +foreign-language brain demo, hand-rolled from the documented tap protocol +([`docs/superpowers/specs/2026-06-30-slice-3-realtime-brain-design.md`](../../docs/superpowers/specs/2026-06-30-slice-3-realtime-brain-design.md)). + +## Why + +Proves the slice-3 tap protocol extension is language-agnostic. A Python +script speaking JSON-via-WSS matches the OpenAI-Realtime-related portions of +the spec without depending on Rust code paths. Same rationale as slice-2's +Python echo brain — the project's `examples/` dir is the home for canonical +foreign-language brain demos. + +The structure mirrors the Rust `rutster-brain-realtime` crate: a WS server +(the tap side the core dials into) bridged bidirectionally to a WS client +(the OpenAI Realtime side). The event mapping is the spec §4.2 table verbatim. + +## Run + +```bash +pip install -r requirements.txt +OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py +``` + +The Python brain binds the tap WS server on `RUTSTER_BRAIN_BIND` (default +`127.0.0.1:8082`). The rutster binary, started normally (`cargo run`), dials +out to `RUTSTER_TAP_URL` (default `ws://127.0.0.1:8082/realtime` — set +`RUTSTER_TAP_URL=ws://127.0.0.1:8082` to match the Python brain's bind). + +Other env vars: + +| Var | Default | Purpose | +|---|---|---| +| `OPENAI_API_KEY` | (required) | OpenAI auth | +| `RUTSTER_BRAIN_BIND` | `127.0.0.1:8082` | host:port for the tap WS server | +| `OPENAI_REALTIME_MODEL` | `gpt-4o-realtime` | OpenAI model query-param | +| `OPENAI_REALTIME_VOICE` | `alloy` | Voice passed in `session.update` | + +## Dependencies + +Only `websockets>=12.0` — both the tap-side server and the OpenAI-side client +speak plain WSS, so the `openai` Python SDK is not needed (the spec §4.2 +mapping is explicit enough to hand-roll against the wire). This is a +deliberate simplification over the slice-3 plan skeleton, which imported the +SDK but never used it. + +## Not in CI + +Per AGENTS.md's "no Python in the dev loop" rule. The slice-3 integration +test uses `rutster-brain-realtime --features=mock` — the in-process Rust +mock — not this Python file. diff --git a/examples/openai_realtime_brain/openai_realtime_brain.py b/examples/openai_realtime_brain/openai_realtime_brain.py new file mode 100644 index 0000000..cf7be5d --- /dev/null +++ b/examples/openai_realtime_brain/openai_realtime_brain.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""OpenAI Realtime reference brain -- Python implementation (slice-3 spec §7.5). + +Mirrors rutster-brain-realtime's WS-server + OpenAI-WS-client glue in +Python, proving the slice-3 tap protocol is language-agnostic. The mapping +is the §4.2 event table verbatim. + +Run: + pip install -r requirements.txt + OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py + +Not in CI (violates the zero-non-Rust-dev-deps dev loop per AGENTS.md). +""" + +import asyncio +import json +import os +import sys + +import websockets + +PROTOCOL_VERSION = 1 +SAMPLES = 480 + +# Configuration via env (matching the Rust binary's naming convention so the +# docs read the same on both sides). +RUTSTER_TAP_BIND = os.environ.get("RUTSTER_BRAIN_BIND", "127.0.0.1:8082") +OPENAI_MODEL = os.environ.get("OPENAI_REALTIME_MODEL", "gpt-4o-realtime") +OPENAI_VOICE = os.environ.get("OPENAI_REALTIME_VOICE", "alloy") + + +def openai_realtime_url(model: str) -> str: + """wss://api.openai.com/v1/realtime?model= (spec §4.2 + §5.3).""" + return f"wss://api.openai.com/v1/realtime?model={model}" + + +def openai_headers(api_key: str): + """The two headers the OpenAI WS handshake requires (spec §5.3).""" + return [ + ("Authorization", f"Bearer {api_key}"), + ("OpenAI-Beta", "realtime=v1"), + ] + + +async def handle_tap_connection(tap_ws): + """Bridge one rutster tap WS connection to one OpenAI Realtime session. + + Spec §4.2 mapping table, in both directions. One tap connection == one + OpenAI Realtime session; stateless across reconnects (spec §5.3), exactly + like the Rust brain and the slice-2 echo brain pattern. + """ + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + await tap_ws.close(code=1011, reason="OPENAI_API_KEY required") + return + + # Open the OpenAI WS with the two required headers. additional_headers is + # the websockets>=12 spelling (was extra_headers in v11). + openai_ws = await websockets.connect( + openai_realtime_url(OPENAI_MODEL), + additional_headers=openai_headers(api_key), + ) + + # S4: send session.update with turn_detection: null. The FOB (the core) + # owns turn-taking; OpenAI's server-side VAD is explicitly disabled so it + # does not contradict core-authoritative playout decisions. See + # openai_client.rs for the Rust-side commentary on why this matters. + await openai_ws.send(json.dumps({ + "type": "session.update", + "session": { + "modalities": ["text", "audio"], + "voice": OPENAI_VOICE, + "input_audio_format": "pcm16", + "output_audio_format": "pcm16", + "sample_rate": 24000, + "turn_detection": None, + }, + })) + print(f"[openai_brain] session.update sent (turn_detection=null); " + f"model={OPENAI_MODEL} voice={OPENAI_VOICE}") + + seq_egress = 0 + + def next_seq(): + nonlocal seq_egress + s = seq_egress + seq_egress += 1 + return s + + async def tap_to_openai(): + """rutster tap events -> OpenAI Realtime events (spec §4.2 left column).""" + async for raw in tap_ws: + try: + env = json.loads(raw) + except json.JSONDecodeError as e: + await tap_ws.send(json.dumps({ + "v": PROTOCOL_VERSION, "type": "error", "seq": next_seq(), + "ts": 0, "code": "decode_failed", "message": str(e), + })) + continue + + t = env.get("type") + if t == "hello": + # Stateless across reconnects: ack hello, do NOT re-send + # session.update (already sent on this OpenAI WS session). + await tap_ws.send(json.dumps({ + "v": PROTOCOL_VERSION, "type": "hello", "seq": next_seq(), + "ts": 0, "session_id": env.get("session_id", ""), + })) + elif t == "audio_in": + # Pass-through: same base64 PCM wire shape. No transcoding. + await openai_ws.send(json.dumps({ + "type": "input_audio_buffer.append", + "audio": env["pcm"], + })) + elif t == "function_call_output": + # Core's tool-registry reply -> OpenAI conversation.item.create + # so the model can continue. Spec §4.2 row 5. + await openai_ws.send(json.dumps({ + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": env["id"], + "output": json.dumps(env.get("result", {})), + }, + })) + elif t in ("bye", "session_end"): + await openai_ws.close() + return + # Unknown frame types: ignore (forward-compat via additive protocol). + + async def openai_to_tap(): + """OpenAI Realtime events -> rutster tap events (spec §4.2 right column).""" + async for raw in openai_ws: + try: + event = json.loads(raw) + except json.JSONDecodeError: + continue + t = event.get("type") + if t == "response.audio.delta": + # Forward as tap audio_out. The delta is base64 PCM16; pass + # through. (Batching into 20 ms frames is a future + # optimization, spec §1.2; v1 passes each delta through.) + await tap_ws.send(json.dumps({ + "v": PROTOCOL_VERSION, "type": "audio_out", + "seq": next_seq(), "ts": 0, + "pcm": event["delta"], "samples": SAMPLES, + })) + elif t == "input_audio_buffer.speech_started": + await tap_ws.send(json.dumps({ + "v": PROTOCOL_VERSION, "type": "speech_started", + "seq": next_seq(), "ts": 0, + })) + elif t == "input_audio_buffer.speech_stopped": + await tap_ws.send(json.dumps({ + "v": PROTOCOL_VERSION, "type": "speech_stopped", + "seq": next_seq(), "ts": 0, + })) + elif t == "response.function_call_arguments.done": + # Tool invocation from the model -> core's tool registry. + await tap_ws.send(json.dumps({ + "v": PROTOCOL_VERSION, "type": "function_call", + "id": event["call_id"], "name": event["name"], + "args": json.loads(event.get("arguments", "{}")), + "seq": next_seq(), "ts": 0, + })) + elif t == "error": + # Surface OpenAI-side errors to the core for observation. + await tap_ws.send(json.dumps({ + "v": PROTOCOL_VERSION, "type": "error", + "seq": next_seq(), "ts": 0, + "code": "openai_error", + "message": event.get("error", {}).get("message", "unknown"), + })) + + try: + await asyncio.gather(tap_to_openai(), openai_to_tap()) + finally: + await openai_ws.close() + + +async def main(): + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + sys.exit("OPENAI_API_KEY required (see README.md)") + + host, port = RUTSTER_TAP_BIND.split(":") + print(f"[openai_brain] listening on ws://{RUTSTER_TAP_BIND} " + f"(OpenAI model={OPENAI_MODEL} voice={OPENAI_VOICE})") + async with websockets.serve(handle_tap_connection, host, int(port)): + await asyncio.Future() # run forever + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/openai_realtime_brain/requirements.txt b/examples/openai_realtime_brain/requirements.txt new file mode 100644 index 0000000..31b5e2f --- /dev/null +++ b/examples/openai_realtime_brain/requirements.txt @@ -0,0 +1 @@ +websockets>=12.0