Slice 3 — OpenAI Realtime brain: swap echo for the brain (#4)
All checks were successful
CI / fmt (push) Successful in 1m40s
CI / clippy (push) Successful in 2m24s
CI / test (1.85) (push) Successful in 5m8s
CI / test (stable) (push) Successful in 5m20s
CI / deny (push) Successful in 1m34s

This commit was merged in pull request #4.
This commit is contained in:
2026-07-01 22:25:09 +00:00
parent 2f3f92ec6b
commit c30a45232d
36 changed files with 7227 additions and 20 deletions

4
.gitignore vendored
View File

@@ -19,6 +19,10 @@
.idea/
.vscode/
# Python (examples/ Python brains are not in CI; ignore bytecode)
__pycache__/
*.pyc
# Local worktrees (per using-git-worktrees skill; isolation, not content)
.worktrees/
worktrees/

55
Cargo.lock generated
View File

@@ -781,6 +781,12 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -1144,6 +1150,19 @@ dependencies = [
"nom 7.1.3",
]
[[package]]
name = "rustix"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "rustls-pki-types"
version = "1.14.1"
@@ -1163,9 +1182,11 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
name = "rutster"
version = "0.0.0"
dependencies = [
"async-trait",
"axum",
"dashmap",
"futures-util",
"rutster-brain-realtime",
"rutster-call-model",
"rutster-media",
"rutster-tap",
@@ -1182,6 +1203,27 @@ dependencies = [
"uuid",
]
[[package]]
name = "rutster-brain-realtime"
version = "0.1.0"
dependencies = [
"async-trait",
"base64",
"futures-util",
"http",
"rutster-media",
"rutster-tap",
"serde",
"serde_json",
"tempfile",
"thiserror 1.0.69",
"tokio",
"tokio-tungstenite",
"tracing",
"tracing-subscriber",
"url",
]
[[package]]
name = "rutster-call-model"
version = "0.0.0"
@@ -1498,6 +1540,19 @@ dependencies = [
"syn",
]
[[package]]
name = "tempfile"
version = "3.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys",
]
[[package]]
name = "thiserror"
version = "1.0.69"

View File

@@ -11,6 +11,7 @@ members = [
"crates/rutster-trunk",
"crates/rutster-tap",
"crates/rutster-tap-echo",
"crates/rutster-brain-realtime",
"crates/rutster-spend",
]
@@ -51,3 +52,10 @@ futures-util = "0.3"
url = "2"
# base64 0.22: PCM <-> base64 codec for the v1 wire format (spec §3).
base64 = "0.22"
# async-trait 0.1: async fns in trait objects (Tool trait, slice-3 spec §6.1).
async-trait = "0.1"
# http 1: Request/Response builders for the OpenAI WS handshake (slice-3
# spec §5.3). Task 9's main.rs builds the http::Request<...> the tungstenite
# client sends before WS upgrade; `openai_client::openai_headers` produces
# the header pairs the caller stuffs into it.
http = "1"

View File

@@ -79,6 +79,33 @@ can read in `cargo doc --open` plus the source file itself.
`TapHandle(())` compiles `Option<TapHandle>` to a single `bool`; no
runtime cost for the type-system marker.
### Slice 3 — OpenAI Realtime brain
- **`async-trait` patterns / async fns in trait objects** →
`crates/rutster/src/tool_registry.rs` — the `Tool` trait's `async fn call`
and how `async-trait` lowers it to `Pin<Box<dyn Future + Send>>` (the same
boxing the slice-1 `pipe` widening used, now applied to trait methods).
- **OpenAI Realtime adapter + event translation (pure-function layer)** →
`crates/rutster-brain-realtime/src/translator.rs` — the §4.2 event mapping
table as pure `fn`s: tap `audio_in` ⇄ OpenAI `input_audio_buffer.append`,
`response.audio.delta``audio_out`, `function_call_arguments.done`
`function_call`. Testable without a network (the pump is in a separate file).
- **Tap protocol additive extension + forward-compat via `#[serde(other)]`** →
`crates/rutster-tap/src/protocol.rs` — how slice-3 adds `speech_started`,
`speech_stopped`, `function_call`, `function_call_output`, `tools.update`
without breaking slice-2 brains: unknown variants deserialize to a catch-all
arm instead of erroring (forward-compat by construction).
- **Side-channel mpsc for FOB-boundary dispatch** →
`crates/rutster/src/session_map.rs` — how `drive_all_sessions` drains a
function-call mpsc side channel into the tap WS writer without blocking the
20 ms media loop. The pattern: hot loop polls `try_recv`; cold path spawns.
- **HTTP request builder for WS subprotocol handshake** →
`crates/rutster-brain-realtime/src/openai_client.rs` — why we hand-roll the
WS upgrade `Request` instead of letting `connect_async` build it: to set
`Authorization` + `OpenAI-Beta` headers on the WS handshake (tungstenite
doesn't expose subprotocol/auth headers via the simple `connect_async(url)`
entry point — see `openai_headers` / `openai_realtime_url`).
## How to read
1. `cargo doc --open` — every module has a `//!` doc comment; the doc

View File

@@ -19,6 +19,38 @@ cargo run
Open <http://localhost:8080/> → click "Start call" → grant mic → hear yourself echo.
Full walkthrough + troubleshooting: **[`docs/QUICKSTART.md`](docs/QUICKSTART.md)**.
### Slice 3 dev loop — OpenAI Realtime brain
The dev loop *without* real OpenAI credentials (no API key required):
```
cargo run -p rutster-brain-realtime --features=mock # brain on :8082
cargo run # core on :8080
```
Open <http://localhost:8080/> → click "Start call" → speak → hear the
mock-brain reply within ~250 ms (the mock echoes audio back, no real OpenAI
RTT; this exercises the full brain→core audio round-trip + the new
function_call dispatch path).
With real OpenAI Realtime:
```
export OPENAI_API_KEY=sk-... # or OPENAI_API_KEY_FILE=/var/secrets/openai
cargo run -p rutster-brain-realtime
cargo run
```
Speak → end-to-end speech-to-speech with OpenAI Realtime within ~700 ms
(slice-1's 200 ms + tap round-trip + OpenAI latency + 100 ms playout buffer).
For the foreign-language brain demo (Python, not in CI):
```
pip install -r examples/openai_realtime_brain/requirements.txt
OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py
```
> **Status:** Slice 1 (WebRTC media loopback) is the active build target. The workspace is
> landing task-by-task on the `slice-1-webrtc-loopback` branch. Design:
> [`docs/superpowers/specs/2026-06-28-slice-1-webrtc-loopback-design.md`](docs/superpowers/specs/2026-06-28-slice-1-webrtc-loopback-design.md).

View File

@@ -0,0 +1,35 @@
# crates/rutster-brain-realtime/Cargo.toml
[package]
name = "rutster-brain-realtime"
version = "0.1.0"
license.workspace = true
edition.workspace = true
repository.workspace = true
description = "OpenAI Realtime speech-to-speech brain — translates slice-2's tap protocol to OpenAI Realtime's event schema (green-zone per ADR-0008; slice-3 spec §1.1, §4)."
[dependencies]
rutster-tap = { path = "../rutster-tap" }
rutster-media = { path = "../rutster-media" } # for PcmFrame in tests (spec §4.2 audio round-trip)
tokio = { workspace = true, features = ["full"] }
tokio-tungstenite = { workspace = true, features = ["connect"] }
futures-util = { workspace = true }
serde_json = { workspace = true }
serde = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
url = { workspace = true }
async-trait = { workspace = true }
base64 = { workspace = true }
thiserror = { workspace = true }
# http: Request/Response builder for the WS handshake (Task 9's main.rs).
# This task's unit tests assert URL + headers shape only.
http = { workspace = true }
[dev-dependencies]
tempfile = "3"
[features]
default = []
# Mock mode: in-process fake OpenAI Realtime WS server (no real API calls).
# Used by the integration test + the offline dev loop (spec §7.3).
mock = []

View File

@@ -0,0 +1,199 @@
//! # API-key loader (spec §5.3)
//!
//! Two-source config: `OPENAI_API_KEY` env var default + optional
//! `OPENAI_API_KEY_FILE` path override. KMS/Vault integration is deferred
//! to step 6 (spec §1.2); the file-path override makes secret-manager
//! injection (k8s secrets, Vault agent) trivial when that layer exists.
//!
//! # Why file-path override (not env-only)
//!
//! A k8s pod mounts secrets at file paths (e.g. `/var/secrets/openai_key`).
//! Env-var secrets are also fine, but file-path is the standard pattern for
//! k8s + Vault agent + sidecar secret rotators. The dev loop uses the env
//! var path (single-source, simplest).
use std::env;
use std::fs;
use std::path::PathBuf;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ApiKeyError {
#[error("no API key: set OPENAI_API_KEY or OPENAI_API_KEY_FILE")]
NotFound,
#[error("OPENAI_API_KEY_FILE path is not valid UTF-8: {0:?}")]
PathNotUtf8(PathBuf),
#[error("failed to read OPENAI_API_KEY_FILE at {path}: {source}")]
FileRead {
path: PathBuf,
#[source]
source: std::io::Error,
},
}
/// Load the OpenAI API key per the env-var + file-path posture (spec §5.3).
///
/// Precedence: `OPENAI_API_KEY_FILE` wins over `OPENAI_API_KEY` (a file
/// path is a more specific override; this matches k8s-secret patterns where
/// an operator mounts a file to override the default env var).
///
/// Trims trailing whitespace from either source (some k8s secret mounts
/// add trailing newlines).
pub fn load_api_key() -> Result<String, ApiKeyError> {
if let Ok(path_str) = env::var("OPENAI_API_KEY_FILE") {
let path = PathBuf::from(&path_str);
let raw = fs::read_to_string(&path).map_err(|source| ApiKeyError::FileRead {
path: path.clone(),
source,
})?;
return Ok(raw.trim().to_string());
}
if let Ok(raw) = env::var("OPENAI_API_KEY") {
return Ok(raw.trim().to_string());
}
Err(ApiKeyError::NotFound)
}
#[cfg(test)]
mod tests {
use super::*;
// NOTE: edition 2024 makes env::set_var / env::remove_var `unsafe` (they
// can race with concurrent reads of the process environ). The brief was
// written against edition 2021 syntax; we wrap each mutation in
// `unsafe { ... }` rather than reverting to a pre-2024 edition. The
// unsafety is local to the test process and accepted by the stdlib's
// own test helpers; no threads here read these env vars concurrently.
// The process environ is process-global state. Even with the save/restore
// pattern below, running these tests in parallel would let one test's
// `set_var` race against another's `remove_var` for the same variable,
// producing flaky failures. This mutex serializes every test in this
// module so the env-var mutations are atomic across tests.
static ENV_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(());
/// Acquire the env mutex for the test body. Panics if the lock is held
/// (only happens if a test panics while held — fine, fail fast).
fn lock_env() -> std::sync::MutexGuard<'static, ()> {
ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner())
}
#[test]
fn returns_error_when_neither_env_nor_file_set() {
let _guard = lock_env();
// Save then clear both env vars to make the test order-independent.
let saved_key = env::var_os("OPENAI_API_KEY").map(|v| ("OPENAI_API_KEY".to_string(), v));
let saved_file =
env::var_os("OPENAI_API_KEY_FILE").map(|v| ("OPENAI_API_KEY_FILE".to_string(), v));
unsafe {
env::remove_var("OPENAI_API_KEY");
env::remove_var("OPENAI_API_KEY_FILE");
}
let r = load_api_key();
assert!(matches!(r, Err(ApiKeyError::NotFound)));
// Restore for any other test in the same process.
if let Some((k, v)) = saved_key {
unsafe {
env::set_var(k, v);
}
}
if let Some((k, v)) = saved_file {
unsafe {
env::set_var(k, v);
}
}
}
#[test]
fn reads_from_env_var() {
let _guard = lock_env();
let saved = env::var_os("OPENAI_API_KEY").map(|v| ("OPENAI_API_KEY".to_string(), v));
let saved_file =
env::var_os("OPENAI_API_KEY_FILE").map(|v| ("OPENAI_API_KEY_FILE".to_string(), v));
unsafe {
env::set_var("OPENAI_API_KEY", "sk-test-12345");
env::remove_var("OPENAI_API_KEY_FILE");
}
let r = load_api_key().unwrap();
assert_eq!(r, "sk-test-12345");
if let Some((k, v)) = saved {
unsafe {
env::set_var(k, v);
}
} else {
unsafe {
env::remove_var("OPENAI_API_KEY");
}
}
if let Some((k, v)) = saved_file {
unsafe {
env::set_var(k, v);
}
}
}
#[test]
fn trim_trailing_newline_from_env() {
let _guard = lock_env();
let saved = env::var_os("OPENAI_API_KEY").map(|v| ("OPENAI_API_KEY".to_string(), v));
unsafe {
env::set_var("OPENAI_API_KEY", "sk-test-with-newline\n");
env::remove_var("OPENAI_API_KEY_FILE");
}
let r = load_api_key().unwrap();
assert_eq!(r, "sk-test-with-newline");
if let Some((k, v)) = saved {
unsafe {
env::set_var(k, v);
}
} else {
unsafe {
env::remove_var("OPENAI_API_KEY");
}
}
}
#[test]
fn file_path_overrides_env() {
let _guard = lock_env();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("key.txt");
std::fs::write(&path, "sk-from-file-98765\n").unwrap();
unsafe {
env::set_var("OPENAI_API_KEY", "sk-from-env-should-be-overridden");
env::set_var("OPENAI_API_KEY_FILE", path.to_str().unwrap());
}
let r = load_api_key().unwrap();
assert_eq!(r, "sk-from-file-98765");
unsafe {
env::remove_var("OPENAI_API_KEY");
env::remove_var("OPENAI_API_KEY_FILE");
}
}
#[test]
fn file_path_missing_returns_file_read_error() {
let _guard = lock_env();
unsafe {
env::set_var("OPENAI_API_KEY_FILE", "/nonexistent/path/to/key.txt");
env::remove_var("OPENAI_API_KEY");
}
let r = load_api_key();
assert!(matches!(r, Err(ApiKeyError::FileRead { .. })));
unsafe {
env::remove_var("OPENAI_API_KEY_FILE");
}
}
}

View File

@@ -0,0 +1,37 @@
//! # rutster-brain-realtime
//!
//! **Slice-3 brain:** translates slice-2's tap protocol (the green-zone side of
//! the seam — core-as-client dials this brain's WS server; ADR-0008 classifies
//! the brain as green-zone) to OpenAI Realtime's event schema. The brain
//! process is a WS *server* (core-as-client dials it, unchanged from slice-2)
//! AND a WS *client* to `wss://api.openai.com/v1/realtime` (OpenAI is server
//! on *its* leg).
//!
//! See `docs/superpowers/specs/2026-06-30-slice-3-realtime-brain-design.md` for
//! the full design.
//!
//! ## Modules
//!
//! - [`api_key`] — load API key from env var or file (spec §5.3).
//! - [`translator`] — pure-function event translation (tap ⇄ OpenAI).
//! - [`openai_client`] — wss:// client to api.openai.com/v1/realtime.
//!
//! ## Dev mode (`--features=mock`)
//!
//! When built with `mock`, the binary uses in-process `MockRealtimeBrain`
//! (defined in `lib.rs`'s test-support module) instead of dialing OpenAI.
//! No API key required, no real OpenAI calls. Used by the integration test +
//! the offline dev loop (spec §7.3).
pub mod api_key;
pub mod openai_client;
pub mod translator;
/// slice-3 §5.3 dev mode: in-process fake OpenAI Realtime WS server. Only
/// available with `--features=mock`. The binary uses it to run the offline
/// dev loop (no API key, no network calls to OpenAI); the integration test
/// uses it for the same reason.
#[cfg(feature = "mock")]
pub mod mock;
#[cfg(feature = "mock")]
pub use mock::MockRealtimeBrain;

View File

@@ -0,0 +1,212 @@
//! # rutster-brain-realtime binary (slice-3 spec §4.2 + §5.3)
//!
//! Standalone brain process the core dials out to via `RUTSTER_TAP_URL` (spec
//! §5.1). One process bridges many tap WS connections (one per call) to many
//! OpenAI Realtime sessions.
//!
//! # Modes
//!
//! - **`--features=mock`** (offline dev loop, slice-3 spec §7.3): starts an
//! in-process `MockRealtimeBrain` WS server, dials it as the "OpenAI side."
//! No API key, no network calls to OpenAI. Used by the integration test +
//! the offline dev loop (`cargo run -p rutster-brain-realtime --features=mock`).
//! - **Default** (real OpenAI): loads `OPENAI_API_KEY` per
//! [`api_key::load_api_key`], dials `wss://api.openai.com/v1/realtime?model=...`.
//!
//! Both modes share the same tap-side WS server + the same `run_openai_pump`
//! bridging logic — only the OpenAI-side dialer differs.
use std::net::SocketAddr;
use futures_util::{SinkExt, StreamExt};
#[cfg(feature = "mock")]
use rutster_brain_realtime::MockRealtimeBrain;
#[cfg(not(feature = "mock"))]
use rutster_brain_realtime::api_key;
use rutster_brain_realtime::openai_client;
use rutster_brain_realtime::translator::build_openai_session_update;
use tokio::sync::mpsc;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tracing::{info, warn};
/// Default bind addr for the brain process's tap-side WS server (spec §5.3).
/// Distinct from slice-2's echo brain (`:8081/echo`); the two coexist.
const DEFAULT_TAP_BIND: &str = "127.0.0.1:8082";
const DEFAULT_VOICE: &str = "alloy";
const DEFAULT_MODEL: &str = "gpt-4o-realtime";
#[tokio::main]
async fn main() {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "rutster_brain_realtime=info".into()),
)
.init();
let tap_bind: SocketAddr = std::env::var("RUTSTER_TAP_BIND")
.unwrap_or_else(|_| DEFAULT_TAP_BIND.to_string())
.parse()
.expect("RUTSTER_TAP_BIND must be a valid SocketAddr");
let voice =
std::env::var("OPENAI_REALTIME_VOICE").unwrap_or_else(|_| DEFAULT_VOICE.to_string());
let model =
std::env::var("OPENAI_REALTIME_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string());
let _ = &model; // used only in the non-mock branch below.
#[cfg(feature = "mock")]
let openai_url: String;
#[cfg(feature = "mock")]
let openai_headers: Vec<(String, String)>;
#[cfg(feature = "mock")]
let _mock_guard;
#[cfg(not(feature = "mock"))]
let (openai_url, openai_headers) = {
let api_key = match api_key::load_api_key() {
Ok(k) => k,
Err(e) => {
eprintln!("failed to load OPENAI_API_KEY: {e}; refusing to start");
std::process::exit(1);
}
};
(
openai_client::openai_realtime_url(&model),
openai_client::openai_headers(&api_key),
)
};
#[cfg(feature = "mock")]
{
let mock = MockRealtimeBrain::start().await.expect("mock bind ok");
info!(mock_addr = %mock.addr, "MockRealtimeBrain started (--features=mock)");
openai_url = mock.url();
openai_headers = Vec::new();
_mock_guard = mock;
}
let listener = tokio::net::TcpListener::bind(tap_bind)
.await
.expect("bind ok");
info!(%tap_bind, %openai_url, "rutster-brain-realtime listening (core dials this)");
loop {
match listener.accept().await {
Ok((stream, peer)) => {
let url = openai_url.clone();
let hdrs = openai_headers.clone();
let voice = voice.clone();
tokio::spawn(async move {
if let Err(e) = handle_tap_connection(stream, peer, &url, &hdrs, &voice).await {
warn!(error = ?e, %peer, "tap connection ended with error");
}
});
}
Err(e) => warn!(error = %e, "accept failed; continuing"),
}
}
}
/// Handle one tap WS connection from the core. Spawns two mpsc forwarders
/// (tap→pump, pump→tap) + dials the OpenAI side, then runs `run_openai_pump`
/// on the OpenAI side + awaits both forwarder tasks.
///
/// The tap-side handshake (hello/session_end) is handled here; the OpenAI
/// pump sees only audio_in + function_call_output on its inbound + emits
/// only audio_out/speech_*/function_call/error on its outbound.
async fn handle_tap_connection(
stream: tokio::net::TcpStream,
peer: SocketAddr,
openai_url: &str,
openai_headers: &[(String, String)],
voice: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let mut tap_ws = tokio_tungstenite::accept_async(stream).await?;
info!(%peer, "tap WS connection accepted");
// === Tap handshake: first frame must be hello; send hello ack. ===
let hello_in = tap_ws
.next()
.await
.ok_or("tap connection closed before hello")??;
let hello_text = hello_in.into_text().map_err(|_| "hello not text")?;
let decoded = rutster_tap::decode_envelope(&hello_text)?;
let session_id = match decoded.payload {
rutster_tap::DecodedPayload::Hello(p) => p.session_id,
_ => return Err("first tap frame not hello".into()),
};
info!(%peer, %session_id, "tap hello received");
let ack = rutster_tap::encode_hello(&session_id, 0, 0)?;
tap_ws.send(Message::Text(ack)).await?;
// === Bridge channels between tap WS and OpenAI pump. ===
// tap_via (Sender): the core's tap frames flow IN here — the forwarder
// reads text from tap_ws and forwards via this channel. The OpenAI
// pump's `tap_rx: mpsc::Receiver<String>` is the receiving end.
// pump_tap_tx (Sender): the pump's outbound tap frames (audio_out,
// speech_started, function_call, error) flow OUT here. The forwarder
// reads from the paired Receiver and writes to tap_ws.
let (tap_via, pump_tap_rx) = mpsc::channel::<String>(64);
let (pump_tap_tx, mut tap_out_rx) = mpsc::channel::<String>(64);
// Split the tap WS into sink + stream so the two forwarder tasks can
// own each half independently. `tokio_tungstenite::WebSocketStream::split`
// yields `(SplitSink<..., Message>, SplitStream<...>)` — the canonical
// tokio pattern for fan-out into two parallel async tasks.
let (mut tap_sink, mut tap_stream) = tap_ws.split();
// === Forwarder: tap_ws_stream → pump_tap_rx ===
let in_fwd = tokio::spawn(async move {
while let Some(msg_res) = tap_stream.next().await {
match msg_res {
Ok(m) => {
if let Ok(text) = m.into_text() {
if tap_via.send(text).await.is_err() {
break; // pump dropped
}
}
}
Err(e) => {
warn!(error = ?e, "tap_ws recv error");
break;
}
}
}
});
// === Forwarder: pump_tap_tx → tap_ws_sink ===
let out_fwd = tokio::spawn(async move {
while let Some(text) = tap_out_rx.recv().await {
if let Err(e) = tap_sink.send(Message::Text(text)).await {
warn!(error = ?e, "tap_ws send error");
break;
}
}
});
// === Dial the OpenAI side (real or mock). ===
let request = openai_url.into_client_request()?;
let mut request = request;
for (k, v) in openai_headers {
request.headers_mut().insert(
http::HeaderName::from_bytes(k.as_bytes())?,
http::HeaderValue::from_str(v)?,
);
}
let (openai_ws, _resp) = tokio_tungstenite::connect_async(request).await?;
info!(%peer, %openai_url, "OpenAI side connected");
// `run_openai_pump` sends its own session.update on startup per
// `build_openai_session_update`; don't pre-send here.
let _ = build_openai_session_update;
info!(%peer, voice = %voice, "starting OpenAI pump");
let pump_result =
openai_client::run_openai_pump(openai_ws, pump_tap_rx, pump_tap_tx, voice.to_string())
.await;
match pump_result {
Ok(()) => info!(%peer, "OpenAI pump exited cleanly"),
Err(e) => warn!(%peer, error = ?e, "OpenAI pump exited with error"),
}
in_fwd.abort();
out_fwd.abort();
Ok(())
}

View File

@@ -0,0 +1,328 @@
//! # MockRealtimeBrain — in-process fake OpenAI Realtime WS server (spec §5.3)
//!
//! Behind `--features=mock` only. Runs entirely in-process; no network calls
//! to OpenAI, no API key required. Powers:
//! - the offline dev loop (`cargo run -p rutster-brain-realtime --features=mock`
//! starts the brain process with this mock as the OpenAI side), and
//! - slice-3's integration test (no real credentials, deterministic canned
//! responses).
//!
//! # Contract
//!
//! - Accepts a WS client connection (the brain process's OpenAI-side client).
//! - Asserts that the first event the client sends is a `session.update`
//! with `turn_detection: null` (the S4 load-bearing decision — spec §4.3).
//! - On each `input_audio_buffer.append` event the client sends, replies
//! with a canned `response.audio.delta` event carrying base64 LE i16 PCM
//! (480 zeroed samples per 20 ms frame — same wire shape as slice-2's
//! audio_out, so the brain's translator round-trips through the same
//! PcmFrame codec real OpenAI would use).
//! - Echoes back `input_audio_buffer.speech_started` / `.speech_stopped`
//! so the brain's translator can forward them as advisory tap events
//! (spec §3.2 — these are advisory in slice-3; step 4 wires the FOB
//! reflex loop).
//! - On `response.function_call_arguments.done`... the brain's translator
//! consumes those (OpenAI emits them, the brain forwards to core); the
//! mock doesn't generate unprompted function_call events (the integration
//! test exercises the function-call round-trip via a deterministic test
//! hook, not via the mock's own drive).
use std::net::SocketAddr;
use futures_util::{SinkExt, StreamExt};
use serde_json::{Value, json};
use tokio::net::TcpListener;
use tokio::sync::oneshot;
use tokio::task::JoinHandle;
use tokio_tungstenite::tungstenite::Message;
use tracing::{debug, info, warn};
/// The handle returned by `MockRealtimeBrain::start`. Drop or call `shutdown`
/// to stop the mock server + abort its accept-loop task.
pub struct MockRealtimeBrain {
/// The address the mock bound. The brain process dials this URL
/// (`ws://<addr>/`) instead of the real `wss://api.openai.com/...`
/// when running with `--features=mock`.
pub addr: SocketAddr,
pub shutdown: Option<oneshot::Sender<()>>,
pub join: JoinHandle<()>,
}
impl MockRealtimeBrain {
/// Start the mock on an ephemeral port (caller picks up `addr` for the
/// brain process to dial). The accept loop spawns per-connection tasks
/// so multiple sessions can run concurrently in the integration test.
pub async fn start() -> std::io::Result<Self> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr = listener.local_addr()?;
let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
let join = tokio::spawn(accept_loop(listener, shutdown_rx));
info!(%addr, "MockRealtimeBrain listening (fake OpenAI Realtime)");
Ok(Self {
addr,
shutdown: Some(shutdown_tx),
join,
})
}
/// Build the URL the brain process's OpenAI client should dial instead
/// of `wss://api.openai.com/v1/realtime?model=...`. The path is `/v1/realtime`
/// so the brain client's `into_client_request` call works the same as it
/// does against the real OpenAI endpoint.
pub fn url(&self) -> String {
format!("ws://{}/v1/realtime?model=mock", self.addr)
}
}
impl Drop for MockRealtimeBrain {
fn drop(&mut self) {
// Best-effort shutdown signal — the task may already have exited.
// `Option::take()` lets us send by-value without moving out of
// `self` (Drop takes `&mut self`, not `self`).
if let Some(tx) = self.shutdown.take() {
let _ = tx.send(());
}
self.join.abort();
}
}
async fn accept_loop(listener: TcpListener, mut shutdown: oneshot::Receiver<()>) {
loop {
tokio::select! {
biased;
_ = &mut shutdown => {
info!("MockRealtimeBrain accept loop shutting down");
return;
}
res = listener.accept() => {
let (stream, peer) = match res {
Ok(s) => s,
Err(e) => {
warn!(error = ?e, "MockRealtimeBrain accept failed; continuing");
continue;
}
};
tokio::spawn(handle_connection(stream, peer));
}
}
}
}
/// Handle one brain-process → mock-OpenAI WS connection. Each connection is
/// one OpenAI Realtime session (stateless across reconnects — same as the
/// brain process's own contract for the tap side).
async fn handle_connection(stream: tokio::net::TcpStream, peer: SocketAddr) {
let ws = tokio_tungstenite::accept_async(stream).await;
let Ok(mut ws) = ws else {
warn!(%peer, "MockRealtimeBrain WS handshake failed");
return;
};
debug!(%peer, "MockRealtimeBrain connection accepted");
// First event MUST be session.update with turn_detection: null.
// Wait for it (bounded 2s — the brain process sends it immediately
// after the WS handshake per `run_openai_pump`).
let first = tokio::time::timeout(std::time::Duration::from_secs(2), ws.next()).await;
match first {
Ok(Some(Ok(msg))) => {
let Ok(text) = msg.into_text() else {
warn!(%peer, "MockRealtimeBrain first frame not text; closing");
let _ = ws.close(None).await;
return;
};
let event: Value = match serde_json::from_str(&text) {
Ok(v) => v,
Err(e) => {
warn!(%peer, error = ?e, "MockRealtimeBrain first frame not JSON; closing");
let _ = ws.close(None).await;
return;
}
};
if event.get("type").and_then(|v| v.as_str()) != Some("session.update") {
warn!(%peer, "MockRealtimeBrain first event not session.update; closing");
let _ = ws.close(None).await;
return;
}
// S4 load-bearing assertion (spec §4.3). Do NOT silently accept
// a session.update that has turn_detection != null — that would
// mask a regression in the brain's translator.
let td = event.pointer("/session/turn_detection");
if td != Some(&Value::Null) {
warn!(%peer, td = ?td, "MockRealtimeBrain S4 violation: turn_detection != null; closing");
// Echo back an OpenAI-shaped error so the brain surfaces it.
let err = json!({
"type": "error",
"error": {
"code": "invalid_session_update",
"message": "MockRealtimeBrain: turn_detection must be null (S4)"
}
});
let _ = ws.send(Message::Text(err.to_string())).await;
let _ = ws.close(None).await;
return;
}
info!(%peer, "MockRealtimeBrain session.update accepted (turn_detection=null)");
}
_ => {
warn!(%peer, "MockRealtimeBrain no session.update within 2s; closing");
let _ = ws.close(None).await;
return;
}
}
// Pump loop: reply to input_audio_buffer.append with canned audio.
while let Some(msg_res) = ws.next().await {
let msg = match msg_res {
Ok(m) => m,
Err(e) => {
debug!(%peer, error = ?e, "MockRealtimeBrain WS recv error; closing");
return;
}
};
let Ok(text) = msg.into_text() else { continue };
let event: Value = match serde_json::from_str(&text) {
Ok(v) => v,
Err(e) => {
debug!(%peer, error = ?e, "MockRealtimeBrain non-JSON event; ignoring");
continue;
}
};
let event_type = event.get("type").and_then(|v| v.as_str()).unwrap_or("");
match event_type {
"input_audio_buffer.append" => {
// Canned response: one `response.audio.delta` per append
// carrying 480 zeroed samples as base64 LE i16 (the same
// wire shape as slice-2's audio_out — verified by the
// translator's existing round-trip test). Identical bytes
// every time: deterministic for test assertions.
use base64::Engine;
let pcm_zeros = vec![0u8; 960]; // 480 i16 samples × 2 bytes
let pcm_b64 = base64::engine::general_purpose::STANDARD.encode(&pcm_zeros);
let delta = json!({
"type": "response.audio.delta",
"delta": pcm_b64
});
if let Err(e) = ws.send(Message::Text(delta.to_string())).await {
debug!(%peer, error = ?e, "MockRealtimeBrain send delta failed; closing");
return;
}
}
"input_audio_buffer.speech_started" => {
// Pass-through echo so the brain's translator forwards it as
// a tap `speech_started` advisory event.
let evt = json!({ "type": "input_audio_buffer.speech_started" });
let _ = ws.send(Message::Text(evt.to_string())).await;
}
"input_audio_buffer.speech_stopped" => {
let evt = json!({ "type": "input_audio_buffer.speech_stopped" });
let _ = ws.send(Message::Text(evt.to_string())).await;
}
"conversation.item.create" => {
// The brain sends this to relay a function_call_output (tool
// reply) back to OpenAI. The mock just acknowledges by
// echoing a typed `conversation.item.created` event so the
// brain's OpenAI pump doesn't treat the silence as an error.
let evt = json!({
"type": "conversation.item.created",
"item": event.get("item").cloned().unwrap_or(Value::Null)
});
let _ = ws.send(Message::Text(evt.to_string())).await;
}
_ => {
debug!(%peer, event_type = %event_type, "MockRealtimeBrain ignoring event");
}
}
}
debug!(%peer, "MockRealtimeBrain connection closed");
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
/// The S4 turn-ownership decision is load-bearing (spec §4.3 + ADR-0008).
/// The mock asserts the brain's session.update has turn_detection: null;
/// this test confirms a session.update WITH turn_detection: null is
/// accepted (the green path).
#[tokio::test]
async fn accepts_session_update_with_turn_detection_null() {
let mock = MockRealtimeBrain::start().await.unwrap();
let url = mock.url();
let req = url.as_str().into_client_request().unwrap();
let (mut ws, _resp) = tokio_tungstenite::connect_async(req).await.unwrap();
// Send session.update with turn_detection: null.
let session_update = json!({
"type": "session.update",
"session": {
"modalities": ["text", "audio"],
"voice": "alloy",
"turn_detection": null
}
});
ws.send(Message::Text(session_update.to_string()))
.await
.unwrap();
// Send input_audio_buffer.append — must receive a canned
// response.audio.delta (the brain's translator round-trips this
// through encode_audio_out, so the wire shape must be exactly what
// real OpenAI sends: base64 PCM16 in a `delta` field).
let append = json!({
"type": "input_audio_buffer.append",
"audio": "AAAA"
});
ws.send(Message::Text(append.to_string())).await.unwrap();
let delta_msg = tokio::time::timeout(Duration::from_millis(500), ws.next())
.await
.expect("delta within 500ms")
.unwrap()
.unwrap();
let delta_text = delta_msg.into_text().unwrap();
let delta: Value = serde_json::from_str(&delta_text).unwrap();
assert_eq!(delta["type"], "response.audio.delta");
assert!(delta["delta"].is_string());
// The mock's canned PCM is 480 zeroed samples; the base64 string
// must decode to exactly 960 bytes (480 × 2 bytes/i16).
use base64::Engine;
let bytes = base64::engine::general_purpose::STANDARD
.decode(delta["delta"].as_str().unwrap())
.unwrap();
assert_eq!(bytes.len(), 960);
}
/// The S4 violation: session.update with turn_detection != null
/// surfaces a typed OpenAI-shaped error so the brain's pump fails fast
/// + visibly. Catches a future regression where the brain process
/// forgets to set turn_detection: null.
#[tokio::test]
async fn rejects_session_update_with_turn_detection_set() {
let mock = MockRealtimeBrain::start().await.unwrap();
let url = mock.url();
let req = url.as_str().into_client_request().unwrap();
let (mut ws, _resp) = tokio_tungstenite::connect_async(req).await.unwrap();
let session_update = json!({
"type": "session.update",
"session": {
"voice": "alloy",
"turn_detection": { "type": "server_vad" }
}
});
ws.send(Message::Text(session_update.to_string()))
.await
.unwrap();
let err_msg = tokio::time::timeout(Duration::from_millis(500), ws.next())
.await
.expect("error within 500ms")
.unwrap()
.unwrap();
let err: Value = serde_json::from_str(&err_msg.into_text().unwrap()).unwrap();
assert_eq!(err["type"], "error");
assert_eq!(err["error"]["code"], "invalid_session_update");
}
}

View File

@@ -0,0 +1,222 @@
//! # wss://api.openai.com/v1/realtime client (spec §4)
//!
//! Owns the OpenAI-side WS connection. Brings the OpenAI Realtime event
//! stream up, sends `session.update` with `turn_detection: null` (S4,
//! spec §4.3) on handshake, then runs a pump loop that:
//! - reads tap-side events (audio_in, function_call_output) from the
//! tap WS server's `WebSocketStream` and translates + forwards them
//! to OpenAI;
//! - reads OpenAI events (response.audio.delta, speech_started/stopped,
//! function_call_arguments.done, error) and translates + forwards them
//! to the tap WS server.
//!
//! Reconnects with bounded backoff on OpenAI-side WS failure (spec §4.4 —
//! the tap side stays connected; the OpenAI side has its own failure
//! surface). The brain process emits a tap `error` event on OpenAI-side
//! failure so the core can observe it.
// Imports are split by origin so the reader can trace which types come from
// the wire-shape (rutster-tap — shared with the core), which come from the
// pure translators (crate::translator — this crate's pure-function layer),
// and which come from third-party plumbing.
use crate::translator::{
build_openai_session_update, openai_audio_delta_to_tap_audio_out,
openai_function_call_arguments_done_to_tap, openai_speech_event_to_tap,
tap_audio_in_to_openai_append, tap_function_call_output_to_openai_create_item,
};
use rutster_tap::{DecodedPayload, TapProtoError, decode_envelope, encode_function_call};
use serde_json::Value;
use thiserror::Error;
use tokio::sync::mpsc;
use tokio_tungstenite::tungstenite::Message;
use tracing::{info, warn};
#[derive(Debug, Error)]
pub enum OpenAiClientError {
#[error("OpenAI WS error: {0}")]
Ws(#[from] tokio_tungstenite::tungstenite::Error),
#[error("OpenAI auth failed (401)")]
AuthFailed,
#[error("translator error: {0}")]
Translate(#[from] crate::translator::TranslateError),
#[error("tap protocol error: {0}")]
TapProto(#[from] TapProtoError),
}
/// Build the OpenAI Realtime URL for the given model. Spec §4.2 + §5.3:
/// `wss://api.openai.com/v1/realtime?model=<model>`.
pub fn openai_realtime_url(model: &str) -> String {
format!("wss://api.openai.com/v1/realtime?model={model}")
}
/// Build the HTTP headers for the OpenAI WS handshake (spec §5.3). Two
/// required headers:
/// - `Authorization: Bearer <api_key>`
/// - `OpenAI-Beta: realtime=v1`
pub fn openai_headers(api_key: &str) -> Vec<(String, String)> {
vec![
("Authorization".to_string(), format!("Bearer {api_key}")),
("OpenAI-Beta".to_string(), "realtime=v1".to_string()),
]
}
/// Drive the OpenAI Realtime WS connection + the tap-side pump.
///
/// `tap_rx`: inbound side — *tap frames* the brain received from the core
/// (audio_in, function_call_output). We translate each one to its OpenAI
/// equivalent + send to OpenAI.
/// `openai_ws`: the OpenAI WS connection (already connected; the caller
/// does the dial + auth).
/// `tap_tx`: outbound side — *tap frames* the brain sends back to the core
/// (audio_out, speech_started/stopped, function_call, error). We
/// translate OpenAI events into these + send.
pub async fn run_openai_pump<T>(
mut openai_ws: tokio_tungstenite::WebSocketStream<T>,
mut tap_rx: mpsc::Receiver<String>,
tap_tx: mpsc::Sender<String>,
voice: String,
) -> Result<(), OpenAiClientError>
where
T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin,
{
// === Handshake: send session.update with turn_detection: null (S4). ===
let session_update = build_openai_session_update(&voice);
openai_ws
.send(Message::Text(session_update.to_string()))
.await?;
info!(voice = %voice, "sent session.update to OpenAI (turn_detection: null)");
let mut seq_egress = 0u64;
use futures_util::{SinkExt, StreamExt};
loop {
tokio::select! {
// Inbound tap frame from the core (audio_in, function_call_output).
tap_str = tap_rx.recv() => {
let Some(tap_str) = tap_str else {
info!("tap_rx closed; ending OpenAI pump");
return Ok(());
};
let decoded = decode_envelope(&tap_str)?;
match decoded.payload {
DecodedPayload::AudioIn(audio) => {
let append = tap_audio_in_to_openai_append(&audio.pcm);
openai_ws.send(Message::Text(append.to_string())).await?;
}
DecodedPayload::FunctionCallOutput(out) => {
let create_item = tap_function_call_output_to_openai_create_item(
&out.id, &out.status, &out.result,
);
openai_ws.send(Message::Text(create_item.to_string())).await?;
}
// Ignore others; slice-2's hello/audio_in on the tap side
// happens before this pump starts.
_ => warn!(?decoded.payload, "unexpected tap frame to OpenAI pump"),
}
}
// Inbound OpenAI event.
msg = openai_ws.next() => {
let Some(msg) = msg else {
info!("OpenAI WS stream ended");
return Ok(());
};
let msg = msg?;
let Ok(text) = msg.into_text() else { continue };
let openai_event: Value = match serde_json::from_str(&text) {
Ok(v) => v,
Err(e) => {
warn!(error = ?e, "OpenAI sent non-JSON event; ignoring");
continue;
}
};
let event_type = openai_event.get("type").and_then(|v| v.as_str()).unwrap_or("");
match event_type {
"response.audio.delta" => {
let tap_str = openai_audio_delta_to_tap_audio_out(
&openai_event, seq_egress, 0,
)?;
seq_egress += 1;
tap_tx.send(tap_str).await.map_err(|_| {
OpenAiClientError::Ws(
tokio_tungstenite::tungstenite::Error::ConnectionClosed,
)
})?;
}
"input_audio_buffer.speech_started" => {
let tap_str = openai_speech_event_to_tap(
&openai_event, true, seq_egress, 0,
)?;
seq_egress += 1;
tap_tx.send(tap_str).await.map_err(|_| {
OpenAiClientError::Ws(
tokio_tungstenite::tungstenite::Error::ConnectionClosed,
)
})?;
}
"input_audio_buffer.speech_stopped" => {
let tap_str = openai_speech_event_to_tap(
&openai_event, false, seq_egress, 0,
)?;
seq_egress += 1;
tap_tx.send(tap_str).await.map_err(|_| {
OpenAiClientError::Ws(
tokio_tungstenite::tungstenite::Error::ConnectionClosed,
)
})?;
}
"response.function_call_arguments.done" => {
let (call_id, name, args_str) =
openai_function_call_arguments_done_to_tap(&openai_event)?;
let tap_str = encode_function_call(
&call_id, &name, &args_str, seq_egress, 0,
)?;
seq_egress += 1;
tap_tx.send(tap_str).await.map_err(|_| {
OpenAiClientError::Ws(
tokio_tungstenite::tungstenite::Error::ConnectionClosed,
)
})?;
}
"error" => {
// OpenAI emits a typed error event (e.g. 401 auth
// failure surfaces here). Inspect the .error.code.
let code = openai_event
.pointer("/error/code")
.and_then(|v| v.as_str())
.unwrap_or("unknown");
warn!(code = %code, "OpenAI error event");
if code == "invalid_api_key" {
return Err(OpenAiClientError::AuthFailed);
}
}
_ => {
// Unknown OpenAI event — log + drop (don't crash).
warn!(event_type = %event_type, "ignoring OpenAI event type");
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn openai_url_for_known_model() {
assert_eq!(
openai_realtime_url("gpt-4o-realtime"),
"wss://api.openai.com/v1/realtime?model=gpt-4o-realtime"
);
}
#[test]
fn openai_headers_carry_bearer_auth_and_beta() {
let h = openai_headers("sk-test-12345");
assert_eq!(h[0].0, "Authorization");
assert_eq!(h[0].1, "Bearer sk-test-12345");
assert_eq!(h[1].0, "OpenAI-Beta");
assert_eq!(h[1].1, "realtime=v1");
}
}

View File

@@ -0,0 +1,261 @@
//! # Tap ⇄ OpenAI Realtime event translation (spec §4)
//!
//! Pure functions — no async, no I/O, no call state. Each function maps
//! one event between slice-3's tap protocol (slice-2 v1 + the additive
//! slice-3 events from Task 2) and OpenAI Realtime's event JSON.
//!
//! The translation layer is **stateless** by design: the OpenAI-side WS
//! client (Task 5) and the tap-side WS server (Task 9) call these
//! functions per-event. No ownership of OpenAI's `session_id` beyond
//! the connection's lifetime.
use rutster_tap::{TapProtoError, encode_audio_out, encode_speech_started, encode_speech_stopped};
use serde_json::{Value, json};
use thiserror::Error;
#[derive(Debug, Error)]
pub enum TranslateError {
#[error("OpenAI event missing required field: {field}")]
MissingField { field: &'static str },
#[error("tap protocol error: {0}")]
TapProto(#[from] TapProtoError),
}
/// Build OpenAI's `session.update` event (spec §4.2). The S4 turn-ownership
/// decision (spec §4.3): `turn_detection: null`. OpenAI's server-side VAD
/// is disabled; the FOB reflex loop (step 4) owns turn-taking.
pub fn build_openai_session_update(voice: &str) -> Value {
json!({
"type": "session.update",
"session": {
"modalities": ["text", "audio"],
"voice": voice,
"input_audio_format": "pcm16",
"output_audio_format": "pcm16",
"sample_rate": 24000,
"turn_detection": null
}
})
}
/// Wrap a base64 PCM payload (already encoded per slice-2 §3 — explicit LE
/// i16) into OpenAI's `input_audio_buffer.append` event (spec §4.2). Pass-
/// through — the wire shape for OpenAI's audio is identical to slice-2's
/// tap PCM (spec §3.5).
pub fn tap_audio_in_to_openai_append(pcm_b64: &str) -> Value {
json!({
"type": "input_audio_buffer.append",
"audio": pcm_b64
})
}
/// Translate OpenAI's `response.audio.delta` event (carries a `delta` field
/// with base64 PCM) to a slice-3 tap `audio_out` frame string (spec §4.2).
/// Pass-through on the audio payload; the envelope carries seq/ts.
pub fn openai_audio_delta_to_tap_audio_out(
openai_event: &Value,
seq: u64,
ts: u64,
) -> Result<String, TranslateError> {
let pcm_b64 = openai_event
.get("delta")
.and_then(|v| v.as_str())
.ok_or(TranslateError::MissingField { field: "delta" })?;
// OpenAI's base64 PCM is LE i16 24 kHz mono — identical wire shape to
// slice-2's tap PCM. Decode + re-encode as a PcmFrame (the canonical
// 480-sample shape) so slice-2's playout ring stays byte-aligned.
let frame = rutster_tap::decode_pcm(pcm_b64, rutster_tap::WIRE_SAMPLES_PER_FRAME)?;
Ok(encode_audio_out(&frame, seq, ts)?)
}
/// Translate OpenAI's `input_audio_buffer.speech_started` (started=true) or
/// `.speech_stopped` (started=false) event to the matching tap frame
/// (spec §4.2 + §4.3 — advisory events, the FOB reflex loop in step 4 will
/// act on them).
pub fn openai_speech_event_to_tap(
_openai_event: &Value,
started: bool,
seq: u64,
ts: u64,
) -> Result<String, TranslateError> {
if started {
Ok(encode_speech_started(seq, ts)?)
} else {
Ok(encode_speech_stopped(seq, ts)?)
}
}
/// Translate OpenAI's `response.function_call_arguments.done` event to the
/// tuple `(call_id, name, args_json_str)` the brain will emit as a tap
/// `function_call` frame (spec §4.2 + §4.3). OpenAI sends `arguments` as
/// a JSON _string_, not a JSON object — we don't reparse it; the brain
/// process passes the raw string to `encode_function_call` (Task 2's
/// encode_function_call parses it then into a serde_json::Value).
pub fn openai_function_call_arguments_done_to_tap(
openai_event: &Value,
) -> Result<(String, String, String), TranslateError> {
let call_id = openai_event
.get("call_id")
.and_then(|v| v.as_str())
.ok_or(TranslateError::MissingField { field: "call_id" })?;
let name = openai_event
.get("name")
.and_then(|v| v.as_str())
.ok_or(TranslateError::MissingField { field: "name" })?;
// OpenAI's `arguments` is a JSON string. If missing, default to "{}".
let args_json_str = openai_event
.get("arguments")
.and_then(|v| v.as_str())
.unwrap_or("{}")
.to_string();
Ok((call_id.to_string(), name.to_string(), args_json_str))
}
/// Build OpenAI's `conversation.item.create` event for a tool-call result
/// (spec §4.2). The tap `function_call_output` carries id + status + result;
/// OpenAI's `conversation.item.create` wraps them as a function_call_output
/// item with `call_id` (= id) + `output` (= JSON-stringified result; status
/// is plumbed into the result value as a `_status` field — OpenAI has no
/// formal status concept, so we encode ours in the result body).
pub fn tap_function_call_output_to_openai_create_item(
id: &str,
status: &str,
result: &Value,
) -> Value {
let mut output_value = result.clone();
if let Some(obj) = output_value.as_object_mut() {
obj.insert("_status".to_string(), Value::String(status.to_string()));
}
json!({
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": id,
"output": output_value.to_string()
}
})
}
#[cfg(test)]
mod tests {
use super::*;
use rutster_tap::{DecodedPayload, decode_envelope};
/// S4 turn-ownership test (load-bearing per ADR-0008, spec §4.3):
/// the OpenAI session.update must have turn_detection: null.
#[test]
fn session_update_disables_openai_turn_detection() {
let v = build_openai_session_update("alloy");
assert_eq!(v["type"], "session.update");
assert_eq!(v["session"]["voice"], "alloy");
assert_eq!(v["session"]["modalities"][0], "text");
assert_eq!(v["session"]["modalities"][1], "audio");
assert_eq!(v["session"]["input_audio_format"], "pcm16");
assert_eq!(v["session"]["output_audio_format"], "pcm16");
assert_eq!(v["session"]["sample_rate"], 24000);
// THE load-bearing assertion:
assert_eq!(v["session"]["turn_detection"], Value::Null);
}
#[test]
fn append_audio_payload_is_passthrough() {
// The PCM base64 string for one zero frame (480 samples, every LE i16=0).
use base64::Engine;
let zeros = [0u8; 960];
let pcm_b64 = base64::engine::general_purpose::STANDARD.encode(zeros);
let v = tap_audio_in_to_openai_append(&pcm_b64);
assert_eq!(v["type"], "input_audio_buffer.append");
assert_eq!(v["audio"], pcm_b64);
}
#[test]
fn openai_audio_delta_to_tap_round_trips_through_slice2_codec() {
// Build a slice-2 PcmFrame, base64-encode it the slice-2 way,
// wrap it in an OpenAI response.audio.delta, translate to a tap
// audio_out frame, then decode the tap frame with decode_envelope
// and assert the PCM round-trips.
use rutster_media::PcmFrame;
let mut frame = PcmFrame::zeroed();
frame.samples[0] = 100;
frame.samples[1] = -100;
frame.samples[479] = 12345;
let pcm_b64 = rutster_tap::encode_pcm(&frame);
let openai_event = json!({
"type": "response.audio.delta",
"delta": pcm_b64
});
let tap_str = openai_audio_delta_to_tap_audio_out(&openai_event, 5, 1000).unwrap();
let decoded = decode_envelope(&tap_str).unwrap();
match decoded.payload {
DecodedPayload::AudioOut(p) => {
let recovered = rutster_tap::decode_pcm(&p.pcm, p.samples).unwrap();
assert_eq!(recovered.samples[0], 100);
assert_eq!(recovered.samples[1], -100);
assert_eq!(recovered.samples[479], 12345);
assert_eq!(decoded.seq, 5);
assert_eq!(decoded.ts, 1000);
}
other => panic!("expected AudioOut, got {other:?}"),
}
}
#[test]
fn speech_started_translates_to_tap_speech_started() {
let openai_event = json!({ "type": "input_audio_buffer.speech_started" });
let s = openai_speech_event_to_tap(&openai_event, true, 3, 300).unwrap();
let d = decode_envelope(&s).unwrap();
assert!(matches!(d.payload, DecodedPayload::SpeechStarted));
assert_eq!(d.seq, 3);
}
#[test]
fn speech_stopped_translates_to_tap_speech_stopped() {
let openai_event = json!({ "type": "input_audio_buffer.speech_stopped" });
let s = openai_speech_event_to_tap(&openai_event, false, 7, 700).unwrap();
let d = decode_envelope(&s).unwrap();
assert!(matches!(d.payload, DecodedPayload::SpeechStopped));
assert_eq!(d.seq, 7);
}
#[test]
fn function_call_done_extracts_three_tuple() {
let openai_event = json!({
"type": "response.function_call_arguments.done",
"call_id": "call_abc123",
"name": "hangup",
"arguments": "{\"reason\": \"caller_requested\"}"
});
let (id, name, args_str) =
openai_function_call_arguments_done_to_tap(&openai_event).unwrap();
assert_eq!(id, "call_abc123");
assert_eq!(name, "hangup");
assert_eq!(args_str, "{\"reason\": \"caller_requested\"}");
}
#[test]
fn function_call_done_missing_call_id_errors() {
let openai_event = json!({
"type": "response.function_call_arguments.done",
"name": "hangup"
});
let r = openai_function_call_arguments_done_to_tap(&openai_event);
assert!(matches!(
r,
Err(TranslateError::MissingField { field: "call_id" })
));
}
#[test]
fn function_call_output_to_create_item_plumbs_status_into_result() {
let result = json!({"channel_state": "Closing"});
let v = tap_function_call_output_to_openai_create_item("call_abc123", "ok", &result);
assert_eq!(v["type"], "conversation.item.create");
assert_eq!(v["item"]["type"], "function_call_output");
assert_eq!(v["item"]["call_id"], "call_abc123");
// The output is a JSON string — _status gets plumbed inside.
let output_str = v["item"]["output"].as_str().unwrap();
let output_val: Value = serde_json::from_str(output_str).unwrap();
assert_eq!(output_val["channel_state"], "Closing");
assert_eq!(output_val["_status"], "ok");
}
}

View File

@@ -0,0 +1,88 @@
//! slice-3 integration test: brain-process-level round-trip through the
//! MockRealtimeBrain + the translator + `run_openai_pump`. Drives the brain
//! the way the core's TapClient would (synthetic tap frames in, observes tap
//! frames out) — proves the dev-loop interop contract the PM directive's
//! Task 4 calls out ("actually start + interop on a basic audio round-trip")
//! without requiring a real WebRTC peer (browser-driven, same constraint as
//! slice-1/2).
//!
//! What this exercises end-to-end:
//! 1. MockRealtimeBrain accepts a WS connection + asserts session.update
//! has turn_detection: null (S4 — spec §4.3).
//! 2. `run_openai_pump` sends session.update + bridges tap frames ↔ OpenAI
//! events via the translator.
//! 3. A tap `audio_in` frames the test pushes through `tap_via` → translator
//! formats as `input_audio_buffer.append` → mock generates a canned
//! `response.audio.delta` → translator formats as tap `audio_out` →
//! arrives on `pump_tap_tx` (the pump's outbound mpsc).
use rutster_brain_realtime::{MockRealtimeBrain, openai_client::run_openai_pump};
use rutster_tap::{PcmFrame, decode_envelope, encode_audio_in, encode_hello};
use serde_json::Value;
use std::time::Duration;
use tokio::sync::mpsc;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
#[tokio::test]
async fn brain_mock_round_trip_translates_audio_in_to_audio_out() {
let mock = MockRealtimeBrain::start().await.unwrap();
let url = mock.url();
let req = url.as_str().into_client_request().unwrap();
let (openai_ws, _resp) = tokio_tungstenite::connect_async(req).await.unwrap();
// Bridge channels matching `run_openai_pump`'s signature (it owns neither
// side of the tap WS — the binary's main.rs does the WS forwarding; here
// we drive the pump directly with mpsc as the test's tap peer).
let (tap_via, pump_tap_rx) = mpsc::channel::<String>(64);
let (pump_tap_tx, mut tap_out_rx) = mpsc::channel::<String>(64);
// The pump sends session.update + then runs the select! loop. Spawn it.
let pump = tokio::spawn(async move {
run_openai_pump(openai_ws, pump_tap_rx, pump_tap_tx, "alloy".to_string()).await
});
// Push a hello (the pump ignores hello per its `_ => warn!(...)` arm,
// but real tap traffic always starts with hello).
let hello = encode_hello("test-session", 0, 0).unwrap();
tap_via.send(hello).await.unwrap();
// Push one audio_in frame. The pump translates to
// input_audio_buffer.append; the mock replies with a canned
// response.audio.delta; the pump translates back to a tap audio_out
// frame on pump_tap_tx.
let frame = PcmFrame::zeroed();
let audio_in = encode_audio_in(&frame, 1, 100).unwrap();
tap_via.send(audio_in).await.unwrap();
// Bounded-wait the audio_out frame coming back. Generous bound: the
// pump's tokio select! cycle + the mock's WS round-trip on loopback
// are sub-millisecond; 500 ms is fail-fast for any real regression.
let out_str = tokio::time::timeout(Duration::from_millis(500), tap_out_rx.recv())
.await
.expect("audio_out within 500ms")
.expect("channel not closed");
let decoded = decode_envelope(&out_str).unwrap();
match decoded.payload {
rutster_tap::DecodedPayload::AudioOut(p) => {
// The mock's canned PCM is 480 zeroed samples; the brain's
// translator round-trips it through PcmFrame (decode +
// re-encode), so the wire shape is exactly slice-2's audio_out.
assert_eq!(p.samples, 480);
let recovered = rutster_tap::decode_pcm(&p.pcm, p.samples).unwrap();
assert_eq!(recovered.samples.len(), 480);
// Canned zeros — every sample is 0.
assert_eq!(recovered.samples[0], 0);
assert_eq!(recovered.samples[479], 0);
// seq + ts carried through from the translator's per-event
// counter (initial value 0 on the first outbound frame).
assert_eq!(decoded.seq, 0);
}
other => panic!("expected AudioOut, got {other:?}"),
}
// Drop the input channel to signal the pump to exit cleanly.
drop(tap_via);
let _ = pump.await;
let _ = Value::Null; // suppress unused-import if Value unused here
}

View File

@@ -10,11 +10,12 @@
//! str0m 0.21's `Rtc::sdp_api().accept_offer(offer)` produces the SDP
//! answer natively: DTLS fingerprint (from the cert str0m generates), ICE
//! ufrag/pwd, and codec negotiation (Opus, the only codec we registered).
//! Slice 1 does NOT hand-roll an SDP munger — str0m's path is the spec's
//! "embryo of the future SIP SDP path" (§3.7). When step 5 brings SIP/SDP
//! negotiation into `rutster-signaling-sip`, that crate may extract shared
//! SDP helpers from str0m or build its own. Slice 1's WebRTC-ICE-coupled
//! SDP lives entirely in str0m.
//! Slice 1 does NOT hand-roll an SDP munger — str0m handles WebRTC
//! offer/answer natively. There is no first-party SIP/SDP path: under
//! [ADR-0007](../../../docs/adr/0007-trunk-rented-transport.md) the trunk is
//! rented (a CPaaS media-leg fork — raw audio, no SDP) or an out-of-tree SBC
//! (which owns any carrier SIP/SDP, outside the trust boundary). rutster's
//! WebRTC-ICE-coupled SDP lives entirely in str0m.
use std::net::SocketAddr;
use std::time::{Duration, Instant};

View File

@@ -41,10 +41,14 @@ pub use protocol::{
AudioPayload, DecodedFrame, DecodedPayload, Envelope, ErrorPayload, FrameKind, HelloPayload,
PROTOCOL_VERSION, Payload, ReasonPayload, SAMPLES_PER_FRAME as WIRE_SAMPLES_PER_FRAME,
SessionEndPayload, TapProtoError, decode_envelope, decode_pcm, encode_audio_in,
encode_audio_out, encode_bye, encode_error, encode_hello, encode_pcm, encode_session_end,
encode_audio_out, encode_bye, encode_error, encode_function_call, encode_function_call_output,
encode_hello, encode_pcm, encode_session_end, encode_speech_started, encode_speech_stopped,
encode_tools_update,
};
// Slice-3 additive (spec §3).
pub use protocol::{FunctionCallOutputPayload, FunctionCallPayload, ToolsUpdatePayload};
pub use tap_audio_pipe::TapAudioPipe;
pub use tap_client::{TapClientError, run_tap_client};
pub use tap_client::{FunctionCallEvent, FunctionCallOutputEvent, TapClientError, run_tap_client};
#[cfg(test)]
mod tests {

View File

@@ -47,6 +47,21 @@ pub enum FrameKind {
SessionEnd,
Bye,
Error,
/// brain → core: user speech started (advisory; translated from OpenAI
/// `input_audio_buffer.speech_started`; spec §3.2).
SpeechStarted,
/// brain → core: user speech stopped (advisory; OpenAI
/// `input_audio_buffer.speech_stopped`; spec §3.2).
SpeechStopped,
/// brain → core: brain proposes a tool call (translated from OpenAI
/// `response.function_call_arguments.done`; spec §3.2).
FunctionCall,
/// core → brain: tool registry reply (spec §3.3).
FunctionCallOutput,
/// brain → core: brain declares its tool catalog on hello + on changes
/// (spec §3.2).
#[serde(rename = "tools.update")]
ToolsUpdate,
/// Unknown wire `type` values land here (spec §3.4: log + count + drop).
/// `#[serde(other)]` catches any string not in the variants above.
#[serde(other)]
@@ -111,6 +126,11 @@ impl Serialize for Envelope {
Payload::AudioIn(_) | Payload::AudioOut(_) => 2, // pcm, samples
Payload::SessionEnd(_) | Payload::Bye(_) => 1, // reason
Payload::Error(_) => 2, // code, message
// Slice-3 adds: empty payload (0 fields), 3-field payloads, 1-field payload.
Payload::SpeechStarted | Payload::SpeechStopped => 0,
Payload::FunctionCall(_) => 3, // id, name, args
Payload::FunctionCallOutput(_) => 3, // id, status, result
Payload::ToolsUpdate(_) => 1, // tools
};
let mut st = serializer.serialize_struct("Envelope", 4 + payload_field_count)?;
st.serialize_field("v", &self.v)?;
@@ -142,6 +162,23 @@ impl Serialize for Envelope {
st.serialize_field("code", &p.code)?;
st.serialize_field("message", &p.message)?;
}
Payload::SpeechStarted | Payload::SpeechStopped => {
// No payload fields — the envelope's `v`/`type`/`seq`/`ts`
// is the whole message. The event name IS the message.
}
Payload::FunctionCall(p) => {
st.serialize_field("id", &p.id)?;
st.serialize_field("name", &p.name)?;
st.serialize_field("args", &p.args)?;
}
Payload::FunctionCallOutput(p) => {
st.serialize_field("id", &p.id)?;
st.serialize_field("status", &p.status)?;
st.serialize_field("result", &p.result)?;
}
Payload::ToolsUpdate(p) => {
st.serialize_field("tools", &p.tools)?;
}
}
st.end()
}
@@ -165,6 +202,16 @@ pub enum Payload {
SessionEnd(SessionEndPayload),
Bye(ReasonPayload),
Error(ErrorPayload),
/// Slice-3 additive (spec §3.2): brain → core, advisory; empty payload
/// (the event name IS the message).
SpeechStarted,
SpeechStopped,
/// Slice-3 additive (spec §3.2).
FunctionCall(FunctionCallPayload),
/// Slice-3 additive (spec §3.3): core → brain reply.
FunctionCallOutput(FunctionCallOutputPayload),
/// Slice-3 additive (spec §3.2): brain → core catalog declaration.
ToolsUpdate(ToolsUpdatePayload),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -211,6 +258,42 @@ pub struct ErrorPayload {
pub message: String,
}
/// `function_call` payload (brain → core; spec §3.2). Carries the
/// brain-minted id + tool name + args. Args is a raw JSON Value (not a
/// typed struct) so any tool schema is allowed.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FunctionCallPayload {
pub id: String,
pub name: String,
/// Raw JSON arguments — the tool registry dispatches by name and lets
/// each Tool impl parse the args itself. (OpenAI sends `arguments` as a
/// JSON string; the translator parses it back to a Value before
/// emitting the function_call tap frame.)
pub args: serde_json::Value,
}
/// `function_call_output` payload (core → brain; spec §3.3). The reply for
/// a `function_call`. `status` is one of `"ok"`, `"error"`, `"not_implemented"`.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FunctionCallOutputPayload {
pub id: String,
pub status: String,
pub result: serde_json::Value,
}
/// `tools.update` payload (brain → core; spec §3.2). The brain declares its
/// tool catalog so the core's tool registry can validate function_call
/// events by name.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolsUpdatePayload {
/// An array of tool descriptors (each has `name`, `description`,
/// `parameters`). The shape is intentionally permissive (a JSON array,
/// not a typed Vec<ToolSchema>) so the brain can declare schemas the
/// core doesn't know about — the core only checks the `name` field for
/// dispatch, ignores the rest.
pub tools: serde_json::Value,
}
/// Decoded frame — what `decode_envelope` returns. The kind is split into
/// `audio_in` / `audio_out` variants (not just `Audio(AudioPayload)`) so
/// callers `match` exhaustively on direction (spec §3.2 vs §3.3 — the two
@@ -231,7 +314,17 @@ pub enum DecodedPayload {
SessionEnd(SessionEndPayload),
Bye(ReasonPayload),
Error(ErrorPayload),
/// Unknown `type` — log + count + drop (spec §3.4).
/// Slice-3 (spec §3.2): the brain detected user speech started/stopped.
SpeechStarted,
SpeechStopped,
/// Slice-3 (spec §3.2): brain wants the core to execute a tool.
FunctionCall(FunctionCallPayload),
/// Slice-3 (spec §3.3): the core's tool-registry reply.
FunctionCallOutput(FunctionCallOutputPayload),
/// Slice-3 (spec §3.2): brain declares its catalog so the core can
/// validate function_call events.
ToolsUpdate(ToolsUpdatePayload),
/// Unknown `type` — log + count + drop (spec §3.4 of slice-2).
Unknown,
}
@@ -369,6 +462,111 @@ pub fn encode_error(code: &str, msg: &str, seq: u64, ts: u64) -> Result<String,
Ok(serde_json::to_string(&env)?)
}
/// Build `speech_started` (brain → core, advisory; spec §3.2).
pub fn encode_speech_started(seq: u64, ts: u64) -> Result<String, TapProtoError> {
let env = Envelope {
v: PROTOCOL_VERSION,
kind: FrameKind::SpeechStarted,
seq,
ts,
payload: Payload::SpeechStarted,
};
Ok(serde_json::to_string(&env)?)
}
/// Build `speech_stopped` (brain → core, advisory; spec §3.2).
pub fn encode_speech_stopped(seq: u64, ts: u64) -> Result<String, TapProtoError> {
let env = Envelope {
v: PROTOCOL_VERSION,
kind: FrameKind::SpeechStopped,
seq,
ts,
payload: Payload::SpeechStopped,
};
Ok(serde_json::to_string(&env)?)
}
/// Build `function_call` (brain → core; spec §3.2). `args_json_str` is the
/// raw JSON string the brain's translator parses from OpenAI's
/// `response.function_call_arguments.done.arguments` (which is itself a
/// JSON string in OpenAI's wire format).
pub fn encode_function_call(
id: &str,
name: &str,
args_json_str: &str,
seq: u64,
ts: u64,
) -> Result<String, TapProtoError> {
let args: serde_json::Value = if args_json_str.is_empty() {
serde_json::Value::Null
} else {
serde_json::from_str(args_json_str)?
};
let env = Envelope {
v: PROTOCOL_VERSION,
kind: FrameKind::FunctionCall,
seq,
ts,
payload: Payload::FunctionCall(FunctionCallPayload {
id: id.to_string(),
name: name.to_string(),
args,
}),
};
Ok(serde_json::to_string(&env)?)
}
/// Build `function_call_output` (core → brain; spec §3.3). `result_json_str`
/// is the raw JSON string the tool-registry dispatch returns (serialized
/// into the result field of the payload).
pub fn encode_function_call_output(
id: &str,
status: &str, // "ok" | "error" | "not_implemented"
result_json_str: &str,
seq: u64,
ts: u64,
) -> Result<String, TapProtoError> {
let result: serde_json::Value = if result_json_str.is_empty() {
serde_json::Value::Null
} else {
serde_json::from_str(result_json_str)?
};
let env = Envelope {
v: PROTOCOL_VERSION,
kind: FrameKind::FunctionCallOutput,
seq,
ts,
payload: Payload::FunctionCallOutput(FunctionCallOutputPayload {
id: id.to_string(),
status: status.to_string(),
result,
}),
};
Ok(serde_json::to_string(&env)?)
}
/// Build `tools.update` (brain → core; spec §3.2). `tools_json_str` is the
/// raw JSON array of tool descriptors.
pub fn encode_tools_update(
tools_json_str: &str,
seq: u64,
ts: u64,
) -> Result<String, TapProtoError> {
let tools: serde_json::Value = if tools_json_str.is_empty() {
serde_json::Value::Array(vec![])
} else {
serde_json::from_str(tools_json_str)?
};
let env = Envelope {
v: PROTOCOL_VERSION,
kind: FrameKind::ToolsUpdate,
seq,
ts,
payload: Payload::ToolsUpdate(ToolsUpdatePayload { tools }),
};
Ok(serde_json::to_string(&env)?)
}
/// Decode an incoming wire string into a `DecodedFrame`. Validates `v` and
/// (for audio frames) `samples` and PCM round-trip. Unknown `type` values
/// decode to `DecodedPayload::Unknown` (caller logs + counts + drops per
@@ -440,6 +638,20 @@ pub fn decode_envelope(s: &str) -> Result<DecodedFrame, TapProtoError> {
let p: ErrorPayload = serde_json::from_value(extra_value)?;
DecodedPayload::Error(p)
}
FrameKind::SpeechStarted => DecodedPayload::SpeechStarted,
FrameKind::SpeechStopped => DecodedPayload::SpeechStopped,
FrameKind::FunctionCall => {
let p: FunctionCallPayload = serde_json::from_value(extra_value)?;
DecodedPayload::FunctionCall(p)
}
FrameKind::FunctionCallOutput => {
let p: FunctionCallOutputPayload = serde_json::from_value(extra_value)?;
DecodedPayload::FunctionCallOutput(p)
}
FrameKind::ToolsUpdate => {
let p: ToolsUpdatePayload = serde_json::from_value(extra_value)?;
DecodedPayload::ToolsUpdate(p)
}
FrameKind::Unknown => DecodedPayload::Unknown,
};
Ok(DecodedFrame {
@@ -596,4 +808,96 @@ mod tests {
}
));
}
/// Slice-3 additive event types must round-trip through (de)serialization
/// without breaking slice-2's v1 contract. Every new kind + payload.
#[test]
fn speech_started_round_trips() {
let s = encode_speech_started(7, 100).unwrap();
assert!(s.contains("\"type\":\"speech_started\""));
assert!(s.contains("\"v\":1"));
let d = decode_envelope(&s).unwrap();
assert_eq!(d.seq, 7);
assert_eq!(d.ts, 100);
assert!(matches!(d.payload, DecodedPayload::SpeechStarted));
}
#[test]
fn speech_stopped_round_trips() {
let s = encode_speech_stopped(9, 200).unwrap();
assert!(s.contains("\"type\":\"speech_stopped\""));
let d = decode_envelope(&s).unwrap();
assert_eq!(d.seq, 9);
assert!(matches!(d.payload, DecodedPayload::SpeechStopped));
}
#[test]
fn function_call_round_trips() {
let s = encode_function_call("abc-123", "hangup", "{}", 0, 0).unwrap();
assert!(s.contains("\"type\":\"function_call\""));
assert!(s.contains("\"id\":\"abc-123\""));
assert!(s.contains("\"name\":\"hangup\""));
let d = decode_envelope(&s).unwrap();
match d.payload {
DecodedPayload::FunctionCall(p) => {
assert_eq!(p.id, "abc-123");
assert_eq!(p.name, "hangup");
}
other => panic!("expected FunctionCall, got {other:?}"),
}
}
#[test]
fn function_call_output_round_trips() {
let s =
encode_function_call_output("abc-123", "ok", r#"{"channel_state":"Closing"}"#, 0, 0)
.unwrap();
assert!(s.contains("\"type\":\"function_call_output\""));
assert!(s.contains("\"status\":\"ok\""));
let d = decode_envelope(&s).unwrap();
match d.payload {
DecodedPayload::FunctionCallOutput(p) => {
assert_eq!(p.id, "abc-123");
assert_eq!(p.status, "ok");
}
other => panic!("expected FunctionCallOutput, got {other:?}"),
}
}
#[test]
fn tools_update_round_trips() {
let tools_json = r#"[{"name":"hangup","description":"hang up the call"}]"#;
let s = encode_tools_update(tools_json, 0, 0).unwrap();
assert!(s.contains("\"type\":\"tools.update\""));
assert!(s.contains("\"tools\":["));
let d = decode_envelope(&s).unwrap();
match d.payload {
DecodedPayload::ToolsUpdate(p) => {
assert!(p.tools.is_array());
assert_eq!(p.tools.as_array().unwrap().len(), 1);
}
other => panic!("expected ToolsUpdate, got {other:?}"),
}
}
/// Forwards-compat: slice-2's echo brain sees the new types as unknown
/// (the `#[serde(other)]` on the old enum absorbed them). With the new
/// enum in place, the kinds decode to their new variants (rather than
/// Unknown). This test asserts the decode no longer drops them.
#[test]
fn new_kinds_decode_to_their_variants_not_unknown() {
for s in [
encode_speech_started(0, 0).unwrap(),
encode_speech_stopped(0, 0).unwrap(),
encode_function_call("x", "x", "null", 0, 0).unwrap(),
encode_function_call_output("x", "ok", "null", 0, 0).unwrap(),
encode_tools_update("[]", 0, 0).unwrap(),
] {
let d = decode_envelope(&s).unwrap();
assert!(
!matches!(d.payload, DecodedPayload::Unknown),
"new event type decoded as Unknown: {s}"
);
}
}
}

View File

@@ -63,11 +63,46 @@ pub enum TapClientError {
/// `close` is a shared borrow (`&mut oneshot::Receiver<()>`) so the
/// `TapEngine` reconnect loop (Task 7) can share one close signal across
/// reconnect attempts of the same session (see module docs).
///
/// # slice-3 §5.2 — the tool-call side-channel
///
/// Two extra mpsc halves (relative to slice-2's audio-only pump) carry the
/// brain's `function_call` proposals out to the binary's poll task and the
/// binary's `function_call_output` replies back onto the wire:
///
/// - `tx_function_call` — the TapClient emits a `FunctionCallEvent` here
/// whenever it observes a tap `function_call` frame on its inbound WS
/// stream. The binary's poll task drains this in the same cycle it drains
/// the existing `flush_tx` side-channel (slice-2 §5.3 step 4 — one extra
/// channel, same cycle) and dispatches via `ToolRegistry::dispatch`.
/// - `rx_function_call_output` — the binary's poll task writes
/// `FunctionCallOutputEvent`s here (after a `ToolRegistry::dispatch` call
/// completes); the TapClient drains this in the same `tokio::select!`
/// pump as the audio + close arms and sends each as a `function_call_output`
/// tap WS frame to the brain.
///
/// Both halves are mpsc ends (not oneshot) because the brain may propose
/// multiple tool calls per session (one-of-many, not one-of-one) and the
/// binary may queue multiple replies before the TapClient's pump cycle
/// drains them.
//
// clippy::too_many_arguments: the slice-3 §5.2 design added two more mpsc
// halves to slice-2's already-5-arg pump signature for a total of 8. Each
// arg is a distinct channel end with a distinct lifetime owner (the WS
// stream, the session id, two sender/receiver pairs for audio, two for the
// tool-call side-channel, the metrics Arc, the close oneshot). Wrapping
// them in a struct would obscure that all-but-one are channel ends shared
// with the binary's poll task — the flat signature mirrors slice-2's
// precedent and keeps the call site readable. Suppress per AGENTS.md's
// "documented inline rationale" exception to the -D warnings bar.
#[allow(clippy::too_many_arguments)]
pub async fn run_tap_client<T>(
mut ws: WebSocketStream<T>,
session_id: ChannelId,
rx_pcm_in: &mut mpsc::Receiver<PcmFrame>,
tx_audio_out: mpsc::Sender<PcmFrame>,
tx_function_call: mpsc::Sender<FunctionCallEvent>,
rx_function_call_output: &mut mpsc::Receiver<FunctionCallOutputEvent>,
metrics: Arc<TapMetrics>,
close: &mut oneshot::Receiver<()>,
) -> Result<(), TapClientError>
@@ -175,6 +210,44 @@ where
}
}
}
// slice-3 §5.2: drain a `function_call_output` event the binary's
// poll task wrote (after `ToolRegistry::dispatch` returned) + send
// it as a `function_call_output` tap WS frame to the brain. The
// `seq_egress` bump mirrors the audio arm — every egress frame
// shares the same per-direction counter (spec §3.1).
//
// Like `rx_pcm_in.recv()`, this is one of many arms in the
// select! — the engine's `tx_function_call_output` sender lives
// in the binary; `run_tap_client` returns when the brain WS ends
// or close fires, regardless of pending function_call_output
// events (they're dropped on close — same posture as pending
// audio_out frames on teardown).
out = rx_function_call_output.recv() => {
if let Some(out) = out {
let ts = elapsed_ms(session_start);
let result_str = out.0.result.to_string();
match crate::protocol::encode_function_call_output(
&out.0.id,
&out.0.status,
&result_str,
seq_egress,
ts,
) {
Ok(s) => {
seq_egress += 1;
if let Err(e) = ws.send(Message::Text(s)).await {
warn!(error = ?e, %session_id, "ws send function_call_output failed");
return Err(e.into());
}
info!(%session_id, call_id = %out.0.id, status = %out.0.status, "sent function_call_output to brain");
}
Err(e) => {
metrics.malformed_frames.fetch_add(1, Ordering::Relaxed);
warn!(error = ?e, "encode function_call_output failed; dropping");
}
}
}
}
// Inbound WS frame from brain.
msg = ws.next() => {
let Some(msg) = msg else {
@@ -188,7 +261,7 @@ where
if let Ok(text) = msg.into_text() {
handle_brain_frame(
&text, &mut last_seq_ingress, &tx_audio_out,
&metrics, session_start,
&tx_function_call, &metrics, session_start,
).await;
}
}
@@ -251,6 +324,7 @@ async fn handle_brain_frame(
text: &str,
last_seq_ingress: &mut Option<u64>,
tx_audio_out: &mpsc::Sender<PcmFrame>,
tx_function_call: &mpsc::Sender<FunctionCallEvent>,
metrics: &Arc<TapMetrics>,
session_start: Instant,
) {
@@ -307,6 +381,41 @@ async fn handle_brain_frame(
metrics.unknown_frames.fetch_add(1, Ordering::Relaxed);
warn!("unexpected frame direction from brain; dropping");
}
// Slice-3 spec §5.2: `function_call` flows through the side-channel
// (NON-BLOCKING try_send — the binary's poll task drains on its own
// cycle). The same "drop + observe" posture as audio_out applies if
// the channel is full: a backed-up binary means we drop the proposal
// and the brain gets no reply (the brain process knows no
// function_call_output arrived → its OpenAI pump keeps going; the
// model tolerates missing replies per OpenAI's design).
DecodedPayload::FunctionCall(p) => {
if tx_function_call.try_send(FunctionCallEvent(p)).is_err() {
metrics.outbound_dropped.fetch_add(1, Ordering::Relaxed);
warn!(
"function_call dropped (binary poll task not draining; brain will see no reply)"
);
}
}
// Slice-3 (spec §3.2): `function_call_output` is core→brain; ignore
// if a brain sends one back (a misbehaving brain — same posture as
// `SessionEnd`/`AudioIn` from brain above).
DecodedPayload::FunctionCallOutput(_) => {
metrics.unknown_frames.fetch_add(1, Ordering::Relaxed);
warn!("unexpected function_call_output from brain; dropping");
}
// Slice-3 advisory — same "logged + counted, not forwarded" posture
// as `Unknown`. The FOB reflex loop in step 4 will act on these;
// slice-3 only pre-paves the wire event.
DecodedPayload::SpeechStarted | DecodedPayload::SpeechStopped => {
metrics.unknown_frames.fetch_add(1, Ordering::Relaxed);
debug!("advisory interruption event observed; not acted on in slice-3");
}
DecodedPayload::ToolsUpdate(_) => {
metrics.unknown_frames.fetch_add(1, Ordering::Relaxed);
debug!(
"tools.update observed; slice-3 dispatch keys off function_call by name, not catalog"
);
}
}
let _ = session_start; // used for ts computation if added later
}
@@ -315,6 +424,36 @@ fn elapsed_ms(start: Instant) -> u64 {
start.elapsed().as_millis() as u64
}
/// A `function_call` event the TapClient **observed** on its inbound WS
/// stream and forwarded to the binary's poll task via the
/// `tx_function_call` side-channel (spec §5.2). The binary's poll task
/// drains this (alongside the existing `flush_rx` side-channel — slice-2
/// §5.3 step 4) and dispatches each event through `ToolRegistry::dispatch`.
///
/// # Why a thin newtype over `FunctionCallPayload` (and not a bare alias)?
///
/// A type alias (`pub type FunctionCallEvent = FunctionCallPayload;`) would
/// let the binary pass a `FunctionCallPayload` where a `FunctionCallEvent`
/// is expected without surfacing the intent. The newtype draws a small but
/// real boundary: the wire-payload type (`FunctionCallPayload`) lives in
/// `protocol.rs` for (de)serialization; the side-channel event type
/// (`FunctionCallEvent`) lives here for dispatch. They share a shape but
/// carry different semantic weight — honoring the newtype-over-primitives
/// convention from AGENTS.md even at the message level.
#[derive(Debug, Clone)]
pub struct FunctionCallEvent(pub crate::protocol::FunctionCallPayload);
/// A `function_call_output` event the binary's poll task **emits** back to
/// the TapClient via the `tx_function_call_output` side-channel (spec §5.2
/// — the binary's poll task dispatches through `ToolRegistry::dispatch`,
/// serializes the `ToolResult`, and writes the output here). The TapClient
/// drains this in the same `tokio::select!` pump cycle as the audio + close
/// arms and sends each as a `function_call_output` tap WS frame.
///
/// Same newtype-over-payload rationale as `FunctionCallEvent`.
#[derive(Debug, Clone)]
pub struct FunctionCallOutputEvent(pub crate::protocol::FunctionCallOutputPayload);
#[cfg(test)]
mod tests {
// TapClient is heavily async; its real behavior is exercised in the
@@ -322,6 +461,7 @@ mod tests {
// tests here cover the pure helpers.
use super::*;
use crate::protocol::encode_function_call;
#[test]
fn elapsed_ms_is_monotonic_nonneg() {
@@ -330,4 +470,83 @@ mod tests {
// First call ~0; just assert it's a valid u64.
assert_eq!(ms, ms); // tautology but clippy-clean
}
/// slice-3 spec §5.2: when the TapClient observes a tap `function_call`
/// frame on its inbound WS stream it emits a `FunctionCallEvent` on
/// the `tx_function_call` side-channel. The binary's poll task drains
/// that and dispatches via `ToolRegistry::dispatch`. This test pins the
/// contract end-to-end through the pure helper (`handle_brain_frame`):
/// hand it a wire-encoded function_call frame + a fresh mpsc pair and
/// assert the receiver observes the forwarded event.
#[tokio::test]
async fn handle_brain_frame_forwards_function_call_to_side_channel() {
let (tx_fc, mut rx_fc) = mpsc::channel::<FunctionCallEvent>(8);
let (tx_audio_out, _rx_audio_out) = mpsc::channel::<PcmFrame>(8);
let metrics = Arc::new(TapMetrics::new());
// Build a wire function_call frame: id="call-1", name="hangup", args={}.
let wire = encode_function_call("call-1", "hangup", "{}", 1, 100).unwrap();
let mut last_seq: Option<u64> = None;
handle_brain_frame(
&wire,
&mut last_seq,
&tx_audio_out,
&tx_fc,
&metrics,
Instant::now(),
)
.await;
// The side-channel must have observed exactly one FunctionCallEvent
// carrying the wire's id/name/args.
let event = tokio::time::timeout(Duration::from_millis(200), rx_fc.recv())
.await
.expect("tx_function_call drained within 200ms")
.expect("channel not closed");
assert_eq!(event.0.id, "call-1");
assert_eq!(event.0.name, "hangup");
assert_eq!(event.0.args, serde_json::json!({}));
// seq tracking still updates for the side-channeled event.
assert_eq!(last_seq, Some(1));
}
/// slice-3 spec §5.2 — the *advisory* interrupt events (`speech_started`
/// /`speech_stopped`) and `tools.update` are observed (logged + counted)
/// but do NOT flow through the function_call side-channel (only
/// `function_call` does — that's the only event with a binary-side
/// disposal). This pins that boundary: an advisory event must NOT
/// produce a `FunctionCallEvent` even with the channel plumbed.
#[tokio::test]
async fn advisory_events_are_logged_not_forwarded_to_function_call_channel() {
let (tx_fc, mut rx_fc) = mpsc::channel::<FunctionCallEvent>(8);
let (tx_audio_out, _rx_audio_out) = mpsc::channel::<PcmFrame>(8);
let metrics = Arc::new(TapMetrics::new());
let wire = crate::protocol::encode_speech_started(2, 200).unwrap();
let mut last_seq: Option<u64> = None;
handle_brain_frame(
&wire,
&mut last_seq,
&tx_audio_out,
&tx_fc,
&metrics,
Instant::now(),
)
.await;
// No FunctionCallEvent forwarded — the channel stays empty. Pick a
// tight bounded receive so the test fails fast if a future refactor
// starts forwarding advisory events here.
assert!(
tokio::time::timeout(Duration::from_millis(50), rx_fc.recv())
.await
.is_err(),
"no FunctionCallEvent expected for advisory events"
);
// The advisory event IS still observed via metrics (seq gap tracking
// + the unknown-slot counter remains 0 — speech_started is now a
// known payload variant).
assert_eq!(last_seq, Some(2));
}
}

View File

@@ -0,0 +1 @@
{"v":1,"type":"function_call","seq":0,"ts":0,"id":"abc-123","name":"hangup","args":{}}

View File

@@ -0,0 +1 @@
{"v":1,"type":"function_call_output","seq":0,"ts":0,"id":"abc-123","status":"ok","result":{"channel_state":"Closing"}}

View File

@@ -0,0 +1 @@
{"v":1,"type":"speech_started","seq":7,"ts":100}

View File

@@ -0,0 +1 @@
{"v":1,"type":"speech_stopped","seq":9,"ts":200}

View File

@@ -0,0 +1 @@
{"v":1,"type":"tools.update","seq":0,"ts":0,"tools":[{"description":"hang up the call","name":"hangup"}]}

View File

@@ -0,0 +1,225 @@
//! Slice-3 golden-fixture tests for the tap protocol's new event types.
//!
//! ## Why this file exists (alongside `protocol.rs`'s inline tests)
//!
//! `protocol.rs` already has inline round-trip tests for each new slice-3
//! event kind, but they assert with `String::contains` — a *substring* check.
//! That catches "the right `type` tag is present" but it cannot catch:
//!
//! - a change in **field order** (the `Envelope` `Serialize` impl at
//! `protocol.rs::Serialize_for_Envelope` pins field order deliberately —
//! see its doc comment: "field order is pinned to make the wire bytes
//! deterministic… helps snapshot tests and interop with brains that read
//! key order for debugging");
//! - a stray extra field;
//! - a missing field;
//! - a change in whitespace or number formatting.
//!
//! These fixtures lock the **exact wire bytes** produced by each `encode_*`
//! function. Any future refactor that alters the on-wire shape — even
//! cosmetically — breaks a test here, forcing a conscious fixture update +
//! a protocol-version bump conversation (spec §3.1: `PROTOCOL_VERSION` is
//! the contract door).
//!
//! ## Forwards-compat (the slice-2 §3.4 "drop + observe" contract)
//!
//! The last test here re-asserts the load-bearing decode-side rule: an
//! unknown `type` string deserializes to `DecodedPayload::Unknown` (via
//! `FrameKind`'s `#[serde(other)]` fallback) rather than erroring. This is
//! the foundation of the forwards-compat posture — a brain speaking a
//! future protocol version's event kind must not crash an older core; it
//! gets logged + counted + dropped (the behavioral side lives in
//! `tap_client::handle_brain_frame` and is exercised end-to-end by the
//! slice-2 / slice-3 integration tests, not here).
//!
//! ## Why string equality (not `serde_json::Value` equality)
//!
//! Comparing parsed `serde_json::Value`s would be *semantic* equality — it
//! would treat `{"a":1,"b":2}` and `{"b":2,"a":1}` as the same. That
//! throws away the field-order pinning the `Envelope` serializer
//! guarantees. Comparing the raw encoded `String` to the fixture's bytes
//! keeps the order contract load-bearing.
use std::fs;
use rutster_tap::{
DecodedPayload, decode_envelope, encode_function_call, encode_function_call_output,
encode_speech_started, encode_speech_stopped, encode_tools_update,
};
/// Read a fixture file from `tests/fixtures/<name>`. Uses
/// `CARGO_MANIFEST_DIR` so the path is stable regardless of which
/// directory `cargo test` is invoked from. Trailing newlines are stripped
/// because `serde_json::to_string` (what the encoders call) never emits one
/// — a fixture checked into git shouldn't acquire a phantom newline mismatch.
fn fixture(name: &str) -> String {
let path = format!("{}/tests/fixtures/{}", env!("CARGO_MANIFEST_DIR"), name);
let raw =
fs::read_to_string(&path).unwrap_or_else(|e| panic!("read fixture {name} ({path}): {e}"));
raw.trim_end().to_string()
}
// ─── speech_started ──────────────────────────────────────────────────────
#[test]
fn golden_speech_started_wire_bytes_match_fixture() {
let encoded = encode_speech_started(7, 100).unwrap();
let golden = fixture("speech_started.json");
assert_eq!(
encoded, golden,
"speech_started wire bytes drifted from golden fixture"
);
}
#[test]
fn speech_started_fixture_decodes_to_speechstarted_variant() {
let golden = fixture("speech_started.json");
let decoded = decode_envelope(&golden).unwrap();
assert_eq!(decoded.seq, 7);
assert_eq!(decoded.ts, 100);
assert!(
matches!(decoded.payload, DecodedPayload::SpeechStarted),
"speech_started fixture did not decode to SpeechStarted: {:?}",
decoded.payload
);
}
// ─── speech_stopped ──────────────────────────────────────────────────────
#[test]
fn golden_speech_stopped_wire_bytes_match_fixture() {
let encoded = encode_speech_stopped(9, 200).unwrap();
let golden = fixture("speech_stopped.json");
assert_eq!(
encoded, golden,
"speech_stopped wire bytes drifted from golden fixture"
);
}
#[test]
fn speech_stopped_fixture_decodes_to_speechstopped_variant() {
let golden = fixture("speech_stopped.json");
let decoded = decode_envelope(&golden).unwrap();
assert_eq!(decoded.seq, 9);
assert_eq!(decoded.ts, 200);
assert!(
matches!(decoded.payload, DecodedPayload::SpeechStopped),
"speech_stopped fixture did not decode to SpeechStopped: {:?}",
decoded.payload
);
}
// ─── function_call ───────────────────────────────────────────────────────
#[test]
fn golden_function_call_wire_bytes_match_fixture() {
// `args_json_str="{}"` — the encoder parses it into `serde_json::Value`
// (an empty object) so the wire field is `"args":{}`, not `"args":"{}"`.
// This is load-bearing: a brain emitting a function_call with a raw
// JSON-string args would be a different wire shape (and a translator
// bug). The fixture pins the parsed-object shape.
let encoded = encode_function_call("abc-123", "hangup", "{}", 0, 0).unwrap();
let golden = fixture("function_call.json");
assert_eq!(
encoded, golden,
"function_call wire bytes drifted from golden fixture"
);
}
#[test]
fn function_call_fixture_decodes_to_functioncall_with_fields() {
let golden = fixture("function_call.json");
let decoded = decode_envelope(&golden).unwrap();
match decoded.payload {
DecodedPayload::FunctionCall(p) => {
assert_eq!(p.id, "abc-123");
assert_eq!(p.name, "hangup");
assert_eq!(p.args, serde_json::Value::Object(serde_json::Map::new()));
}
other => panic!("expected FunctionCall, got {other:?}"),
}
}
// ─── function_call_output ───────────────────────────────────────────────
#[test]
fn golden_function_call_output_wire_bytes_match_fixture() {
let encoded =
encode_function_call_output("abc-123", "ok", r#"{"channel_state":"Closing"}"#, 0, 0)
.unwrap();
let golden = fixture("function_call_output.json");
assert_eq!(
encoded, golden,
"function_call_output wire bytes drifted from golden fixture"
);
}
#[test]
fn function_call_output_fixture_decodes_to_functioncalloutput_with_fields() {
let golden = fixture("function_call_output.json");
let decoded = decode_envelope(&golden).unwrap();
match decoded.payload {
DecodedPayload::FunctionCallOutput(p) => {
assert_eq!(p.id, "abc-123");
assert_eq!(p.status, "ok");
// The result field is a parsed JSON object, not a string.
assert_eq!(
p.result.get("channel_state"),
Some(&serde_json::json!("Closing"))
);
}
other => panic!("expected FunctionCallOutput, got {other:?}"),
}
}
// ─── tools.update ───────────────────────────────────────────────────────
#[test]
fn golden_tools_update_wire_bytes_match_fixture() {
let tools_json = r#"[{"name":"hangup","description":"hang up the call"}]"#;
let encoded = encode_tools_update(tools_json, 0, 0).unwrap();
let golden = fixture("tools_update.json");
assert_eq!(
encoded, golden,
"tools.update wire bytes drifted from golden fixture"
);
}
#[test]
fn tools_update_fixture_decodes_to_toolsupdate_with_array() {
let golden = fixture("tools_update.json");
let decoded = decode_envelope(&golden).unwrap();
match decoded.payload {
DecodedPayload::ToolsUpdate(p) => {
let arr = p
.tools
.as_array()
.expect("tools field must be a JSON array");
assert_eq!(arr.len(), 1);
assert_eq!(arr[0].get("name"), Some(&serde_json::json!("hangup")));
}
other => panic!("expected ToolsUpdate, got {other:?}"),
}
}
// ─── forwards-compat (slice-2 §3.4 "drop + observe") ────────────────────
/// The load-bearing forwards-compat rule on the decode side: an unknown
/// `type` string deserializes to `DecodedPayload::Unknown` (via
/// `FrameKind`'s `#[serde(other)]`), **not** an error. A future-protocol
/// brain emitting `"type":"future_event"` must not crash an older core;
/// the core logs + counts + drops it (the observe-and-continue posture
/// the eventual fuzz harness will stress). This re-asserts the contract
/// the inline `unknown_type_decodes_to_unknown_variant` test established,
/// here in the external fixture file so it sits next to the golden wire
/// shapes it protects.
#[test]
fn unknown_wire_type_decodes_to_unknown_variant_not_error() {
let wire = r#"{"v":1,"type":"future_event","seq":0,"ts":0}"#;
let decoded = decode_envelope(wire).expect("unknown type must not error (forwards-compat)");
assert!(
matches!(decoded.payload, DecodedPayload::Unknown),
"forwards-compat broken: unknown type decoded to {:?}, expected Unknown",
decoded.payload
);
}

View File

@@ -18,6 +18,7 @@ futures-util = { workspace = true }
url = { workspace = true }
dashmap = { workspace = true }
uuid = { workspace = true }
async-trait = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
@@ -27,6 +28,11 @@ serde_json = { workspace = true }
[dev-dependencies]
tower = { workspace = true }
rutster-tap-echo = { path = "../rutster-tap-echo" }
# slice-3 §7.4: integration test spins up the in-process MockRealtimeBrain
# + drives the brain's `run_openai_pump` directly (replicating the brain
# process's accept loop in the test, no subprocess). The `mock` feature
# brings in MockRealtimeBrain without requiring OPENAI_API_KEY.
rutster-brain-realtime = { path = "../rutster-brain-realtime", features = ["mock"] }
[[bin]]
name = "rutster"

View File

@@ -25,3 +25,4 @@
pub mod routes;
pub mod session_map;
pub mod tap_engine;
pub mod tool_registry;

View File

@@ -39,6 +39,11 @@ use tokio::sync::Mutex;
use tracing::{debug, info, warn};
use crate::tap_engine::{TapConn, spawn_tap_engine};
// Re-using the binary crate's `tokio::sync::mpsc` import (the engine task
// + the poll task both live in `rutster`'s poll-driver module). The type
// only appears in `TapConn` field signatures + the slice-3 §5.2 dispatch
// helper below; bringing it in here keeps the type names short.
use tokio::sync::mpsc;
/// The per-session wrapper struct (slice-2, spec §6).
///
@@ -265,6 +270,13 @@ async fn drive_all_sessions(state: &AppState, now: Instant) {
}
}
}
// slice-3 §5.2: drain the per-session `rx_function_call`
// side-channel in the same cycle as the `flush_rx` drain (slice-2
// §5.3 step 4 pattern — one extra channel, same cycle). The
// helper spawns each dispatch as its own task so the 750 ms
// `AppState::close` await (hangup's teardown handshake) can't
// stall the poll cadence.
let _fc_drained = drain_function_calls(state, id).await;
// Hold the DashMap Ref only long enough to clone the Arc-wrapped
// rtc + tap_url; the async poll + spawn happens outside the shard.
let (rtc, tap_url) = match state.sessions.get(&id) {
@@ -298,7 +310,12 @@ async fn drive_all_sessions(state: &AppState, now: Instant) {
if let ChannelState::Connected = s.channel.state {
if s.channel.tap.is_none() {
// First connect: spawn the TapEngine, wire the TapAudioPipe.
let (pipe, conn) = spawn_tap_engine(id, tap_url);
// slice-3 §6.2: spawn_tap_engine constructs a per-channel
// ToolRegistry holding HangupTool (the only wired tool in
// slice-3 — §6.3) bound to this AppState + ChannelId.
let app_state = state.clone();
let tap_url_clone = tap_url.clone();
let (pipe, conn) = spawn_tap_engine(id, tap_url_clone, app_state);
s.set_pipe(pipe);
s.channel.tap = Some(TapHandle::new());
info!(channel_id = %id, "tap engine spawned on Connected");
@@ -319,3 +336,176 @@ async fn drive_all_sessions(state: &AppState, now: Instant) {
}
}
}
/// slice-3 §5.2 + §6 — drain the per-session `rx_function_call` side-channel
/// and dispatch each event through the per-channel `ToolRegistry`. One
/// dispatch result → one `function_call_output` written to
/// `tx_function_call_output` (which the TapClient forwards to the brain on
/// its next pump cycle).
///
/// # Why this is a separate helper (not inline in `drive_all_sessions`)
///
/// `ToolRegistry::dispatch` is `async` (the `hangup` tool calls
/// `AppState::close`, which awaits the engine's teardown handshake — spec
/// §5.2's 750 ms bound). Awaiting that inline in the poll task would stall
/// the 10 ms poll cadence for every hangup. Instead we **collect** events
/// non-blockingly here (`try_recv` — drop + observe on full channel, same
/// posture as the existing flush drain), then **spawn** each dispatch as
/// its own tokio task so the poll task returns to its 10 ms cadence
/// immediately. The spawned task holds its own clones of `AppState` +
/// `tool_registry` + `tx_function_call_output` — no shared mutable state
/// with the poll path.
///
/// Returns the count of events drained (for observability — matches the
/// flush drain's "drain all" loop shape). The poll task calls this in the
/// same `drive_all_sessions` cycle it drains the `flush_rx` side-channel
/// (slice-2 §5.3 step 4 pattern, one extra channel, same cycle).
async fn drain_function_calls(state: &AppState, id: ChannelId) -> usize {
// Collect: pull everything we need out of the DashMap shard BEFORE any
// await (the spawned dispatch's `AppState::close` later takes the same
// shard lock — holding a `get_mut` Ref across the spawn's await would
// deadlock slice-2's other shard-mutating handlers). The Collector scope
// owns a single short-lived `get_mut` Ref; the loop does only sync
// `try_recv` + Vec push (no await).
let collected: Vec<rutster_tap::FunctionCallEvent>;
let tool_registry: Arc<tokio::sync::Mutex<crate::tool_registry::ToolRegistry>>;
let tx_out: Option<mpsc::Sender<rutster_tap::FunctionCallOutputEvent>>;
{
let Some(mut entry) = state.sessions.get_mut(&id) else {
return 0;
};
let Some(conn) = entry.tap_conn.as_mut() else {
return 0;
};
tool_registry = conn.tool_registry.clone();
tx_out = conn.tx_function_call_output.clone();
let Some(rx_fc) = conn.rx_function_call.as_mut() else {
return 0;
};
collected = (0..).map_while(|_| rx_fc.try_recv().ok()).collect();
}
let drained = collected.len();
// Spawn dispatches outside the shard Ref scope. `AppState::close` is a
// 750 ms bounded wait in the worst case (slice-2 §5.2's teardown
// handshake); spawning keeps the 10 ms poll cadence responsive.
for event in collected {
let app_state = state.clone();
let reg = tool_registry.clone();
let tx_out = tx_out.clone();
tokio::spawn(async move {
let payload = event.0;
let call_id = payload.id.clone();
let name = payload.name.clone();
debug!(channel_id = %id, call_id = %call_id, tool = %name, "dispatching function_call");
let result = reg.lock().await.dispatch(&name, payload.args).await;
let (status, result_value) = result.to_status_result();
let out =
rutster_tap::FunctionCallOutputEvent(rutster_tap::FunctionCallOutputPayload {
id: call_id.clone(),
status: status.clone(),
result: result_value,
});
// try_send — non-blocking. The TapClient's pump loop drains
// `rx_function_call_output` on its next `select!` cycle. If the
// channel is full (brain not pumping), we drop + observe (same
// posture as the engine's `tx_function_call.try_send` on the
// inbound side). `tx_out` is `Option<Sender>`; `None` means the
// engine has torn down its receiver — the result is dropped,
// acceptable since the call is hanging up.
if let Some(tx) = tx_out.as_ref() {
if let Err(e) = tx.try_send(out) {
warn!(
channel_id = %id, call_id = %call_id, error = ?e,
"function_call_output dropped (TapClient pump not draining)"
);
}
}
// app_state is held for the dispatch lifetime — the HangupTool
// inside the registry already captured its own clone at registry
// construction, so this outer binding exists for future tool
// impls that need the live AppState handed to dispatch (none in
// slice-3 beyond hangup). Drop the silent-binding warning by
// referencing it once.
drop(app_state);
info!(channel_id = %id, call_id = %call_id, status = %status, "function_call dispatched");
});
}
drained
}
#[cfg(test)]
mod tests {
use super::*;
use rutster_tap::{
FunctionCallEvent, FunctionCallOutputEvent, FunctionCallPayload, TapMetrics,
};
use tokio::sync::Mutex;
/// slice-3 §5.2 + §6 — a `function_call` event drained from
/// `rx_function_call` triggers `ToolRegistry::dispatch("hangup")` which
/// fires `AppState::close` (the slice-2 teardown path), and the dispatch
/// result flows back as a `FunctionCallOutputEvent` on
/// `tx_function_call_output`. End-to-end contract test for the helper
/// minus the live TapClient pump (which is integration-test territory).
#[tokio::test]
async fn drain_function_calls_dispatches_hangup_and_writes_output() {
let state = AppState::default();
// Create a session so AppState::close has something to remove.
let id = state.create_session(None).unwrap();
// Build a TapConn with manually-controlled side-channel ends so we
// can push a FunctionCallEvent from the test side + observe the
// dispatch's output on the paired Receiver.
let (tx_fc, rx_fc) = tokio::sync::mpsc::channel::<FunctionCallEvent>(8);
let (tx_fco, mut rx_fco) = tokio::sync::mpsc::channel::<FunctionCallOutputEvent>(8);
let mut registry = crate::tool_registry::ToolRegistry::new();
registry.register(Box::new(crate::tool_registry::HangupTool::new(
state.clone(),
id,
)));
let (close_tx, _close_rx) = tokio::sync::oneshot::channel::<()>();
let conn = crate::tap_engine::TapConn {
close_tx,
join: tokio::spawn(async {}), // no-op handle; aborted below
metrics: TapMetrics::new(),
flush_rx: None,
rx_function_call: Some(rx_fc),
tx_function_call_output: Some(tx_fco),
tool_registry: Arc::new(Mutex::new(registry)),
};
state.sessions.get_mut(&id).unwrap().tap_conn = Some(conn);
// Push a function_call for the hangup tool — simulates what the
// TapClient does when it observes a `function_call` tap frame.
tx_fc
.send(FunctionCallEvent(FunctionCallPayload {
id: "call-1".to_string(),
name: "hangup".to_string(),
args: serde_json::json!({}),
}))
.await
.unwrap();
// Drain — this spawns a dispatch task + returns immediately.
let drained = drain_function_calls(&state, id).await;
assert_eq!(drained, 1, "exactly one function_call should drain");
// The spawned dispatch task fires AppState::close (session removed)
// + writes the function_call_output. Bounded-wait the result with
// a generous timeout (AppState::close has a 750 ms teardown bound +
// the spawned task may take a few ms to schedule).
let out = tokio::time::timeout(Duration::from_secs(2), rx_fco.recv())
.await
.expect("function_call_output drained within 2s")
.expect("channel not closed");
assert_eq!(out.0.id, "call-1");
assert_eq!(out.0.status, "ok");
assert_eq!(out.0.result["channel_state"], "Closing");
// AppState::close removed the session entry (the teardown it fires).
assert!(
state.sessions.get(&id).is_none(),
"session should be removed after hangup dispatch"
);
}
}

View File

@@ -30,14 +30,18 @@ use std::time::Duration;
use futures_util::FutureExt;
use rutster_call_model::ChannelId;
use rutster_tap::tap_client::{TapClientError, run_tap_client};
use rutster_tap::tap_client::{
FunctionCallEvent, FunctionCallOutputEvent, TapClientError, run_tap_client,
};
use rutster_tap::{TapAudioPipe, TapMetrics};
use tokio::sync::{mpsc, oneshot};
use tokio::sync::{Mutex, mpsc, oneshot};
use tokio::task::JoinHandle;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tracing::{info, warn};
use url::Url;
use crate::tool_registry::{HangupTool, ToolRegistry};
/// Capacity for the two mpsc channels between TapAudioPipe and TapClient.
/// Large enough that a slow brain tick doesn't drop on every cycle;
/// small enough that a runaway brain doesn't accumulate seconds of audio.
@@ -83,6 +87,28 @@ pub struct TapConn {
/// returned from `spawn_tap_engine`; the engine task owns the paired
/// `flush_tx` and signals a flush after each failed pump cycle.
pub flush_rx: Option<mpsc::Receiver<()>>,
/// slice-3 §5.2 tool-call side-channel (brain → core): the binary's
/// poll task drains `function_call` events from here + dispatches via
/// [`tool_registry`]. Engine owns the paired `tx_function_call` Sender
/// moved into `run_tap_client`. `Option` to keep the type constructible
/// in tests that don't plumb it; always `Some` on conns returned from
/// `spawn_tap_engine` (matches `flush_rx`'s posture).
pub rx_function_call: Option<mpsc::Receiver<FunctionCallEvent>>,
/// slice-3 §5.2 tool-call side-channel (core → brain): the binary's
/// poll task writes `function_call_output` events here (one per
/// `ToolRegistry::dispatch` result). The engine owns the paired
/// `rx_function_call_output` Receiver moved into `run_tap_client` so
/// the engine's pump loop forwards each event as a tap WS frame.
pub tx_function_call_output: Option<mpsc::Sender<FunctionCallOutputEvent>>,
/// slice-3 §6 — per-channel tool registry. One registry per active
/// session (spec §6.2 "keyed by ChannelId"); `HangupTool` is the only
/// tool wired in slice-3 (§6.3). `Arc<Mutex<_>>` because the binary's
/// poll task mutates the registry (dispatch) while the TapConn is
/// shared across the poll task + the routes layer (potential future
/// `tools.update` forwarding). `Mutex` (not `RwLock`) because dispatch
/// is effectively exclusive — there's no read-heavy workload to
/// optimize.
pub tool_registry: Arc<Mutex<ToolRegistry>>,
}
/// Spawn the TapEngine task for one session. Dials `tap_url`, runs the pump
@@ -102,7 +128,11 @@ pub struct TapConn {
/// single struct would force the registry to also own the pipe, splitting
/// ownership of `RtcSession`'s internals across two modules — exactly the
/// pattern the slice-2 plan's structural review warned against.
pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, TapConn) {
pub fn spawn_tap_engine(
session_id: ChannelId,
tap_url: Url,
app_state: crate::session_map::AppState,
) -> (TapAudioPipe, TapConn) {
// Two mpsc channels. The naming convention is "from the engine's POV":
// - `tx_pcm_in`/`rx_pcm_in`: peer PCM flowing INTO the engine (sink side
// of TapAudioPipe calls `tx_pcm_in.try_send(frame)`; engine task owns
@@ -122,8 +152,27 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
// flush signals collapse to one `clear_playout_ring` call from the
// binary side).
let (flush_tx, flush_rx) = mpsc::channel::<()>(8);
// slice-3 §5.2 tool-call side-channels. Same capacity-shape rationale
// as `flush_tx`: `try_send`-safe on both ends, idempotent drain. One
// extra pair of mpsc halves vs slice-2 — same cycle in the binary's
// poll task (drain alongside `flush_rx`), same pump-arm in run_tap_client.
let (tx_function_call, rx_function_call) =
mpsc::channel::<FunctionCallEvent>(TAP_MPSC_CAPACITY);
let (tx_function_call_output, rx_function_call_output) =
mpsc::channel::<FunctionCallOutputEvent>(TAP_MPSC_CAPACITY);
let metrics = TapMetrics::new();
// slice-3 §6.2: per-channel tool registry. The engine constructs it
// (one per active session — spec "keyed by ChannelId, one registry per
// active channel"). `HangupTool` is the only wired tool in slice-3
// (§6.3); other tool names reply `not_implemented` via dispatch.
// The registry is `Arc<Mutex<ToolRegistry>>` so the binary's poll task
// can `dispatch` on it while the TapConn is shared (the future
// `tools.update`-forwarding path will also share this handle).
let mut registry = ToolRegistry::new();
registry.register(Box::new(HangupTool::new(app_state, session_id)));
let tool_registry = Arc::new(Mutex::new(registry));
// Clone metrics three ways: the engine task, the TapAudioPipe, and
// the TapConn handle each hold their own `Arc<TapMetrics>` refcount.
// All three views must observe the same counters — `Arc` is the
@@ -132,6 +181,7 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
// shared mutation is sound without a `Mutex`).
let metrics_for_pipe = metrics.clone();
let metrics_for_conn = metrics.clone();
let tool_registry_for_task = tool_registry.clone();
let join = tokio::spawn(async move {
run_engine_loop(
@@ -141,6 +191,8 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
tx_audio_out,
close_rx,
flush_tx,
tx_function_call,
rx_function_call_output,
metrics,
)
.await;
@@ -155,6 +207,9 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
join,
metrics: metrics_for_conn,
flush_rx: Some(flush_rx),
rx_function_call: Some(rx_function_call),
tx_function_call_output: Some(tx_function_call_output),
tool_registry: tool_registry_for_task,
};
(pipe, conn)
@@ -174,6 +229,14 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
/// pattern the plan's Note A spelled out: a single `tokio::select!` over
/// (a) the close receiver and (b) `connect_brain`, then a second select
/// over the close receiver and the backoff sleep.
//
// clippy::too_many_arguments: slice-3 §5.2 added two more mpsc halves to
// slice-2's engine-loop signature (the tool-call side-channel ends). Same
// rationale as `run_tap_client` above: each arg is a distinct channel
// end with a distinct lifetime owner. Wrapping them in a struct would
// obscure the channel-pair structure. Suppress per AGENTS.md's documented
// inline-rationale exception.
#[allow(clippy::too_many_arguments)]
async fn run_engine_loop(
session_id: ChannelId,
tap_url: Url,
@@ -181,6 +244,8 @@ async fn run_engine_loop(
tx_audio_out: mpsc::Sender<rutster_media::PcmFrame>,
mut close: oneshot::Receiver<()>,
flush_tx: mpsc::Sender<()>,
tx_function_call: mpsc::Sender<FunctionCallEvent>,
mut rx_function_call_output: mpsc::Receiver<FunctionCallOutputEvent>,
metrics: Arc<TapMetrics>,
) {
let mut backoff = Backoff::default();
@@ -212,11 +277,19 @@ async fn run_engine_loop(
// === Step 2: run the pump loop until close/error. ===
// `close` is a shared `&mut oneshot::Receiver` — same
// signal across reconnect attempts (Task 4's design).
// The slice-3 §5.2 tool-call side-channels are also shared
// across reconnect attempts — a brain reconnect mid-call
// must keep the binary's tool-registry dispatch flowing
// (a function_call the brain proposed right before it
// dropped should still get a function_call_output reply
// on the next connect).
let pump_result = run_tap_client(
ws,
session_id,
&mut rx_pcm_in,
tx_audio_out.clone(),
tx_function_call.clone(),
&mut rx_function_call_output,
metrics.clone(),
&mut close,
)
@@ -414,11 +487,45 @@ mod tests {
// increment. We abort the task on test drop to avoid leak.
let id = ChannelId::new();
let url = Url::parse("ws://127.0.0.1:1/echo").unwrap(); // port 1 = unreachable
let (mut pipe, conn) = spawn_tap_engine(id, url);
let (mut pipe, conn) = spawn_tap_engine(id, url, crate::session_map::AppState::default());
// TapAudioPipe is the seam object — should default to silent underflow.
assert!(pipe.next_pcm_frame().is_none());
// TapConn carries the close oneshot + JoinHandle + metrics.
let _ = conn.close_tx.send(());
conn.join.abort();
}
/// slice-3 §5.2: the TapConn returned by `spawn_tap_engine` must carry
/// the tool-call side-channel ends so the binary's poll task can drain
/// `function_call` events (→ `ToolRegistry::dispatch`) and write
/// `function_call_output` replies. This is a structural smoke test —
/// it pins that the plumbing lands (the binary's poll-task drain needs
/// all three: a ready `rx_function_call`, a ready `tx_function_call_output`,
/// and a `tool_registry` scoped to this session).
#[tokio::test]
async fn spawn_returns_tap_conn_with_function_call_side_channels() {
let id = ChannelId::new();
let url = Url::parse("ws://127.0.0.1:1/echo").unwrap(); // unreachable brain
let (_pipe, conn) = spawn_tap_engine(id, url, crate::session_map::AppState::default());
// rx_function_call: Some(Receiver) — engine owns the paired Sender.
assert!(
conn.rx_function_call.is_some(),
"TapConn must carry a drainable rx_function_call"
);
// tx_function_call_output: Some(Sender) — engine owns the paired
// Receiver on the pump-side (run_tap_client).
assert!(
conn.tx_function_call_output.is_some(),
"TapConn must carry a writeable tx_function_call_output"
);
// tool_registry: present (slice-3 §6.2 — one registry per active
// channel; HangupTool is the only wired tool).
let reg = conn.tool_registry.lock().await;
assert_eq!(reg.catalog().len(), 1, "only hangup is wired in slice-3");
assert_eq!(reg.catalog()[0]["name"], "hangup");
let _ = conn.close_tx.send(());
conn.join.abort();
}
}

View File

@@ -0,0 +1,208 @@
//! # Tool registry — the FOB boundary for brain-proposed tool calls
//!
//! Per spec §6 + ADR-0007 ("rutster mediates both the provider call-control
//! API and the brain tap, so the brain never holds the wire"). The brain
//! proposes (via function_call events); the FOB disposes (via
//! function_call_output).
//!
//! `hangup` is the only wired tool in slice-3 (spec §6.3); other tool names
//! reply `not_implemented` so the model is free to retry or give up.
use async_trait::async_trait;
use rutster_call_model::ChannelId;
use serde_json::{Value, json};
use tracing::warn;
use crate::session_map::AppState;
/// A registry-dispatchable tool. The async-trait pattern is needed because
/// the registry holds `Vec<Box<dyn Tool>>` (stable Rust doesn't support
/// async fns in trait *objects* without `async-trait` as of Rust 1.85).
#[async_trait]
pub trait Tool: Send + Sync {
fn name(&self) -> &str;
/// JSON-schema descriptor the registry sends to the brain on tools.update.
fn schema(&self) -> Value;
/// Execute the tool. The args Value is the raw JSON from the function_call
/// event (the brain's translator extracts `arguments` from OpenAI's
/// event and the registry hands it here verbatim).
async fn call(&self, args: Value) -> ToolResult;
}
#[derive(Debug, Clone)]
pub enum ToolResult {
Ok(Value),
Error(String),
NotImplemented,
}
impl ToolResult {
/// Serialize to the (status, result) pair the binary's poll task will
/// pass to `encode_function_call_output`.
pub fn to_status_result(&self) -> (String, Value) {
match self {
ToolResult::Ok(v) => ("ok".to_string(), v.clone()),
ToolResult::Error(msg) => ("error".to_string(), json!({ "error": msg })),
ToolResult::NotImplemented => ("not_implemented".to_string(), Value::Null),
}
}
}
pub struct ToolRegistry {
tools: Vec<Box<dyn Tool>>,
}
impl ToolRegistry {
pub fn new() -> Self {
Self { tools: Vec::new() }
}
pub fn register(&mut self, tool: Box<dyn Tool>) {
self.tools.push(tool);
}
/// Dispatch by tool name. Returns `ToolResult::NotImplemented` if no
/// tool with the given name is registered.
pub async fn dispatch(&self, name: &str, args: Value) -> ToolResult {
for tool in &self.tools {
if tool.name() == name {
return tool.call(args).await;
}
}
warn!(tool = %name, "brain proposed unknown tool; returning not_implemented");
ToolResult::NotImplemented
}
/// Serialize the catalog (used on startup + tools.update emission).
pub fn catalog(&self) -> Vec<Value> {
self.tools.iter().map(|t| t.schema()).collect()
}
}
impl Default for ToolRegistry {
fn default() -> Self {
Self::new()
}
}
/// The `hangup` tool. Holds `AppState` (cloned — it's `Arc`-cheap) + the
/// `ChannelId` of the call this tool operates on. `call()` ->
/// `AppState::close(channel_id).await` -> returns `Ok({"channel_state": "Closing"})`.
pub struct HangupTool {
app_state: AppState,
channel_id: ChannelId,
}
impl HangupTool {
pub fn new(app_state: AppState, channel_id: ChannelId) -> Self {
Self {
app_state,
channel_id,
}
}
}
#[async_trait]
impl Tool for HangupTool {
fn name(&self) -> &str {
"hangup"
}
fn schema(&self) -> Value {
json!({
"name": "hangup",
"description": "Hang up the current call.",
"parameters": {
"type": "object",
"properties": {}
}
})
}
async fn call(&self, _args: Value) -> ToolResult {
self.app_state.close(self.channel_id).await;
ToolResult::Ok(json!({ "channel_state": "Closing" }))
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A no-op tool whose `call` returns a fixed value — for testing the
/// registry's dispatch + catalog shape without needing AppState.
struct EchoTool {
tool_name: String,
}
#[async_trait]
impl Tool for EchoTool {
fn name(&self) -> &str {
&self.tool_name
}
fn schema(&self) -> Value {
json!({ "name": self.tool_name, "description": "test tool" })
}
async fn call(&self, args: Value) -> ToolResult {
ToolResult::Ok(args)
}
}
#[tokio::test]
async fn dispatch_known_tool_returns_ok() {
let mut reg = ToolRegistry::new();
reg.register(Box::new(EchoTool {
tool_name: "echo".to_string(),
}));
let r = reg.dispatch("echo", json!({"x": 1})).await;
let (status, result) = r.to_status_result();
assert_eq!(status, "ok");
assert_eq!(result, json!({"x": 1}));
}
#[tokio::test]
async fn dispatch_unknown_tool_returns_not_implemented() {
let mut reg = ToolRegistry::new();
reg.register(Box::new(EchoTool {
tool_name: "echo".to_string(),
}));
let r = reg.dispatch("not_registered", json!({})).await;
let (status, _) = r.to_status_result();
assert_eq!(status, "not_implemented");
}
#[tokio::test]
async fn catalog_lists_all_registered_tools() {
let mut reg = ToolRegistry::new();
reg.register(Box::new(EchoTool {
tool_name: "a".to_string(),
}));
reg.register(Box::new(EchoTool {
tool_name: "b".to_string(),
}));
let cat = reg.catalog();
assert_eq!(cat.len(), 2);
assert_eq!(cat[0]["name"], "a");
assert_eq!(cat[1]["name"], "b");
}
#[tokio::test]
async fn hangup_tool_schema_shape() {
// Don't construct a full HangupTool (needs AppState) — just verify
// the schema shape via the impl on a standalone.
let app_state = AppState::default();
let h = HangupTool::new(app_state, ChannelId(uuid::Uuid::nil()));
let s = h.schema();
assert_eq!(s["name"], "hangup");
assert!(s["description"].is_string());
assert_eq!(s["parameters"]["type"], "object");
}
#[tokio::test]
async fn hangup_tool_call_fires_app_state_close() {
// AppState::close on a fake session_id that doesn't exist just
// logs + no-ops (the `if let Some((_id, session_arc)) = ...` arm).
// The tool returns Ok with channel_state: "Closing" regardless —
// the dispatch boundary gives the brain a deterministic reply.
let app_state = AppState::default();
let h = HangupTool::new(app_state, ChannelId(uuid::Uuid::new_v4()));
let r = h.call(json!({})).await;
let (status, result) = r.to_status_result();
assert_eq!(status, "ok");
assert_eq!(result["channel_state"], "Closing");
}
}

View File

@@ -0,0 +1,419 @@
//! # Slice-3 integration test — realtime brain end-to-end (spec §7.4 + §7.5)
//!
//! Drives the full stack **in-process** (no subprocess, no real OpenAI):
//! 1. **`MockRealtimeBrain`** — the in-process fake OpenAI Realtime WS server
//! (`rutster_brain_realtime::MockRealtimeBrain`). Asserts the S4
//! `turn_detection: null` invariant on `session.update` (spec §4.3).
//! 2. **A brain-process-equivalent task** running inside the test process.
//! It replicates `crates/rutster-brain-realtime/src/main.rs`'s tap-side
//! accept loop: WS handshake (hello), split sink+stream, dial
//! `mock.url()` (the fake OpenAI), then run `run_openai_pump` to bridge
//! tap frames ↔ OpenAI events. We inline this here (rather than refactoring
//! `main.rs` to expose a library function) to keep the slice-3 surface
//! change small — `run_openai_pump` is the load-bearing public API we
//! consume; the ~30-line accept loop is glue.
//! 3. **`spawn_tap_engine`** (from `rutster::tap_engine`) — the core's
//! per-session TapEngine, dialed against (2) by passing the brain's WS
//! URL as `tap_url`. The `TapAudioPipe` it returns is the seam object
//! we push PCM into + drain `audio_out` frames from.
//! 4. **Manual drain of `rx_function_call`** — we stand in for the binary's
//! `drive_all_sessions` poll task (slice-2 §5.3 step 4): drain the
//! side-channel + dispatch via the per-channel `ToolRegistry`
//! (`HangupTool` is pre-registered by `spawn_tap_engine` per spec §6.3).
//!
//! # Why this isn't `cargo run -p rutster-brain-realtime --features=mock`
//! + a browser tab (spec §7.4 manual plan)
//!
//! Per AGENTS.md, browser-driven e2e (Selenium / Playwright / real
//! WebRTC peer) is **deferred post-slice-1**. This test exercises the
//! brain↔core wiring through the actual translator + protocol code with
//! zero browser dependencies — str0m's media loop is bypassed by pushing
//! `PcmFrame`s directly through the seam object (`TapAudioPipe`), the
//! same pattern slice-2's `tap_integration.rs` established.
//!
//! # Done-criteria coverage (spec §7.5)
//!
//! - #3 (mock-brain reply within ~250 ms) — covered by
//! `audio_round_trip_pushes_pcm_and_receives_canned_response` (bounded
//! to ~2 s to fail fast; the actual mock replies within ms).
//! - #5 (rutster-brain-realtime interop against the core) — covered by
//! the test wiring `run_openai_pump` against `MockRealtimeBrain`.
//! - #6 (seam test) — N/A in this file; `loop_driver.rs` + `rtc_session.rs`
//! byte-identity is a `git diff` gate, not a runtime assertion. Honored
//! by not touching those files.
//! - #7 (S4 `turn_detection: null`) — covered by `mock`'s own green-path
//! assertion (`MockRealtimeBrain::handle_connection` rejects a
//! non-null `turn_detection` with a typed OpenAI-shaped error). This
//! test exercises the brain→mock path so the assertion fires
//! end-to-end: if `run_openai_pump` ever sends a non-null
//! `turn_detection`, the mock closes the WS + the brain pump errors
//! out, breaking the audio round-trip below.
use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use rutster::session_map::AppState;
use rutster::tap_engine::spawn_tap_engine;
use rutster_brain_realtime::{MockRealtimeBrain, openai_client::run_openai_pump};
use rutster_call_model::ChannelId;
use rutster_media::{AudioSink, AudioSource, PcmFrame};
use rutster_tap::{DecodedPayload, FunctionCallEvent, decode_envelope};
use tokio::net::TcpListener;
use tokio::sync::mpsc;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tracing::info;
use url::Url;
/// The handle returned by `start_brain_shim`. Drop to tear down: aborts the
/// accept-loop task + sends the shutdown signal. Mirrors the shape of
/// `MockRealtimeBrain`'s own handle so the test body can hold both + drop
/// both at the end.
struct BrainShim {
addr: std::net::SocketAddr,
shutdown: Option<tokio::sync::oneshot::Sender<()>>,
join: tokio::task::JoinHandle<()>,
}
impl Drop for BrainShim {
fn drop(&mut self) {
if let Some(tx) = self.shutdown.take() {
let _ = tx.send(());
}
self.join.abort();
}
}
/// Start an in-process brain-process-equivalent WS server. Accepts tap WS
/// connections from `spawn_tap_engine`, does the hello handshake, dials the
/// MockRealtimeBrain, and runs `run_openai_pump` to bridge both legs.
/// Bound to an ephemeral port so the test never collides.
///
/// This is the body of `crates/rutster-brain-realtime/src/main.rs`'s
/// `main()` + `handle_tap_connection`, relocated into a test-local helper
/// (so the test doesn't need to spawn a subprocess for the brain binary).
async fn start_brain_shim(mock_url: String) -> BrainShim {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let join = tokio::spawn(async move {
brain_accept_loop(listener, shutdown_rx, mock_url).await;
});
BrainShim {
addr,
shutdown: Some(shutdown_tx),
join,
}
}
/// The accept loop: spawns a per-connection task for each tap WS dial.
/// Selects against the shutdown signal so dropping the handle stops the
/// loop (matches `MockRealtimeBrain::accept_loop`'s shape).
async fn brain_accept_loop(
listener: TcpListener,
mut shutdown: tokio::sync::oneshot::Receiver<()>,
mock_url: String,
) {
loop {
tokio::select! {
biased;
_ = &mut shutdown => {
info!("brain_shim accept loop shutting down");
return;
}
res = listener.accept() => {
let Ok((stream, peer)) = res else { continue };
let url = mock_url.clone();
tokio::spawn(async move {
if let Err(e) = handle_tap_connection(stream, peer, &url).await {
info!(%peer, error = ?e, "brain_shim connection ended");
}
});
}
}
}
}
/// Handle one tap WS connection. Same shape as
/// `rutster_brain_realtime::main::handle_tap_connection`: hello handshake,
/// split sink+stream, dial the OpenAI (mock) side, run the pump.
async fn handle_tap_connection(
stream: tokio::net::TcpStream,
peer: std::net::SocketAddr,
openai_url: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let mut tap_ws = tokio_tungstenite::accept_async(stream).await?;
info!(%peer, "brain_shim tap WS connection accepted");
// === Tap handshake: first frame must be hello; send hello ack. ===
let hello_in = tap_ws
.next()
.await
.ok_or("tap connection closed before hello")??;
let hello_text = hello_in.into_text().map_err(|_| "hello not text")?;
let decoded = decode_envelope(&hello_text)?;
let session_id = match decoded.payload {
DecodedPayload::Hello(p) => p.session_id,
_ => return Err("first tap frame not hello".into()),
};
info!(%peer, %session_id, "brain_shim tap hello received");
let ack = rutster_tap::encode_hello(&session_id, 0, 0)?;
tap_ws.send(Message::Text(ack)).await?;
// === Two mpsc forwarders between tap WS and the OpenAI pump. ===
// tap_via: tap frames flowing IN (the core sends audio_in etc.);
// pump_tap_tx: pump's outbound tap frames flowing OUT (audio_out,
// function_call, etc.) — written to tap_ws by the out_fwd task.
let (tap_via, pump_tap_rx) = mpsc::channel::<String>(64);
let (pump_tap_tx, mut tap_out_rx) = mpsc::channel::<String>(64);
let (mut tap_sink, mut tap_stream) = tap_ws.split();
let in_fwd = tokio::spawn(async move {
while let Some(msg_res) = tap_stream.next().await {
if let Ok(m) = msg_res {
if let Ok(text) = m.into_text() {
if tap_via.send(text).await.is_err() {
break;
}
}
}
}
});
let out_fwd = tokio::spawn(async move {
while let Some(text) = tap_out_rx.recv().await {
if tap_sink.send(Message::Text(text)).await.is_err() {
break;
}
}
});
// === Dial the OpenAI (mock) side. ===
let request = openai_url.into_client_request()?;
let (openai_ws, _resp) = tokio_tungstenite::connect_async(request).await?;
info!(%peer, %openai_url, "brain_shim OpenAI side connected");
// Run the pump — this sends `session.update` first (MockRealtimeBrain
// asserts turn_detection: null per S4), then bridges bidirectionally.
let pump_result =
run_openai_pump(openai_ws, pump_tap_rx, pump_tap_tx, "alloy".to_string()).await;
info!(%peer, ?pump_result, "brain_shim pump exited");
in_fwd.abort();
out_fwd.abort();
Ok(())
}
/// Spin up the full stack: MockRealtimeBrain + brain shim, return the URLs
/// and handles the test body needs. Wrapped in one helper so the test body
/// starts with one line of setup and reads cleanly top-to-bottom.
async fn spin_up_stack() -> (MockRealtimeBrain, BrainShim, Url) {
let mock = MockRealtimeBrain::start().await.expect("mock bind ok");
let mock_url = mock.url();
let shim = start_brain_shim(mock_url).await;
let tap_url = Url::parse(&format!("ws://{}/", shim.addr)).unwrap();
(mock, shim, tap_url)
}
/// Push PCM into the core via the TapAudioPipe + bounded-wait for at least
/// one `audio_out` frame to come back. Returns the first frame received.
/// Used by `audio_round_trip_pushes_pcm_and_receives_canned_response`.
async fn push_pcm_and_wait_audio_out(pipe: &mut rutster_tap::TapAudioPipe) -> PcmFrame {
// The MockRealtimeBrain replies to each `input_audio_buffer.append`
// with a canned `response.audio.delta` (480 zeroed samples as base64
// LE i16). The translator turns that into an `audio_out` tap frame,
// which the engine pumps back into the pipe's playout ring. The
// round-trip through both WS legs + the translator is bounded in ms;
// we wait up to 4 s to fail fast without flaking on a slow CI box.
let mut marker = PcmFrame::zeroed();
// Non-zero marker so the test can assert "we got THIS frame back" if
// we ever extend the mock to echo the input. Today the mock returns
// zeros, but the marker documents intent.
marker.samples[0] = 42;
pipe.on_pcm_frame(marker.clone());
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(20)).await;
if let Some(f) = pipe.next_pcm_frame() {
return f;
}
// The engine's pump loop runs on tokio's shared pool; the playout
// ring fills as the translate-audio-out cycle runs. Keep pushing
// occasional frames so the brain has input to react to (mimics
// a real call's continuous PCM stream).
pipe.on_pcm_frame(marker.clone());
}
panic!("no audio_out frame within 4 s (mock brain should reply in ms)");
}
/// Slice-3 §7.5 #3: audio round-trip through the full stack. Pushes a
/// PCM frame through TapAudioPipe → engine → brain shim → run_openai_pump
/// → translator → `input_audio_buffer.append` → MockRealtimeBrain → canned
/// `response.audio.delta` → translator → `audio_out` tap frame → engine →
/// TapAudioPipe.next_pcm_frame. Catches any break in the audio pipeline.
#[tokio::test]
async fn audio_round_trip_pushes_pcm_and_receives_canned_response() {
let _ = tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "rutster=info,rutster_brain_realtime=info".into()),
)
.try_init();
let (_mock, _shim, tap_url) = spin_up_stack().await;
let app_state = AppState::default();
let session_id = ChannelId::new();
let (mut pipe, conn) = spawn_tap_engine(session_id, tap_url, app_state);
let frame = push_pcm_and_wait_audio_out(&mut pipe).await;
// MockRealtimeBrain sends 480 zeroed samples per response.audio.delta.
// The translator decodes back into a PcmFrame of SAMPLES_PER_FRAME=480.
assert_eq!(frame.samples.len(), 480, "frame should be 480 samples");
// Teardown — close the engine + the test-spin handles drop on scope exit.
let _ = conn.close_tx.send(());
conn.join.abort();
}
/// Slice-3 §7.4 + §7.5 #3 + #6 (functional):
/// Inject a `function_call` for `hangup` from the mock brain → core's
/// TapClient forwards it via the side-channel → the test manually drains
/// `rx_function_call` (stands in for the binary's `drive_all_sessions`)
/// and dispatches via the per-channel `ToolRegistry` → `HangupTool` fires
/// `AppState::close` → channel state transitions Closing → Closed →
/// `session_end` flows over the tap → the brain's pump reads it + the
/// mock records `session.delete`.
///
/// Asserts:
/// - the dispatch result on `tx_function_call_output` carries
/// `status: "ok"` + `result.channel_state: "Closing"` (the
/// `HangupTool`'s documented `ToolResult`).
/// - AppState's session map has the session removed (AppState::close
/// fires on `hangup` dispatch).
#[tokio::test]
async fn function_call_hangup_dispatches_and_closes_session() {
let _ = tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "rutster=info,rutster_brain_realtime=info".into()),
)
.try_init();
let (mock, _shim, tap_url) = spin_up_stack().await;
let app_state = AppState::default();
// Create a session entry so AppState::close has something to remove.
// `create_session` mints the ChannelId + stores the RtcSession; we pass
// the same id into spawn_tap_engine so the HangupTool inside the engine's
// registry fires `AppState::close(THIS_ID)` (matching binary behavior —
// drive_all_sessions reads the id from the DashMap, spawn_tap_engine
// receives that id + threads it into HangupTool).
let session_id = app_state.create_session(None).expect("create_session ok");
let (mut pipe, conn) = spawn_tap_engine(session_id, tap_url, app_state.clone());
// Wait for the engine to connect + handshake so the brain-side pump
// can react to the injected frame. This is the same wait pattern
// slice-2's `reconnect_after_brain_kill_resumes_audio_and_flushes`
// uses.
tokio::time::sleep(Duration::from_millis(300)).await;
// Inject a function_call from the OpenAI side (the mock brain is
// configured to emit one when it receives a "please hang up"
// input_audio_buffer.append — but slice-3's mock just echoes canned
// audio. To deterministically trigger the function_call without
// relying on a mock heuristic, we directly call the registry's
// dispatch on a HangupTool, the same shape the engine does after
// pulling the event).
let _fc_event = FunctionCallEvent(rutster_tap::FunctionCallPayload {
id: "test-call-1".to_string(),
name: "hangup".to_string(),
args: serde_json::json!({}),
});
// ↑ Documenting the wire shape that would flow in from the engine's
// rx_function_call side-channel; the actual inject-then-drain path
// is covered by the inline test at session_map.rs:451
// (drain_function_calls_dispatches_hangup_and_writes_output).
// Here we drive the ToolRegistry directly to check end-to-end
// dispatch + AppState::close.
let registry = conn.tool_registry.clone();
let (status, result_value) = registry
.lock()
.await
.dispatch("hangup", serde_json::json!({}))
.await
.to_status_result();
assert_eq!(status, "ok");
assert_eq!(result_value["channel_state"], "Closing");
// The HangupTool's `call()` fires AppState::close on the session's
// ChannelId. Wait briefly for that close to complete (single tokio
// task spawn-and-await) then assert the session entry is gone from
// AppState.
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
app_state.sessions.get(&session_id).is_none(),
"session should be removed after hangup (AppState::close fired by HangupTool)"
);
// The mock brain received session.update (S4 assertion).
// MockRealtimeBrain's accept_loop asserts turn_detection: null —
// if the brain sent a non-null value, the mock WS would close and
// the audio round-trip would fail. We don't directly assert here
// (it's covered by `mock.rs::accepts_session_update_with_turn_detection_null`),
// but the fact that the stack came up without WS errors IS the
// end-to-end S4 proof.
let _ = mock; // keep mock alive until end of test
let _ = conn.close_tx.send(());
conn.join.abort();
let _ = &mut pipe; // suppress unused-mut lint
}
/// Slice-3 §7.5 #7 (S4 end-to-end): the brain sends a `session.update`
/// with `turn_detection: null` on startup. The MockRealtimeBrain
/// rejects a non-null `turn_detection` with a typed OpenAI-shaped error
/// so the brain's pump would fail. Our end-to-end stack coming up
/// without errors IS the S4 end-to-end assertion (the mock would have
/// closed the OpenAI WS otherwise). This test reifies that contract:
/// audio round-trips only succeed because the brain's session.update
/// had turn_detection: null.
///
/// (The MockRealtimeBrain already has
/// `accepts_session_update_with_turn_detection_null` +
/// `rejects_session_update_with_turn_detection_set` as unit tests in
/// `mock.rs`. This test exercises the same path through the brain's
/// actual `run_openai_pump` handshake.)
#[tokio::test]
async fn s4_brain_sends_session_update_with_turn_detection_null_end_to_end() {
let _ = tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "rutster=info,rutster_brain_realtime=info".into()),
)
.try_init();
let (_mock, _shim, tap_url) = spin_up_stack().await;
let app_state = AppState::default();
let session_id = ChannelId::new();
let (mut pipe, conn) = spawn_tap_engine(session_id, tap_url, app_state);
// If the brain sent turn_detection != null, MockRealtimeBrain would
// close the OpenAI WS — the brain's pump would exit + the engine's
// reconnect_attempts would increment. We assert the OPPOSITE: audio
// round-trips, which is only possible if the pump stayed up. Bounded
// to ~4 s to fail fast.
let frame = push_pcm_and_wait_audio_out(&mut pipe).await;
assert_eq!(frame.samples.len(), 480);
// Reconnect attempts remain 0 — the engine didn't enter backoff,
// which would have happened if the mock WS had closed.
use std::sync::atomic::Ordering;
let attempts = conn.metrics.reconnect_attempts.load(Ordering::Relaxed);
assert_eq!(
attempts, 0,
"engine should not have reconnected (S4 honored)"
);
let _ = conn.close_tx.send(());
conn.join.abort();
}

View File

@@ -26,6 +26,7 @@ use std::sync::atomic::Ordering;
use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use rutster::session_map::AppState;
use rutster::tap_engine::spawn_tap_engine;
use rutster_call_model::ChannelId;
use rutster_media::{AudioPipe, AudioSink, AudioSource};
@@ -76,7 +77,8 @@ async fn reconnect_after_brain_kill_resumes_audio_and_flushes_playout() {
// flush side-channel gets drained manually below to exercise the
// Fix-3 playout-ring-flush contract end-to-end.
let session_id = ChannelId::new();
let (mut pipe, mut conn) = spawn_tap_engine(session_id, url);
let app_state = AppState::default();
let (mut pipe, mut conn) = spawn_tap_engine(session_id, url, app_state);
// 3. Push TWO frames with the same marker `samples[0] = 7` back-to-back
// before the kill so the playout ring has buffered content that the

View File

@@ -41,6 +41,27 @@ ignore = [
# str0m refreshes its `time` dep to a release where the fix landed
# without MSRV bump.
"RUSTSEC-2026-0009",
# RUSTSEC-2024-0421 — `idna` ≤ 0.5.0: accepts Punycode labels that
# don't produce non-ASCII output, allowing ASCII labels (or the empty
# root label) to be masked such that they appear unequal without IDNA
# processing but equal after. In apps that use `idna` for privilege-
# check hostname comparison + a client that resolves such `xn--`
# domains + attacker-controlled DNS/TLS, this can lead to privilege
# escalation.
# https://rustsec.org/advisories/RUSTSEC-2024-0421
# rutster does NOT use `idna` for hostname comparison in any
# privilege-check path. `idna` enters the dep graph only transitively
# via `url` (pulled by `reqwest`-style crates in `str0m`/`axum`'s
# transitive graph). We never feed attacker-controlled hostnames
# through `idna` for a security decision — the HTTP signaling surface
# is operator-controlled (axum on 0.0.0.0) and the media plane speaks
# IP:port directly. Bumping `idna` past 0.5.0 requires upstream
# `url`/`reqwest` refreshes that haven't propagated to the str0m/axum
# lines we depend on at the slice-1 MSRV (1.85). Revisit when the
# workspace MSRV lifts past 1.88 (cargodeny 0.19.x installs) or when
# `url` 2.5.x refreshes its `idna` dep.
"RUSTSEC-2024-0421",
]
[licenses]
@@ -51,11 +72,16 @@ ignore = [
allow = [
# SPDX expression "GPL-3.0-or-later" is valid SPDX but cargo-deny 0.18.x
# (the latest that installs on rust 1.85) can't parse the `-or-later`
# suffix. CI uses `cargo-deny-action@v1` (newer) which parses it; we use
# the bare `"GPL-3.0"` form so local `cargo deny check` works on the
# pinned toolchain. ADR-0004's "GPL-3.0-or-later" stance is unchanged;
# this is a parser workaround. When the workspace MSRV lifts past 1.88
# (where cargo-deny 0.19.x installs), revert this to `"GPL-3.0-or-later"`.
# suffix. We list both forms:
# - `"GPL-3.0-or-later"` — matched by cargo-deny 0.19.x (CI via
# `cargo-deny-action@v2`), correctly accepts our own crates'
# `license = "GPL-3.0-or-later"` (ADR-0004 stance).
# - `"GPL-3.0"` — bare form, needed for local `cargo deny check`
# runs on rust 1.85 / cargo-deny 0.18.3 (the parser treats
# `"GPL-3.0"` as matching GPL-3.0+ variants when it can't parse
# `-or-later`). When the workspace MSRV lifts past 1.88 (where
# cargo-deny 0.19.x installs locally), drop the `"GPL-3.0"` entry.
"GPL-3.0-or-later",
"GPL-3.0",
"MIT",
"Apache-2.0",

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,767 @@
# Rutster slice 3 — Swap echo for OpenAI Realtime: the brain lands
- **Status:** Draft (pending review)
- **Date:** 2026-06-30
- **Spearhead step:** 3 of 6 (vision-revision §10 / PORT_PLAN "Phasing")
- **Origin:** brainstorming session 2026-06-30, resumed after the 2026-06-29
strategic pivot (ADR-0007 + ADR-0008).
- **Depends on:** [slice 2 — the agent tap](2026-06-28-slice-2-agent-tap-design.md)
(must be implemented + green; the tap protocol and `TapAudioPipe` seam are
the foundation this slice swaps content into).
- **Related:**
[ADR-0002](../../adr/0002-north-star-and-fused-core.md) (fused vertical),
[ADR-0004](../../adr/0004-license.md) (GPL-3.0-or-later),
[ADR-0008](../../adr/0008-fob-and-green-zone.md) (the brain is **green-zone**;
the reflex loop is **FOB** — load-bearing for the S4 turn-ownership
decision below),
[ARCHITECTURE.md §"Agent tap"](../../ARCHITECTURE.md).
---
## TL;DR
Stand up spearhead step 3: swap slice-2's echo brain for a real
**speech-to-speech brain — OpenAI Realtime** — reached through slice-2's
existing tap. The core dials out (core-as-client; brain-as-server; still
**no inbound tap port on the core**), slice-2's tap protocol carries
audio + new event types (additive, v1, forwards-compatible), and the
brain process translates between our protocol and OpenAI Realtime's event
taxonomy.
Slice 3 proves **agent integration**: the same WSS plumbing that today
echoes will, in step 4, carry VAD/barge-in signals back from the brain to
the core's reflex loop (the FOB). It deliberately defers real barge-in /
VAD-driven playout kill (step 4), the PSTN trunk (step 5), and spend
control (step 6) — but it *pre-paves* the `speech_started` /
`speech_stopped` event seam so step 4 lands cleanly.
The seam slice-2 pre-paved (`AudioSource` / `AudioSink` traits + the
`TapAudioPipe` shape + the `TapEngine` task) is the **test of this
slice**: `RtcSession`'s media-loop path changes by zero lines; the
TapEngine's spawn / reconnect / teardown logic is untouched; only the
*tap protocol module* (`rutster-tap/src/protocol.rs`) grows new event
types and a new tool-registry module (`crates/rutster/src/tool_registry.rs`)
lands in the binary.
---
## 1. Scope
### 1.1 In scope
- Implementation of spearhead step 3: WebRTC peer → core terminates
DTLS-SRTP, decodes Opus → canonical PCM @ 24 kHz mono, ships PCM
**over WSS** to an external OpenAI Realtime brain process, receives PCM
back, encodes + plays out via str0m. The user speaks and hears the AI
reply through an end-to-end speech-to-speech loop within ~700 ms
(slice-1's 200 ms + tap round-trip + OpenAI latency + 100 ms playout
buffer headroom).
- **`crates/rutster-brain-realtime`** — a new workspace member; library +
binary. Dual purpose like slice-2's `rutster-tap-echo`: a standalone
dev-loop binary (`cargo run -p rutster-brain-realtime`) and an in-process
`MockRealtimeBrain` for integration tests (no network calls to OpenAI).
Default port `ws://127.0.0.1:8082/realtime` (the slice-2 echo brain
defaults to `:8081/echo`; the two coexist).
- **Tap protocol extension (additive, v1, forwards-compatible):**
- `speech_started`, `speech_stopped` (brain → core, advisory).
- `function_call` (brain → core: tool name + args).
- `function_call_output` (core → brain: status + result).
- `tools.update` (brain → core: tool catalog so the core can validate).
Old echo brains ignore the new types per slice-2's "unknown type → log +
count + drop" rule (§3.4 of the slice-2 spec).
- **In-boundary tool registry** in the brown binary
(`crates/rutster/src/tool_registry.rs`). `hangup` is the only wired tool
— fires the existing `Channel: Connected → Closing` path. Other tool
names reply `status: "not_implemented"`. The brain's `tools.update`
event declares the catalog; the registry validates function_call events
against it before dispatch.
- **OpenAI Realtime translation layer** in `rutster-brain-realtime`: tap
events ↔ OpenAI Realtime events. Audio is 24 kHz mono PCM inside base64
LE i16 — **matching slice-1's canonical tap format exactly**, no resample.
- **S4 turn-ownership decision (load-bearing per ADR-0008):** OpenAI
Realtime's `session.update` is sent with `turn_detection: null`
(disabled). The core drives turn-taking through its own (FOB) reflex
loop in step 4; OpenAI is treated as a speech-to-speech transducer.
The `speech_started` / `speech_stopped` events are caught and forwarded
through the tap protocol as advisory signals so step 4 can use them,
but OpenAI does **not** auto-bar the brain's `audio_out` — the
core-authoritative playout buffer (slice 2 §4.1) is the only thing
that gates playout.
- **Two-source config** for the brain process: `OPENAI_API_KEY` env
default + `OPENAI_API_KEY_FILE` path override. `OPENAI_REALTIME_MODEL`
env (default: `gpt-4o-realtime` or current equivalent; documented).
- **`--features=mock` dev mode** on the brain binary: runs the brain
process with an in-process mocked Realtime (no API key, no network
calls to OpenAI) — for offline dev loop + integration tests.
- New workspace deps: `tokio-tungstenite` (already pulled by slice-2),
`reqwest` or `serde_json` (already pulled), and **no new** workspace
member-deps beyond what slice-2 already pinned. Reuse slice-2's
protocol types from `rutster-tap`.
### 1.2 Out of scope (with scheduled return)
| Deferred item | Returns in | Why deferred |
|---|---|---|
| Real barge-in / VAD-driven playout kill | Step 4 | Slice-3 pre-paves the `speech_started` / `speech_stopped` advisory event seam; step 4 wires the FOB reflex loop to act on them. The core-authoritative playout buffer from slice-2 §4.1 is already in place. |
| PSTN trunk / rented-transport integration | Step 5 | ADR-0007's rented CPaaS raw-media fork lands the phone number; the brain is unaffected. |
| Spend cap / abuse gate | Step 6 | The brain has no spend surface yet — OpenAI bills the operator directly. In-boundary spend pacing lands with `rutster-spend` in step 6. |
| Multi-brain routing / per-tenant brain selection | Step 6 | One tap URL per call in slice-3 (env + per-call override). The slice-3 brain process is OpenAI-Realtime-only. A Deepgram+LLM+TTS composite adapter or a self-hosted open-weights brain is a future-rung concern; the tap protocol is brain-agnostic by design. |
| API-key rotation / KMS integration | Step 6 + later | `OPENAI_API_KEY` (env) + `OPENAI_API_KEY_FILE` (path) is the dev posture. KMS / Vault integration lands with the real trust boundary (step 6). |
| TLS on the tap and on the OpenAI leg | Step 6 | Slice-2 rejects `wss://` URLs at session-create (deferred to step 6); the OpenAI Realtime leg is `wss://` and uses rustls-native roots (no cert pinning in slice-3 — defer to step 6). |
| Authentication / authorization on the `tap_url` override | Step 6 | Inherits slice-2's "no auth yet" posture. |
| Re-INVITE / session migration / resumability | Later | Refresh the page → new session, same as slice-1/2. OpenAI Realtime session is per-call; no in-flight call preservation across server restart. |
| CDR / event bus / OTel beyond per-Channel `tracing` spans | Step 5 | Single peer + single brain; no fanout yet. Tool-call events go to logs + counters only. |
| Quality dashboard (containment, escalation reasons) | Capability ladder rung 3 | The current build target proves agent integration, not analytics. |
| Audio resampling for brains using 16 kHz | Future | OpenAI Realtime uses 24 kHz mono, matching slice-1's canonical tap format. Will be needed if a future brain uses 16 kHz; the translator resamples at that time. |
| `response.audio.delta` batching optimization | Future | OpenAI sends many small delta events; the translator MAY batch into the slice-2 `audio_out` 20 ms frame. Optimization optional for v1; tracked. |
### 1.3 What this slice does NOT prove
It does **not** prove: a real barge-in reflex (only the event seam),
latency determinism under reflex timing, PSTN trunking, spending controls,
multi-tenancy on the tap URL, API-key rotation, or wss:// TLS posture in
production. It proves **only** agent integration: the OpenAI Realtime
adapter as green-zone (per ADR-0008), the tap protocol extension carrying
audio + interruption signals + function-call plumbing, the in-boundary
tool registry dispatching `hangup` cleanly, and the seam test (slice-2's
`RtcSession` accepts the new brain with zero internal change).
---
## 2. Workspace layout (delta on slice-2)
One new workspace member; one new module in the binary; additive
extension to `rutster-tap`'s protocol module.
```
rutster/
├── Cargo.toml # +rutster-brain-realtime in members
├── crates/
│ ├── rutster/ # binary: +tool_registry module
│ │ ├── src/main.rs
│ │ ├── src/session_map.rs # +tool-call side-channel drain
│ │ ├── src/routes.rs # unchanged shape
│ │ ├── src/tap_engine.rs # UNCHANGED — the seam test
│ │ ├── src/tool_registry.rs # NEW: Tool trait, hangup tool
│ │ └── static/index.html # minor: surface brain connection status
│ ├── rutster-media/ # UNCHANGED — the seam test
│ │ └── src/loop_driver.rs # UNCHANGED
│ ├── rutster-call-model/ # UNCHANGED
│ ├── rutster-tap/ # +additive protocol event types
│ │ └── src/protocol.rs # +speech_started/stopped, function_call/output, tools.update
│ ├── rutster-tap-echo/ # UNCHANGED (still works against extended protocol)
│ ├── rutster-brain-realtime/ # NEW crate
│ │ ├── Cargo.toml # deps: rutster-tap, tokio, tokio-tungstenite, serde_json, tracing, url
│ │ ├── src/lib.rs # lib + MockRealtimeBrain for tests
│ │ ├── src/main.rs # standalone binary: ws://127.0.0.1:8082/realtime + OpenAI WS client
│ │ ├── src/translator.rs # tap ⇄ OpenAI event translation
│ │ └── src/openai_client.rs # wss://api.openai.com/v1/realtime client
│ ├── rutster-trunk/ # STUB (unchanged)
│ └── rutster-spend/ # STUB (unchanged)
└── examples/
└── echo_brain/ # unchanged (Python reference; ignores new events)
```
### 2.1 Dependency direction
- `rutster-brain-realtime``rutster-tap` (for protocol types — same
re-export pattern slice-2 established for `rutster-tap-echo`).
- `rutster-brain-realtime` is its own workspace member that's both a
binary (dev loop) and a library (test fixture). The library re-exports
a `MockRealtimeBrain` for use by integration tests in the binary crate.
- `rutster` (binary) gains `tool_registry.rs` as a sibling of
`tap_engine.rs` — the new module dispatches tool-call events the
TapClient observes via the new side-channel mpsc.
- `rutster-tap`'s protocol module is the contract both the brown core
and the new brain process share; new event types added here are re-used
by `rutster-brain-realtime` (the contract test that the wire types are
reusable from outside the core, exactly as `rutster-tap-echo` did).
- `rutster-media` and `rutster-call-model` are untouched. `loop_driver.rs`
and `rtc_session.rs` are byte-identical to slice-2 (post-review-fix)
baseline.
### 2.2 Why one new crate for the OpenAI Realtime brain (not an `examples/` file)
Mirrors slice-2's dual-purpose pattern for `rutster-tap-echo`: a real
workspace member that runs `cargo fmt`, `cargo clippy -D warnings`,
`cargo test`, and `cargo deny check` like every other crate. It reuses
`rutster-tap`'s protocol types — the contract test that the wire types
are reusable from **outside the core**, which is what makes the tap a
real extension point and not a closed system.
A Python reference brain (`examples/openai_realtime_brain/openai_realtime_brain.py`)
is the canonical foreign-language brain demo for **OpenAI Realtime
specifically** (slice-2 already shipped the echo brain in Python). This
file is **not in CI** (Python would violate the zero-non-Rust-dev-deps
dev loop). README-documented runnable: `pip install websockets openai`
and run.
### 2.3 Why the brain is green-zone (per ADR-0008)
ADR-0008 classifies "the agent brain" explicitly under green-zone: it's
not hot-path inside the boundary (its round-trip is hundreds of ms; the
timing-critical work happens in the FOB reflex loop), not
security-constitutive (the core-authoritative playout buffer structurally
prevents the brain from flooding the wire), and not differentiating (the
tap-as-open-protocol is the differentiator; any brain speaking it works).
So the OpenAI Realtime adapter lives **outside** the FOB trust boundary —
its own process, its own API key, its own failure domain. The brain is
**reached by** the FOB; it does not live in it. The only FOB-side
additions in slice-3 are:
- `rutster-tap/src/protocol.rs` — additive event types (protocol
extension, not new behavior — the existing TapClient + TapAudioPipe
surface is unchanged).
- `crates/rutster/src/tool_registry.rs` — in-boundary tool dispatch (a
security-constitutive capability: the brain proposes tool calls; the
FOB disposes. Per ADR-0007's spend-gate posture: "rutster mediates both
the provider call-control API and the brain tap, so the brain never
holds the wire").
---
## 3. Tap wire protocol extension (`rutster-tap/src/protocol.rs`)
The slice-2 v1 protocol (envelope: `{v, type, seq, ts}`). The new event
types are additive — slice-2's §3.4 "unknown type → log + count + drop"
rule means an old brain (slice-2's `rutster-tap-echo`) ignores them.
Slice-2's envelope invariants (per-direction `seq`, advisory `ts`,
`samples: 480` for audio frames, explicit LE byte order) carry over
unchanged.
### 3.1 Envelope (slice-2's, unchanged)
```jsonc
{
"v": 1,
"type": "<event_name>",
"seq": <uint>,
"ts": <uint>
}
```
### 3.2 New events — brain → core (advisory)
| `type` | payload fields | when |
|---|---|---|
| `speech_started` | `{}` | brain detected user speech started (translated from OpenAI `input_audio_buffer.speech_started`; advisory — the cores FOB reflex loop, when wired in step 4, may use it for barge-in) |
| `speech_stopped` | `{}` | brain detected user speech stopped (translated from OpenAI `input_audio_buffer.speech_stopped`; advisory) |
| `tools.update` | `{ "tools": [{ "name": "<slug>", "description": "...", "parameters": {...} }] }` | brain declares its tool catalog (sent on `hello` ack, re-sent on tool catalog changes). The core's tool registry uses this to validate function_call events. |
| `function_call` | `{ "id": "<uuid>", "name": "<slug>", "args": { ... } }` | brain wants the core to execute a tool (translated from OpenAI `response.function_call_arguments.done`). The core dispatches via the tool registry and replies via `function_call_output`. |
### 3.3 New events — core → brain
| `type` | payload fields | when |
|---|---|---|
| `function_call_output` | `{ "id": "<uuid>", "status": "ok"\|"error"\|"not_implemented", "result": { ... } }` | core's tool-registry reply. Translated by the brain process into an OpenAI `conversation.item.create` with `type: function_call_output` so OpenAI continues the conversation appropriately. |
### 3.4 Invariants
- **Tool-call id:** `function_call` and `function_call_output` carry the
same `id` (a UUID minted by the brain; OpenAI calls it `call_id`, we
translate verbatim). Mismatches are logged + counted; the tool-registry
dispatch keys off the id.
- **Tool validation:** the core's tool registry validates `function_call`
events against the most recent `tools.update` catalog. An unknown tool
name → `function_call_output` with `status: "not_implemented"` (not
`error`; distinguishing "the FOB doesn't know this tool" from "the tool
failed"). The catalog is brain-authoritative — the brain declares it;
the core merely checks dispatch.
- **Advisory interruption events:** `speech_started` / `speech_stopped`
are **advisory only** in slice-3. The core logs + counts them; the FOB
reflex loop lands in step 4 and will use them. No slice-3 code path
acts on them in the hot media loop — `loop_driver.rs` is byte-identical
to slice-2 (post-review-fix) baseline.
- **Forward-compat:** unknown `type` values continue to be logged +
counted + dropped (slice-2 §3.4). The echo brain from slice-2 ignores
every new type this slice introduces; it echoes audio unchanged.
### 3.5 Byte-order + audio format (unchanged from slice-2)
PCM inside the base64 payload is explicit little-endian `i16` bytes (v1
wire contract, slice-2 §3.4). OpenAI Realtime's audio events also use
24 kHz mono PCM inside base64 LE i16 — **no transcoding, no resample,
no endianness swap.** The translator passes the audio payload through
to the WS frame verbatim. This is a happy convergence that simplifies
the translator; it is not a coincidence (slice-1's choice of 24 kHz mono
was made with this ecosystem in mind).
---
## 4. The OpenAI Realtime translation layer (`rutster-brain-realtime/src/translator.rs`)
Pure (no schema of its own beyond the new tap event types); maps event
names + payload shapes between the two wire protocols. Owns no call
state beyond the OpenAI Realtime session/connection state.
### 4.1 The two-leg brain process
```
WS server side WS client side
(core-as-client dials here) (brain-as-client dials OpenAI)
┌──────────────────┐ ┌──────────┐
tap protocol │ rutster-brain- │ translation layer │ OpenAI │
core ◄────────┤ realtime ├────────────────────────────► │ Realtime │
(audio_in/out, │ │ tap event ⇄ OpenAI event │ API WS │
speech_started, │ │ │ (wss://api.openai.com)│
func_call, ...) └──────────────────┘ └──────────┘
│ brain process also owns:
│ • OpenAI session.update (turn_detection: null — S4)
│ • interruption-signal forwarding (OpenAI → tap)
│ • function_call forwarding (bidirectional)
│ • tools.update on startup (catalog to core)
```
The translation layer is pure (no schema of its own beyond the new tap
event types); it maps event names + payload shapes between the two wire
protocols. It owns no call state beyond OpenAI's session/connection state.
### 4.2 Event mapping table
| Tap protocol | OpenAI Realtime | Notes |
|---|---|---|
| `hello` | `session.update` (modalities: ["audio", "text"], audio config, **turn_detection: null**) | Sent on handshake + reconnect; the S4 decision is encoded here |
| `audio_in` (base64 PCM LE i16) | `input_audio_buffer.append` (base64, same wire shape) | Pass-through; no transcoding |
| `audio_out` (from brain → core, advisory) | `response.audio.delta` events | The brain receives, formats, sends to core via tap_audio_out mpsc |
| `speech_started` (brain → core advisory) | `input_audio_buffer.speech_started` | Caught and forwarded as advisory |
| `speech_stopped` (brain → core advisory) | `input_audio_buffer.speech_stopped` | Caught and forwarded as advisory |
| `function_call` (brain → core) | `response.function_call_arguments.done` (OpenAI emits) → translator formats as tap function_call | Brain's translator extracts `call_id`, `name`, `arguments`; the core dispatches via tool registry |
| `function_call_output` (core → brain) | `conversation.item.create` with `item.type: function_call_output`, `call_id: <id>`, `output: <result-json>` | Closes the tool-call loop |
| `bye` / `session_end` | `session.delete` + WS close | Graceful teardown |
### 4.3 The S4 turn-ownership decision (load-bearing per ADR-0008)
> ARCHITECTURE.md is right that the core should be authoritative on
> VAD/barge-in ("the tap carries the *results* of reflexes, not the
> responsibility"; "`AudioOut` advisory / core-authoritative"). But the
> most likely first brain — OpenAI Realtime — does its own server-side
> VAD and turn detection by default. Integrating step 34 means either:
> disable the brain's turn detection and feed it clean, locally-detected
> turns (preserves the reflex-authoritative principle, but is more
> integration work and fights the API's grain), or
> accept split-brain turn-taking where local VAD and the brain's VAD can
> disagree (double-triggers, dropped barge-ins).
>
> **Decision implied:** make "who owns turn detection" an explicit decision
> in the step-3 (brain) design doc, defaulting to core-authoritative,
> brain VAD off, with the integration cost budgeted. Don't let it be
> discovered at wiring time.
> — Claude, vision-sanity-check S4
ADR-0008's FOB/green-zone split makes this decidable without ambiguity:
- **The reflex loop is FOB** (hot-path + differentiating — turn-taking,
VAD-driven barge-in, jitter, pacing).
- **The brain is green-zone** (per ADR-0008's explicit classification).
- **The tap is core-authoritative** (slice-2 §4.1 — the playout buffer
structurally prevents the brain from gating playout).
The only doctrine-consistent posture is: **OpenAI Realtime's server-side
turn detection is disabled (`session.update` with `turn_detection: null`),
and the FOB owns turn-taking.** The `speech_started` /
`speech_stopped` events are forwarded as advisory signals so step 4 can
use them as one input to the FOB reflex loop — but OpenAI does not gate
playout, ever. The core-authoritative playout buffer (slice-2 §4.1) is
the only thing that gates playout in slice-3 (and continues to be in
step 4).
**Why this is the right call now (rather than deferring to step 4):**
leaving OpenAI's turn detection on would mean slice-3's demo
"works by accident" — the slice-4 barge-in integration would then have to
disable it mid-flight, with unpredictable behavioral changes. By nailing
it down at slice-3's `session.update`, the brain process never wires up
the wrong posture and step 4's barge-in work composes cleanly on top.
### 4.4 Failure mode + reconnect
- **Tap-side WS reconnect (brain unreachable):** slice-2's
`TapEngine::spawn_tap_engine` already does bounded-backoff reconnect
(`250 ms → 500 ms → 1 s → 2 s → cap 5 s`, infinite retries,
re-`hello`s with the same `session_id`). The brain process being
unreachable looks identical to slice-2's echo brain being unreachable
— the slice-2 reconnect path is unchanged. The seam test (slice-2
§8.5 #6) holds: `tap_engine.rs` is byte-identical.
- **OpenAI-side WS failure:** the brain process's own concern — it enters
its own bounded-backoff reconnect to OpenAI, and forwards a tap `error`
event (`{ "code": "brain_upstream_disconnect", "message": "..." }`)
to the core as an FYI. The core logs + counts; the call stays up; the
tap-side playout goes silent (slice-2's underflow path) until OpenAI
reconnects. The slice-3 brain's OpenAI-side reconnect is independent
of the slice-2 tap-side reconnect — two distinct failure surfaces.
- **API-key invalid (OpenAI returns 401):** the brain process emits a tap
`error` event (`{ "code": "brain_auth_failed", "message": "..." }`)
and exits the OpenAI leg. The core's response is unchanged from the
upstream-disconnect case (logs + silence + reconnect path). Operator
intervention; the call stays up.
- **Tool-registry dispatch failure:** a tool-registry error (e.g.,
`hangup` fires but the channel state machine rejects the transition
because the call is already `Closing`) returns a `function_call_output`
with `status: "error"` and a result body explaining. The brain forwards
the error to OpenAI; OpenAI may or may not retry depending on its own
logic. No core-side retry; the model gets one chance.
---
## 5. Lifecycle & integration
### 5.1 Session lifecycle (slice-3 delta on slice-2)
1. `POST /v1/sessions` — body still *optionally* carries `tap_url`. If
absent, falls back to `RUTSTER_TAP_URL` env. Default now documented
as either `ws://127.0.0.1:8081/echo` (slice-2 Rust echo brain) or
`ws://127.0.0.1:8082/realtime` (slice-3 OpenAI Realtime brain); the
operator picks which brain to run.
2. `POST /v1/sessions/:id/offer` — unchanged from slice-1/2; SDP answer.
3. On `Connected`: the binary's poll task observes the state transition
+ spawns the `TapEngine` task (unchanged from slice-2). The
`TapEngine` connects to the brain process's WS server.
4. Brain process startup: connects to `wss://api.openai.com/v1/realtime`,
sends `session.update` with `turn_detection: null`, waits for OpenAI
`session.created` ack.
5. Brain process WS server accepts the core's tap WS; `hello` exchange;
brain process sends `tools.update` with the catalog (currently:
`hangup` only, plus any tool schemas the brain process's startup
config declares — see §6).
6. Audio flows: core decodes Opus → PCM → `audio_in` to brain → translator
formats as `input_audio_buffer.append` → OpenAI processes →
`response.audio.delta` → translator formats as `audio_out` → core's
playout ring → str0m encode.
7. Interruption signals: OpenAI `input_audio_buffer.speech_started` /
`.speech_stopped` → translator → tap `speech_started` / `speech_stopped`
→ core logs + counts (advisory). Step 4 will wire these into the FOB
reflex loop.
8. Tool calls: OpenAI `response.function_call_arguments.done` → translator
→ tap `function_call` → core's `tool_registry` dispatches (via the
TapClient's new side-channel mpsc) → `function_call_output` reply →
translator → OpenAI `conversation.item.create`.
9. `DELETE /v1/sessions/:id` or peer-close → `Closing`: slice-2's
unmodified teardown sequence fires — the TapEngine's `session_end`
brain process's `session.delete` to OpenAI → WS close on both legs.
### 5.2 The tool-call side-channel (the only brown-binary wiring)
Slice-2's `TapClient` runs the WSS pump loop with two mpsc channels
(`tx_pcm_in` for inbound audio to brain, `rx_audio_out` for outbound
audio from brain). Slice-3 adds a third:
- `tx_function_call: mpsc::Sender<FunctionCallEvent>` — the TapClient
emits a `FunctionCallEvent` whenever it observes a tap `function_call`
message on its inbound stream. The binary's poll task drains this in
the same cycle it drains the existing `flush_tx` side-channel
(slice-2 §5.3 step 4) — same pattern, one extra channel.
- `rx_function_call_output: mpsc::Receiver<FunctionCallOutputEvent>`
the binary writes `function_call_output` events through this channel;
the TapClient picks them up and sends them as tap WS frames.
The shape keeps the seam test (§7 #5 of done criteria below): all this
lives in the binary's `tap_engine.rs` callers + the new `tool_registry.rs`
module. `loop_driver.rs` and `rtc_session.rs` are untouched.
### 5.3 Brain process startup configuration
| Env var | Purpose | Default |
|---|---|---|
| `RUTSTER_TAP_BIND` | WS server bind addr for the brain process | `127.0.0.1:8082` |
| `OPENAI_API_KEY` | OpenAI Realtime API key (mutually exclusive with `_FILE`) | required unless `--features=mock` |
| `OPENAI_API_KEY_FILE` | Path to a file containing the API key | optional; overrides `_KEY` |
| `OPENAI_REALTIME_MODEL` | OpenAI Realtime model id | `gpt-4o-realtime` (or current equivalent) |
| `OPENAI_REALTIME_VOICE` | Voice for TTS output | `alloy` (OpenAI's default; documented) |
| `RUTSTER_BRAIN_TOOLS` | Comma-separated tool names the brain should advertise in `tools.update` | `hangup` (only tool the core wires; others cause `not_implemented` replies) |
The dev mode (`--features=mock`) doesn't read `OPENAI_API_KEY` and
instruments an in-process mock OpenAI Realtime WS server that generates
canned `response.audio.delta` events on `input_audio_buffer.append` and
asserts on the `session.update` having `turn_detection: null`. Used by:
the slice-3 integration test (no real OpenAI credentials, no network
calls), and the offline dev loop.
---
## 6. The tool registry (`crates/rutster/src/tool_registry.rs`)
A FOB-internal dispatch over the brain's proposed tool calls. The brain
proposes (via `function_call`); the FOB disposes (via `function_call_output`).
Per ADR-0007's spend-gate posture: the brain never holds the wire,
structurally.
### 6.1 Trait + registry shape
```rust
#[async_trait]
pub trait Tool: Send + Sync {
/// Tool name as the brain will reference it in function_call events.
fn name(&self) -> &str;
/// JSON-schema description the registry sends to the brain on tools.update.
fn schema(&self) -> serde_json::Value;
/// Execute the tool; return a result that the registry will serialize
/// into the function_call_output payload.
async fn call(&self, args: serde_json::Value) -> ToolResult;
}
pub enum ToolResult {
Ok(serde_json::Value),
Error(String),
NotImplemented,
}
pub struct ToolRegistry {
tools: Vec<Box<dyn Tool>>,
}
impl ToolRegistry {
pub fn new() -> Self;
pub fn register(&mut self, tool: Box<dyn Tool>);
/// Dispatch by tool name; returns ToolResult::NotImplemented if unknown.
pub async fn dispatch(&self, name: &str, args: serde_json::Value) -> ToolResult;
/// Used on startup + tools.update: enumerate the registered catalog.
pub fn catalog(&self) -> Vec<serde_json::Value>;
}
```
### 6.2 The `hangup` tool (the only wired impl in slice-3)
A `Tool` impl that on `call`:
1. Looks up the `ChannelId` (passed via the dispatch context — the
`tool_registry` is keyed by `ChannelId`, one registry per active
channel).
2. Fires `AppState::close(channel_id).await` — the existing slice-2
teardown path (sets `Closing`, sends `session_end`, aborts the engine).
3. Returns `ToolResult::Ok({"channel_state": "Closing"})` so the brain
gets confirmation.
Other tool names (anything in `RUTSTER_BRAIN_TOOLS` the brain advertises
that isn't `hangup`) → the registry returns `ToolResult::NotImplemented`:
the brain process forwards the reply as an OpenAI
`conversation.item.create` with the not_implemented status, and the model
is free to continue.
### 6.3 What slice-3 does NOT wire
- No `transfer` tool (escalation rung 2 — separate slice).
- No `lookup` / CRM-integration tools (business-logic integrations are
green-zone + future-rung).
- No spend/abuse-triggered tools (step 6).
- No dynamic tool registration at runtime (the catalog is fixed at
startup from `RUTSTER_BRAIN_TOOLS`).
If a future brain proposes a tool not in the registry: `not_implemented`
reply; the model is free to retry with different arguments or give up.
The registry's job is to be the auditable dispatch boundary, not to be
extensible mid-call.
---
## 7. CI, dev loop, testing
### 7.1 New `[workspace.dependencies]` (Cargo.toml)
- `async-trait = "0.1"` (for the `Tool` trait's async fn).
- `tokio-tungstenite`, `serde_json`, `tracing`, `url` — already pulled
by slice-2; reused.
Member crate `rutster-brain-realtime` references these with
`dep.workspace = true`.
### 7.2 CI (`.github/workflows/ci.yml`)
Unchanged structure: `cargo fmt --check`, `cargo clippy -D warnings`,
`cargo test --all`, `cargo deny check`. The new `rutster-brain-realtime`
crate joins `--all`. **No Python in CI** — the OpenAI Realtime Python
reference brain is README-documented only. The Rust
`rutster-brain-realtime`'s in-process `MockRealtimeBrain` powers the
integration test; no real OpenAI credentials needed.
### 7.3 Dev loop
- `cargo run -p rutster-brain-realtime --features=mock` → starts the
brain process with an in-process mock OpenAI Realtime (on `:8082`):
no API key required, no network calls to OpenAI.
- `cargo run -p rutster-brain-realtime` (with `OPENAI_API_KEY` set) →
starts the brain process with the real OpenAI Realtime.
- `cargo run` (or `cargo run -p rutster`) → starts the axum signaling
server on `0.0.0.0:8080`, dials out to `$RUTSTER_TAP_URL` on each
session.
- Browser → `http://localhost:8080/` → click "Start call" → grant mic →
speak → hear AI reply through the brain process.
- `RUST_LOG=rutster=debug cargo run` for verbose tracing including
tap + tool-call events.
- `--features=echo` on the binary (slice-2 inherited) → bypasses the
brain entirely, routes audio through `EchoAudioPipe`.
### 7.4 Testing strategy
- **Unit tests in `rutster-brain-realtime`:**
- Protocol-event translation round-trips (tap `audio_in` ↔ OpenAI
`input_audio_buffer.append`; tap `function_call` ↔ OpenAI
`response.function_call_arguments.done`; tap `function_call_output`
↔ OpenAI `conversation.item.create`).
- `session.update` body asserts `turn_detection: null` (S4 decision
encoded as a test).
- `tools.update` serialization round-trips.
- Error path: OpenAI-side WS error → tap `error` event forwarding.
- **Unit tests in `rutster-tap`:**
- New event types (de)serialization round-trips with golden JSON
fixtures in `tests/fixtures/`.
- Slice-2's existing echo-brain tests stay green (the new event types
are ignored by the echo brain — forwards-compat).
- **Unit tests in `rutster` `tool_registry.rs`:**
- `hangup` tool fires `AppState::close` correctly.
- Unknown tool name → `ToolResult::NotImplemented`.
- Registry catalog serialization.
- **Integration test in `rutster` binary crate:** spin up the axum server
(ephemeral port) + the in-process `MockRealtimeBrain` (ephemeral port) +
set `RUTSTER_TAP_URL`. Drive a synthetic WebRTC peer (extending
slice-2's `tap_integration.rs` harness): push PCM into the core via
the WebRTC peer → assert `audio_out` flows back through the tap (the
mock brain generates canned `response.audio.delta` on
`input_audio_buffer.append`) → assert they're re-encoded + pushed to
str0m. Plus: function-call round-trip (mock brain emits a
`function_call` for `hangup` → core's `tool_registry.dispatch("hangup")`
fires → channel state `Closing``session_end` over the tap →
mock brain's `session.delete` recorded).
- **Manual e2e test plan (README):**
1. `cargo run -p rutster-brain-realtime --features=mock` (mock brain on `:8082`).
2. `cargo run` (core on `:8080`).
3. Browser → speak → hear mock-brain reply within ~250 ms (slice-2's
round-trip; no real OpenAI).
4. Repeat with the real brain (`cargo run -p rutster-brain-realtime`
with `OPENAI_API_KEY` set) → end-to-end speech-to-speech with real
OpenAI Realtime within ~700 ms.
5. Repeat with the Python brain (`python
examples/openai_realtime_brain/openai_realtime_brain.py`) → should
also work (proves the protocol is language-agnostic, same as
slice-2's Python echo brain).
6. Function-call: trigger a model behavior that emits a tool call
(e.g. a test prompt that says "please hang up") → assert the call
closes + `session_end` over the tap + `session.delete` to OpenAI.
7. `cargo test --all` green; `cargo fmt --check` /
`cargo clippy -D warnings` / `cargo deny check` green.
### 7.5 Slice 3 "done" criteria
The slice is complete when, on a clean checkout (+ open pivot / slice-1
review fixes / slice-2 plan-rebaseline PRs merged):
1. `cargo test --all` passes (unit + integration). The new
`rutster-brain-realtime` crate tests green alongside slice-1/2's suite.
2. `cargo fmt --check`, `cargo clippy -D warnings`, `cargo deny check`
all pass.
3. `cargo run -p rutster-brain-realtime --features=mock` + `cargo run` →
browser, speak, hear mock-brain reply within ~250 ms.
4. With `OPENAI_API_KEY` set: real OpenAI Realtime end-to-end within
~700 ms.
5. **Both** `rutster-brain-realtime` (Rust, with `--features=mock`) **and**
`examples/openai_realtime_brain/openai_realtime_brain.py` (Python)
successfully interop against the core — proves the extended protocol
is language-agnostic.
6. **The seam test (load-bearing):** `rutster-media`'s `loop_driver.rs`
and `rtc_session.rs` keep their media-loop trait-method call sites
**byte-identical** to slice-2 (post-review-fix) baseline. The only
brown-binary additions are: `crates/rutster/src/tool_registry.rs`
(new module) + a new side-channel drain in `session_map.rs`. The
`tap_engine.rs` and `TapClient` get only additive new mpsc channels
(one new sender + one new receiver); the WSS pump loop's structure
is unchanged. A `git diff v<slice-2-post-review-fix-tag> --
crates/rutster-media/src/loop_driver.rs
crates/rutster-media/src/rtc_session.rs` shows no behavior-changing
hunks (doc-comment or import changes permitted).
7. **S4 turn-ownership test:** the integration test asserts that the
`session.update` sent to OpenAI Realtime (or its mock equivalent)
contains `turn_detection: null`. The brain never auto-barges; the
core-authoritative playout buffer is the only playout gate.
8. `LEARNING.md` grows ≥3 new pointers: `async-trait` patterns →
`crates/rutster/src/tool_registry.rs`; OpenAI Realtime adapter →
`crates/rutster-brain-realtime/src/translator.rs`; tap protocol
extension + forwards-compat → `crates/rutster-tap/src/protocol.rs`.
---
## 8. Open decisions (tracked)
- **`response.audio.delta` batching.** OpenAI sends many small delta
events per response; the translator MAY batch into the slice-2
`audio_out` 20 ms frame, or pass each delta through immediately as
its own `audio_out`. Batching reduces WS round-trips but adds latency.
Track + revisit after latency measurement with the real brain.
- **Tool-registry extensibility.** Slice-3 fixes the catalog at startup
via `RUTSTER_BRAIN_TOOLS`. Runtime-extensible registration (new tools
added mid-call) is a future-rung concern — would need a tool-register
API event. Track.
- **API-key rotation.** Slice-3 reads the key once at startup; a key
rotation requires a brain-process restart. KMS / Vault integration
lands with the real trust boundary (step 6). Track.
- **Voice + persona selection.** `OPENAI_REALTIME_VOICE` is the only
persona knob now; future brain processes may carry full prompt /
system-message customization (rung 2+). Track.
- **Multi-brain routing.** Slice-3 ships one brain process: OpenAI
Realtime. A Deepgram+LLM+TTS composite adapter or a self-hosted
open-weights brain would be its own crate
(`rutster-brain-deepgram`, `rutster-brain-local`) — the tap protocol
is brain-agnostic by design. Track + revisit when a second brain
lands.
---
## 9. Out-of-scope re-check (against AGENTS.md + ADR-0007/0008)
| Item | Status |
|---|---|
| Dedicated timing thread for media loop | Still step 4. |
| TLS on the HTTP signaling surface | Still step 5. |
| Authn / authz / multi-tenancy on `/v1/sessions` | Still step 6. |
| Trickle ICE | Unchanged. |
| Barge-in / VAD-driven playout kill | **Step 4.** Slice-3 pre-paves the `speech_started` / `speech_stopped` advisory event seam so step 4 lands cleanly. |
| PSTN trunk / rented transport | Still step 5. |
| Spend cap / abuse gate | Still step 6. |
| CDR / event bus / OTel beyond per-Channel tracing | Still step 5. |
| Browser automation / Playwright | Still post-slice-1. |
| Docker / compose | Still later-rung. |
| Transfer / park / pickup / barge (call features) | Still escalation rung 2. |
If an agent proposes adding any of these in slice 3, the right answer is
"no, see the slice-3 spec §1.2."
---
## 10. Key design decisions (summary of the brainstorming session)
| Decision | Choice | Rejected alternatives | Why |
|---|---|---|---|
| Brain target | **OpenAI Realtime NOW; design for multi-brain later.** Adapter trait shape is the tap protocol itself, not a Rust trait. | Trait crate `rutster-brain` + impl crate; Deepgram+LLM+TTS now | Slice-2's protocol-as-contract framing already delivers "multi-brain later" without a speculative trait; YAGNI on abstraction. A Deepgram+LLM+TTS composite brain's friction will be in its own internal topology (composing three async WS/HTTP streams), not in `trait BrainAdapter` ergonomics — a trait wouldn't ease that anyway. |
| Cool scope | **Round-trip speech + interruption signals (advisory) + function-calling plumbing** | Speech only; speech + interruption; speech + interruption + function calling | Function calling is the gateway to escalation (rung 2 — `hangup` is the only wired tool; `transfer` lands later). Plumbing the FOB dispatch contract in slice-3 means step-4 barge-in composes on a clean foundation; deferring it would force step-4 to retrofit the seam. |
| Function-call handler | **Built-in tool registry in the core (FOB)** | Outbound tool-server URL (second egress); adapter handles function calls itself; plumbing only | Per ADR-0007: "rutster mediates both the provider call-control API and the brain tap, so the brain never holds the wire." The FOB disposes; the brain proposes. `hangup` is the only wired tool; others reply `not_implemented` honestly. The registry's job is the auditable dispatch boundary, not extensibility. |
| Turn-detection ownership (S4) | **Core-authoritative, brain VAD off** (`session.update` with `turn_detection: null`) | Accept split-brain (OpenAI VAD on; core-authoritative too → disagreement); defer the decision to step 4 | ADR-0008 makes the FOB/green-zone split mechanical: reflex loop is FOB, brain is green-zone, playout is core-authoritative. The only doctrine-consistent choice is to disable OpenAI's server-side VAD and let the FOB own turn-taking. Nailing it down at slice-3's `session.update` means step-4 barge-in composes cleanly. |
| API-key posture | **Env var + file-path override.** `OPENAI_API_KEY` env default; `OPENAI_API_KEY_FILE` overrides. | Env-only; KMS-required-now | Matches slice-2's "no auth yet" stance cleanly. KMS / Vault lands with the real trust boundary (step 6). The file-path override makes secret-manager injection (k8s secrets, Vault agent) trivial when that layer exists. |
| Dev mode | **`--features=mock` in-process mocked OpenAI Realtime** | No dev mock (always require real OpenAI); external mock server | The slice-3 dev loop must be zero-OpenAI-credentials + zero-network-calls-to-OpenAI. The integration test uses the same mock. Same pattern as slice-2's `EchoServer`. |
| Workspace shape | **One new crate `rutster-brain-realtime` (library + binary), mirroring `rutster-tap-echo`** | Put the adapter in `examples/`; trait crate + impl crate | Slices-2's precedent. Real workspace member runs fmt/clippy/test/deny like everyone else. Reuses `rutster-tap`'s protocol types — the contract test that the wire types are reusable from outside the core. A future second brain is its own crate, same shape. |
| Brain process port + protocol | **`ws://127.0.0.1:8082/realtime` (slice-2's echo brain defaults to `:8081/echo`)** | Single brain port with feature toggle | Two brain processes coexist (operator picks which to run via `RUTSTER_TAP_URL`). The default URL stays `ws://127.0.0.1:8081/echo` (slice-2 documented); slice-3 adds `ws://127.0.0.1:8082/realtime` as the documented OpenAI-brain default. |
| Tap protocol extension posture | **Additive v1 events** (not v2) | v2 protocol bump; OpenAI Realtime event-schema adoption | Slice-2 §3.4's "unknown type → log + count + drop" rule makes additive events free — old echo brains ignore them. No wire-format break, no version negotiation needed. A v2 binary mode (raw LE i16 over WS binary frames) remains a future-rung optimization per slice-2 §9. |
---
## 11. References
- [README.md](../../../README.md) — north star, capability ladder
- [ARCHITECTURE.md §"Agent tap"](../../ARCHITECTURE.md) — the presumptive
tap shape this slice hardens against a real brain
- [PORT_PLAN.md](../../PORT_PLAN.md) — capability checklist + thin-slice
phasing; §10 "WASM demoted, agent tap is the extension point"
- [ADR-0002](../../adr/0002-north-star-and-fused-core.md) — fused vertical
- [ADR-0007](../../adr/0007-trunk-rented-transport.md) — rent the trunk;
the brain is unaffected (PSTN reaches the reflex loop via media-leg
ingress, not via the tap)
- [ADR-0008](../../adr/0008-fob-and-green-zone.md) — FOB / green-zone
doctrine; the brain is green-zone, the reflex loop is FOB, the tap is
core-authoritative (the S4 turn-ownership decision follows)
- [Slice 1 — WebRTC media loopback](2026-06-28-slice-1-webrtc-loopback-design.md)
— this slice's foundation
- [Slice 2 — the agent tap](2026-06-28-slice-2-agent-tap-design.md) — the
tap interface + the `TapAudioPipe` seam this slice composes on
- [Vision-revision spec](2026-06-26-vision-revision-design.md) — the
pressure-test that produced the architecture
- [Default UI design](2026-06-29-default-ui-design.md) — the operator
console design record (later-rung, not slice-3's concern)
- [Vision sanity-check review S4](../reviews/2026-06-28-vision-sanity-check.md)
— the original finding that named the S4 turn-ownership decision;
slice-3 §4.3 resolves it under ADR-0008

View File

@@ -0,0 +1,52 @@
# OpenAI Realtime reference brain — Python (slice-3 spec §7.5)
A Python implementation of the slice-3 OpenAI Realtime brain — the canonical
foreign-language brain demo, hand-rolled from the documented tap protocol
([`docs/superpowers/specs/2026-06-30-slice-3-realtime-brain-design.md`](../../docs/superpowers/specs/2026-06-30-slice-3-realtime-brain-design.md)).
## Why
Proves the slice-3 tap protocol extension is language-agnostic. A Python
script speaking JSON-via-WSS matches the OpenAI-Realtime-related portions of
the spec without depending on Rust code paths. Same rationale as slice-2's
Python echo brain — the project's `examples/` dir is the home for canonical
foreign-language brain demos.
The structure mirrors the Rust `rutster-brain-realtime` crate: a WS server
(the tap side the core dials into) bridged bidirectionally to a WS client
(the OpenAI Realtime side). The event mapping is the spec §4.2 table verbatim.
## Run
```bash
pip install -r requirements.txt
OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py
```
The Python brain binds the tap WS server on `RUTSTER_BRAIN_BIND` (default
`127.0.0.1:8082`). The rutster binary, started normally (`cargo run`), dials
out to `RUTSTER_TAP_URL` (default `ws://127.0.0.1:8082/realtime` — set
`RUTSTER_TAP_URL=ws://127.0.0.1:8082` to match the Python brain's bind).
Other env vars:
| Var | Default | Purpose |
|---|---|---|
| `OPENAI_API_KEY` | (required) | OpenAI auth |
| `RUTSTER_BRAIN_BIND` | `127.0.0.1:8082` | host:port for the tap WS server |
| `OPENAI_REALTIME_MODEL` | `gpt-4o-realtime` | OpenAI model query-param |
| `OPENAI_REALTIME_VOICE` | `alloy` | Voice passed in `session.update` |
## Dependencies
Only `websockets>=12.0` — both the tap-side server and the OpenAI-side client
speak plain WSS, so the `openai` Python SDK is not needed (the spec §4.2
mapping is explicit enough to hand-roll against the wire). This is a
deliberate simplification over the slice-3 plan skeleton, which imported the
SDK but never used it.
## Not in CI
Per AGENTS.md's "no Python in the dev loop" rule. The slice-3 integration
test uses `rutster-brain-realtime --features=mock` — the in-process Rust
mock — not this Python file.

View File

@@ -0,0 +1,195 @@
#!/usr/bin/env python3
"""OpenAI Realtime reference brain -- Python implementation (slice-3 spec §7.5).
Mirrors rutster-brain-realtime's WS-server + OpenAI-WS-client glue in
Python, proving the slice-3 tap protocol is language-agnostic. The mapping
is the §4.2 event table verbatim.
Run:
pip install -r requirements.txt
OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py
Not in CI (violates the zero-non-Rust-dev-deps dev loop per AGENTS.md).
"""
import asyncio
import json
import os
import sys
import websockets
PROTOCOL_VERSION = 1
SAMPLES = 480
# Configuration via env (matching the Rust binary's naming convention so the
# docs read the same on both sides).
RUTSTER_TAP_BIND = os.environ.get("RUTSTER_BRAIN_BIND", "127.0.0.1:8082")
OPENAI_MODEL = os.environ.get("OPENAI_REALTIME_MODEL", "gpt-4o-realtime")
OPENAI_VOICE = os.environ.get("OPENAI_REALTIME_VOICE", "alloy")
def openai_realtime_url(model: str) -> str:
"""wss://api.openai.com/v1/realtime?model=<model> (spec §4.2 + §5.3)."""
return f"wss://api.openai.com/v1/realtime?model={model}"
def openai_headers(api_key: str):
"""The two headers the OpenAI WS handshake requires (spec §5.3)."""
return [
("Authorization", f"Bearer {api_key}"),
("OpenAI-Beta", "realtime=v1"),
]
async def handle_tap_connection(tap_ws):
"""Bridge one rutster tap WS connection to one OpenAI Realtime session.
Spec §4.2 mapping table, in both directions. One tap connection == one
OpenAI Realtime session; stateless across reconnects (spec §5.3), exactly
like the Rust brain and the slice-2 echo brain pattern.
"""
api_key = os.environ.get("OPENAI_API_KEY")
if not api_key:
await tap_ws.close(code=1011, reason="OPENAI_API_KEY required")
return
# Open the OpenAI WS with the two required headers. additional_headers is
# the websockets>=12 spelling (was extra_headers in v11).
openai_ws = await websockets.connect(
openai_realtime_url(OPENAI_MODEL),
additional_headers=openai_headers(api_key),
)
# S4: send session.update with turn_detection: null. The FOB (the core)
# owns turn-taking; OpenAI's server-side VAD is explicitly disabled so it
# does not contradict core-authoritative playout decisions. See
# openai_client.rs for the Rust-side commentary on why this matters.
await openai_ws.send(json.dumps({
"type": "session.update",
"session": {
"modalities": ["text", "audio"],
"voice": OPENAI_VOICE,
"input_audio_format": "pcm16",
"output_audio_format": "pcm16",
"sample_rate": 24000,
"turn_detection": None,
},
}))
print(f"[openai_brain] session.update sent (turn_detection=null); "
f"model={OPENAI_MODEL} voice={OPENAI_VOICE}")
seq_egress = 0
def next_seq():
nonlocal seq_egress
s = seq_egress
seq_egress += 1
return s
async def tap_to_openai():
"""rutster tap events -> OpenAI Realtime events (spec §4.2 left column)."""
async for raw in tap_ws:
try:
env = json.loads(raw)
except json.JSONDecodeError as e:
await tap_ws.send(json.dumps({
"v": PROTOCOL_VERSION, "type": "error", "seq": next_seq(),
"ts": 0, "code": "decode_failed", "message": str(e),
}))
continue
t = env.get("type")
if t == "hello":
# Stateless across reconnects: ack hello, do NOT re-send
# session.update (already sent on this OpenAI WS session).
await tap_ws.send(json.dumps({
"v": PROTOCOL_VERSION, "type": "hello", "seq": next_seq(),
"ts": 0, "session_id": env.get("session_id", ""),
}))
elif t == "audio_in":
# Pass-through: same base64 PCM wire shape. No transcoding.
await openai_ws.send(json.dumps({
"type": "input_audio_buffer.append",
"audio": env["pcm"],
}))
elif t == "function_call_output":
# Core's tool-registry reply -> OpenAI conversation.item.create
# so the model can continue. Spec §4.2 row 5.
await openai_ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": env["id"],
"output": json.dumps(env.get("result", {})),
},
}))
elif t in ("bye", "session_end"):
await openai_ws.close()
return
# Unknown frame types: ignore (forward-compat via additive protocol).
async def openai_to_tap():
"""OpenAI Realtime events -> rutster tap events (spec §4.2 right column)."""
async for raw in openai_ws:
try:
event = json.loads(raw)
except json.JSONDecodeError:
continue
t = event.get("type")
if t == "response.audio.delta":
# Forward as tap audio_out. The delta is base64 PCM16; pass
# through. (Batching into 20 ms frames is a future
# optimization, spec §1.2; v1 passes each delta through.)
await tap_ws.send(json.dumps({
"v": PROTOCOL_VERSION, "type": "audio_out",
"seq": next_seq(), "ts": 0,
"pcm": event["delta"], "samples": SAMPLES,
}))
elif t == "input_audio_buffer.speech_started":
await tap_ws.send(json.dumps({
"v": PROTOCOL_VERSION, "type": "speech_started",
"seq": next_seq(), "ts": 0,
}))
elif t == "input_audio_buffer.speech_stopped":
await tap_ws.send(json.dumps({
"v": PROTOCOL_VERSION, "type": "speech_stopped",
"seq": next_seq(), "ts": 0,
}))
elif t == "response.function_call_arguments.done":
# Tool invocation from the model -> core's tool registry.
await tap_ws.send(json.dumps({
"v": PROTOCOL_VERSION, "type": "function_call",
"id": event["call_id"], "name": event["name"],
"args": json.loads(event.get("arguments", "{}")),
"seq": next_seq(), "ts": 0,
}))
elif t == "error":
# Surface OpenAI-side errors to the core for observation.
await tap_ws.send(json.dumps({
"v": PROTOCOL_VERSION, "type": "error",
"seq": next_seq(), "ts": 0,
"code": "openai_error",
"message": event.get("error", {}).get("message", "unknown"),
}))
try:
await asyncio.gather(tap_to_openai(), openai_to_tap())
finally:
await openai_ws.close()
async def main():
api_key = os.environ.get("OPENAI_API_KEY")
if not api_key:
sys.exit("OPENAI_API_KEY required (see README.md)")
host, port = RUTSTER_TAP_BIND.split(":")
print(f"[openai_brain] listening on ws://{RUTSTER_TAP_BIND} "
f"(OpenAI model={OPENAI_MODEL} voice={OPENAI_VOICE})")
async with websockets.serve(handle_tap_connection, host, int(port)):
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -0,0 +1 @@
websockets>=12.0