Slice 3 — OpenAI Realtime brain: swap echo for the brain (#4)
This commit was merged in pull request #4.
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -19,6 +19,10 @@
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# Python (examples/ Python brains are not in CI; ignore bytecode)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Local worktrees (per using-git-worktrees skill; isolation, not content)
|
||||
.worktrees/
|
||||
worktrees/
|
||||
|
||||
55
Cargo.lock
generated
55
Cargo.lock
generated
@@ -781,6 +781,12 @@ version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
@@ -1144,6 +1150,19 @@ dependencies = [
|
||||
"nom 7.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.14.1"
|
||||
@@ -1163,9 +1182,11 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||
name = "rutster"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"dashmap",
|
||||
"futures-util",
|
||||
"rutster-brain-realtime",
|
||||
"rutster-call-model",
|
||||
"rutster-media",
|
||||
"rutster-tap",
|
||||
@@ -1182,6 +1203,27 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rutster-brain-realtime"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"futures-util",
|
||||
"http",
|
||||
"rutster-media",
|
||||
"rutster-tap",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rutster-call-model"
|
||||
version = "0.0.0"
|
||||
@@ -1498,6 +1540,19 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.69"
|
||||
|
||||
@@ -11,6 +11,7 @@ members = [
|
||||
"crates/rutster-trunk",
|
||||
"crates/rutster-tap",
|
||||
"crates/rutster-tap-echo",
|
||||
"crates/rutster-brain-realtime",
|
||||
"crates/rutster-spend",
|
||||
]
|
||||
|
||||
@@ -51,3 +52,10 @@ futures-util = "0.3"
|
||||
url = "2"
|
||||
# base64 0.22: PCM <-> base64 codec for the v1 wire format (spec §3).
|
||||
base64 = "0.22"
|
||||
# async-trait 0.1: async fns in trait objects (Tool trait, slice-3 spec §6.1).
|
||||
async-trait = "0.1"
|
||||
# http 1: Request/Response builders for the OpenAI WS handshake (slice-3
|
||||
# spec §5.3). Task 9's main.rs builds the http::Request<...> the tungstenite
|
||||
# client sends before WS upgrade; `openai_client::openai_headers` produces
|
||||
# the header pairs the caller stuffs into it.
|
||||
http = "1"
|
||||
|
||||
27
LEARNING.md
27
LEARNING.md
@@ -79,6 +79,33 @@ can read in `cargo doc --open` plus the source file itself.
|
||||
— `TapHandle(())` compiles `Option<TapHandle>` to a single `bool`; no
|
||||
runtime cost for the type-system marker.
|
||||
|
||||
### Slice 3 — OpenAI Realtime brain
|
||||
|
||||
- **`async-trait` patterns / async fns in trait objects** →
|
||||
`crates/rutster/src/tool_registry.rs` — the `Tool` trait's `async fn call`
|
||||
and how `async-trait` lowers it to `Pin<Box<dyn Future + Send>>` (the same
|
||||
boxing the slice-1 `pipe` widening used, now applied to trait methods).
|
||||
- **OpenAI Realtime adapter + event translation (pure-function layer)** →
|
||||
`crates/rutster-brain-realtime/src/translator.rs` — the §4.2 event mapping
|
||||
table as pure `fn`s: tap `audio_in` ⇄ OpenAI `input_audio_buffer.append`,
|
||||
`response.audio.delta` ⇄ `audio_out`, `function_call_arguments.done` ⇄
|
||||
`function_call`. Testable without a network (the pump is in a separate file).
|
||||
- **Tap protocol additive extension + forward-compat via `#[serde(other)]`** →
|
||||
`crates/rutster-tap/src/protocol.rs` — how slice-3 adds `speech_started`,
|
||||
`speech_stopped`, `function_call`, `function_call_output`, `tools.update`
|
||||
without breaking slice-2 brains: unknown variants deserialize to a catch-all
|
||||
arm instead of erroring (forward-compat by construction).
|
||||
- **Side-channel mpsc for FOB-boundary dispatch** →
|
||||
`crates/rutster/src/session_map.rs` — how `drive_all_sessions` drains a
|
||||
function-call mpsc side channel into the tap WS writer without blocking the
|
||||
20 ms media loop. The pattern: hot loop polls `try_recv`; cold path spawns.
|
||||
- **HTTP request builder for WS subprotocol handshake** →
|
||||
`crates/rutster-brain-realtime/src/openai_client.rs` — why we hand-roll the
|
||||
WS upgrade `Request` instead of letting `connect_async` build it: to set
|
||||
`Authorization` + `OpenAI-Beta` headers on the WS handshake (tungstenite
|
||||
doesn't expose subprotocol/auth headers via the simple `connect_async(url)`
|
||||
entry point — see `openai_headers` / `openai_realtime_url`).
|
||||
|
||||
## How to read
|
||||
|
||||
1. `cargo doc --open` — every module has a `//!` doc comment; the doc
|
||||
|
||||
32
README.md
32
README.md
@@ -19,6 +19,38 @@ cargo run
|
||||
Open <http://localhost:8080/> → click "Start call" → grant mic → hear yourself echo.
|
||||
Full walkthrough + troubleshooting: **[`docs/QUICKSTART.md`](docs/QUICKSTART.md)**.
|
||||
|
||||
### Slice 3 dev loop — OpenAI Realtime brain
|
||||
|
||||
The dev loop *without* real OpenAI credentials (no API key required):
|
||||
|
||||
```
|
||||
cargo run -p rutster-brain-realtime --features=mock # brain on :8082
|
||||
cargo run # core on :8080
|
||||
```
|
||||
|
||||
Open <http://localhost:8080/> → click "Start call" → speak → hear the
|
||||
mock-brain reply within ~250 ms (the mock echoes audio back, no real OpenAI
|
||||
RTT; this exercises the full brain→core audio round-trip + the new
|
||||
function_call dispatch path).
|
||||
|
||||
With real OpenAI Realtime:
|
||||
|
||||
```
|
||||
export OPENAI_API_KEY=sk-... # or OPENAI_API_KEY_FILE=/var/secrets/openai
|
||||
cargo run -p rutster-brain-realtime
|
||||
cargo run
|
||||
```
|
||||
|
||||
Speak → end-to-end speech-to-speech with OpenAI Realtime within ~700 ms
|
||||
(slice-1's 200 ms + tap round-trip + OpenAI latency + 100 ms playout buffer).
|
||||
|
||||
For the foreign-language brain demo (Python, not in CI):
|
||||
|
||||
```
|
||||
pip install -r examples/openai_realtime_brain/requirements.txt
|
||||
OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py
|
||||
```
|
||||
|
||||
> **Status:** Slice 1 (WebRTC media loopback) is the active build target. The workspace is
|
||||
> landing task-by-task on the `slice-1-webrtc-loopback` branch. Design:
|
||||
> [`docs/superpowers/specs/2026-06-28-slice-1-webrtc-loopback-design.md`](docs/superpowers/specs/2026-06-28-slice-1-webrtc-loopback-design.md).
|
||||
|
||||
35
crates/rutster-brain-realtime/Cargo.toml
Normal file
35
crates/rutster-brain-realtime/Cargo.toml
Normal file
@@ -0,0 +1,35 @@
|
||||
# crates/rutster-brain-realtime/Cargo.toml
|
||||
[package]
|
||||
name = "rutster-brain-realtime"
|
||||
version = "0.1.0"
|
||||
license.workspace = true
|
||||
edition.workspace = true
|
||||
repository.workspace = true
|
||||
description = "OpenAI Realtime speech-to-speech brain — translates slice-2's tap protocol to OpenAI Realtime's event schema (green-zone per ADR-0008; slice-3 spec §1.1, §4)."
|
||||
|
||||
[dependencies]
|
||||
rutster-tap = { path = "../rutster-tap" }
|
||||
rutster-media = { path = "../rutster-media" } # for PcmFrame in tests (spec §4.2 audio round-trip)
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tokio-tungstenite = { workspace = true, features = ["connect"] }
|
||||
futures-util = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
url = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
# http: Request/Response builder for the WS handshake (Task 9's main.rs).
|
||||
# This task's unit tests assert URL + headers shape only.
|
||||
http = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Mock mode: in-process fake OpenAI Realtime WS server (no real API calls).
|
||||
# Used by the integration test + the offline dev loop (spec §7.3).
|
||||
mock = []
|
||||
199
crates/rutster-brain-realtime/src/api_key.rs
Normal file
199
crates/rutster-brain-realtime/src/api_key.rs
Normal file
@@ -0,0 +1,199 @@
|
||||
//! # API-key loader (spec §5.3)
|
||||
//!
|
||||
//! Two-source config: `OPENAI_API_KEY` env var default + optional
|
||||
//! `OPENAI_API_KEY_FILE` path override. KMS/Vault integration is deferred
|
||||
//! to step 6 (spec §1.2); the file-path override makes secret-manager
|
||||
//! injection (k8s secrets, Vault agent) trivial when that layer exists.
|
||||
//!
|
||||
//! # Why file-path override (not env-only)
|
||||
//!
|
||||
//! A k8s pod mounts secrets at file paths (e.g. `/var/secrets/openai_key`).
|
||||
//! Env-var secrets are also fine, but file-path is the standard pattern for
|
||||
//! k8s + Vault agent + sidecar secret rotators. The dev loop uses the env
|
||||
//! var path (single-source, simplest).
|
||||
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ApiKeyError {
|
||||
#[error("no API key: set OPENAI_API_KEY or OPENAI_API_KEY_FILE")]
|
||||
NotFound,
|
||||
#[error("OPENAI_API_KEY_FILE path is not valid UTF-8: {0:?}")]
|
||||
PathNotUtf8(PathBuf),
|
||||
#[error("failed to read OPENAI_API_KEY_FILE at {path}: {source}")]
|
||||
FileRead {
|
||||
path: PathBuf,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
}
|
||||
|
||||
/// Load the OpenAI API key per the env-var + file-path posture (spec §5.3).
|
||||
///
|
||||
/// Precedence: `OPENAI_API_KEY_FILE` wins over `OPENAI_API_KEY` (a file
|
||||
/// path is a more specific override; this matches k8s-secret patterns where
|
||||
/// an operator mounts a file to override the default env var).
|
||||
///
|
||||
/// Trims trailing whitespace from either source (some k8s secret mounts
|
||||
/// add trailing newlines).
|
||||
pub fn load_api_key() -> Result<String, ApiKeyError> {
|
||||
if let Ok(path_str) = env::var("OPENAI_API_KEY_FILE") {
|
||||
let path = PathBuf::from(&path_str);
|
||||
let raw = fs::read_to_string(&path).map_err(|source| ApiKeyError::FileRead {
|
||||
path: path.clone(),
|
||||
source,
|
||||
})?;
|
||||
return Ok(raw.trim().to_string());
|
||||
}
|
||||
if let Ok(raw) = env::var("OPENAI_API_KEY") {
|
||||
return Ok(raw.trim().to_string());
|
||||
}
|
||||
Err(ApiKeyError::NotFound)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// NOTE: edition 2024 makes env::set_var / env::remove_var `unsafe` (they
|
||||
// can race with concurrent reads of the process environ). The brief was
|
||||
// written against edition 2021 syntax; we wrap each mutation in
|
||||
// `unsafe { ... }` rather than reverting to a pre-2024 edition. The
|
||||
// unsafety is local to the test process and accepted by the stdlib's
|
||||
// own test helpers; no threads here read these env vars concurrently.
|
||||
|
||||
// The process environ is process-global state. Even with the save/restore
|
||||
// pattern below, running these tests in parallel would let one test's
|
||||
// `set_var` race against another's `remove_var` for the same variable,
|
||||
// producing flaky failures. This mutex serializes every test in this
|
||||
// module so the env-var mutations are atomic across tests.
|
||||
static ENV_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
/// Acquire the env mutex for the test body. Panics if the lock is held
|
||||
/// (only happens if a test panics while held — fine, fail fast).
|
||||
fn lock_env() -> std::sync::MutexGuard<'static, ()> {
|
||||
ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_error_when_neither_env_nor_file_set() {
|
||||
let _guard = lock_env();
|
||||
// Save then clear both env vars to make the test order-independent.
|
||||
let saved_key = env::var_os("OPENAI_API_KEY").map(|v| ("OPENAI_API_KEY".to_string(), v));
|
||||
let saved_file =
|
||||
env::var_os("OPENAI_API_KEY_FILE").map(|v| ("OPENAI_API_KEY_FILE".to_string(), v));
|
||||
unsafe {
|
||||
env::remove_var("OPENAI_API_KEY");
|
||||
env::remove_var("OPENAI_API_KEY_FILE");
|
||||
}
|
||||
|
||||
let r = load_api_key();
|
||||
assert!(matches!(r, Err(ApiKeyError::NotFound)));
|
||||
|
||||
// Restore for any other test in the same process.
|
||||
if let Some((k, v)) = saved_key {
|
||||
unsafe {
|
||||
env::set_var(k, v);
|
||||
}
|
||||
}
|
||||
if let Some((k, v)) = saved_file {
|
||||
unsafe {
|
||||
env::set_var(k, v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_from_env_var() {
|
||||
let _guard = lock_env();
|
||||
let saved = env::var_os("OPENAI_API_KEY").map(|v| ("OPENAI_API_KEY".to_string(), v));
|
||||
let saved_file =
|
||||
env::var_os("OPENAI_API_KEY_FILE").map(|v| ("OPENAI_API_KEY_FILE".to_string(), v));
|
||||
unsafe {
|
||||
env::set_var("OPENAI_API_KEY", "sk-test-12345");
|
||||
env::remove_var("OPENAI_API_KEY_FILE");
|
||||
}
|
||||
|
||||
let r = load_api_key().unwrap();
|
||||
assert_eq!(r, "sk-test-12345");
|
||||
|
||||
if let Some((k, v)) = saved {
|
||||
unsafe {
|
||||
env::set_var(k, v);
|
||||
}
|
||||
} else {
|
||||
unsafe {
|
||||
env::remove_var("OPENAI_API_KEY");
|
||||
}
|
||||
}
|
||||
if let Some((k, v)) = saved_file {
|
||||
unsafe {
|
||||
env::set_var(k, v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trim_trailing_newline_from_env() {
|
||||
let _guard = lock_env();
|
||||
let saved = env::var_os("OPENAI_API_KEY").map(|v| ("OPENAI_API_KEY".to_string(), v));
|
||||
unsafe {
|
||||
env::set_var("OPENAI_API_KEY", "sk-test-with-newline\n");
|
||||
env::remove_var("OPENAI_API_KEY_FILE");
|
||||
}
|
||||
|
||||
let r = load_api_key().unwrap();
|
||||
assert_eq!(r, "sk-test-with-newline");
|
||||
|
||||
if let Some((k, v)) = saved {
|
||||
unsafe {
|
||||
env::set_var(k, v);
|
||||
}
|
||||
} else {
|
||||
unsafe {
|
||||
env::remove_var("OPENAI_API_KEY");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_path_overrides_env() {
|
||||
let _guard = lock_env();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("key.txt");
|
||||
std::fs::write(&path, "sk-from-file-98765\n").unwrap();
|
||||
|
||||
unsafe {
|
||||
env::set_var("OPENAI_API_KEY", "sk-from-env-should-be-overridden");
|
||||
env::set_var("OPENAI_API_KEY_FILE", path.to_str().unwrap());
|
||||
}
|
||||
|
||||
let r = load_api_key().unwrap();
|
||||
assert_eq!(r, "sk-from-file-98765");
|
||||
|
||||
unsafe {
|
||||
env::remove_var("OPENAI_API_KEY");
|
||||
env::remove_var("OPENAI_API_KEY_FILE");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_path_missing_returns_file_read_error() {
|
||||
let _guard = lock_env();
|
||||
unsafe {
|
||||
env::set_var("OPENAI_API_KEY_FILE", "/nonexistent/path/to/key.txt");
|
||||
env::remove_var("OPENAI_API_KEY");
|
||||
}
|
||||
|
||||
let r = load_api_key();
|
||||
assert!(matches!(r, Err(ApiKeyError::FileRead { .. })));
|
||||
|
||||
unsafe {
|
||||
env::remove_var("OPENAI_API_KEY_FILE");
|
||||
}
|
||||
}
|
||||
}
|
||||
37
crates/rutster-brain-realtime/src/lib.rs
Normal file
37
crates/rutster-brain-realtime/src/lib.rs
Normal file
@@ -0,0 +1,37 @@
|
||||
//! # rutster-brain-realtime
|
||||
//!
|
||||
//! **Slice-3 brain:** translates slice-2's tap protocol (the green-zone side of
|
||||
//! the seam — core-as-client dials this brain's WS server; ADR-0008 classifies
|
||||
//! the brain as green-zone) to OpenAI Realtime's event schema. The brain
|
||||
//! process is a WS *server* (core-as-client dials it, unchanged from slice-2)
|
||||
//! AND a WS *client* to `wss://api.openai.com/v1/realtime` (OpenAI is server
|
||||
//! on *its* leg).
|
||||
//!
|
||||
//! See `docs/superpowers/specs/2026-06-30-slice-3-realtime-brain-design.md` for
|
||||
//! the full design.
|
||||
//!
|
||||
//! ## Modules
|
||||
//!
|
||||
//! - [`api_key`] — load API key from env var or file (spec §5.3).
|
||||
//! - [`translator`] — pure-function event translation (tap ⇄ OpenAI).
|
||||
//! - [`openai_client`] — wss:// client to api.openai.com/v1/realtime.
|
||||
//!
|
||||
//! ## Dev mode (`--features=mock`)
|
||||
//!
|
||||
//! When built with `mock`, the binary uses in-process `MockRealtimeBrain`
|
||||
//! (defined in `lib.rs`'s test-support module) instead of dialing OpenAI.
|
||||
//! No API key required, no real OpenAI calls. Used by the integration test +
|
||||
//! the offline dev loop (spec §7.3).
|
||||
|
||||
pub mod api_key;
|
||||
pub mod openai_client;
|
||||
pub mod translator;
|
||||
|
||||
/// slice-3 §5.3 dev mode: in-process fake OpenAI Realtime WS server. Only
|
||||
/// available with `--features=mock`. The binary uses it to run the offline
|
||||
/// dev loop (no API key, no network calls to OpenAI); the integration test
|
||||
/// uses it for the same reason.
|
||||
#[cfg(feature = "mock")]
|
||||
pub mod mock;
|
||||
#[cfg(feature = "mock")]
|
||||
pub use mock::MockRealtimeBrain;
|
||||
212
crates/rutster-brain-realtime/src/main.rs
Normal file
212
crates/rutster-brain-realtime/src/main.rs
Normal file
@@ -0,0 +1,212 @@
|
||||
//! # rutster-brain-realtime binary (slice-3 spec §4.2 + §5.3)
|
||||
//!
|
||||
//! Standalone brain process the core dials out to via `RUTSTER_TAP_URL` (spec
|
||||
//! §5.1). One process bridges many tap WS connections (one per call) to many
|
||||
//! OpenAI Realtime sessions.
|
||||
//!
|
||||
//! # Modes
|
||||
//!
|
||||
//! - **`--features=mock`** (offline dev loop, slice-3 spec §7.3): starts an
|
||||
//! in-process `MockRealtimeBrain` WS server, dials it as the "OpenAI side."
|
||||
//! No API key, no network calls to OpenAI. Used by the integration test +
|
||||
//! the offline dev loop (`cargo run -p rutster-brain-realtime --features=mock`).
|
||||
//! - **Default** (real OpenAI): loads `OPENAI_API_KEY` per
|
||||
//! [`api_key::load_api_key`], dials `wss://api.openai.com/v1/realtime?model=...`.
|
||||
//!
|
||||
//! Both modes share the same tap-side WS server + the same `run_openai_pump`
|
||||
//! bridging logic — only the OpenAI-side dialer differs.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
#[cfg(feature = "mock")]
|
||||
use rutster_brain_realtime::MockRealtimeBrain;
|
||||
#[cfg(not(feature = "mock"))]
|
||||
use rutster_brain_realtime::api_key;
|
||||
use rutster_brain_realtime::openai_client;
|
||||
use rutster_brain_realtime::translator::build_openai_session_update;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tracing::{info, warn};
|
||||
|
||||
/// Default bind addr for the brain process's tap-side WS server (spec §5.3).
|
||||
/// Distinct from slice-2's echo brain (`:8081/echo`); the two coexist.
|
||||
const DEFAULT_TAP_BIND: &str = "127.0.0.1:8082";
|
||||
const DEFAULT_VOICE: &str = "alloy";
|
||||
const DEFAULT_MODEL: &str = "gpt-4o-realtime";
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| "rutster_brain_realtime=info".into()),
|
||||
)
|
||||
.init();
|
||||
|
||||
let tap_bind: SocketAddr = std::env::var("RUTSTER_TAP_BIND")
|
||||
.unwrap_or_else(|_| DEFAULT_TAP_BIND.to_string())
|
||||
.parse()
|
||||
.expect("RUTSTER_TAP_BIND must be a valid SocketAddr");
|
||||
let voice =
|
||||
std::env::var("OPENAI_REALTIME_VOICE").unwrap_or_else(|_| DEFAULT_VOICE.to_string());
|
||||
let model =
|
||||
std::env::var("OPENAI_REALTIME_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string());
|
||||
let _ = &model; // used only in the non-mock branch below.
|
||||
|
||||
#[cfg(feature = "mock")]
|
||||
let openai_url: String;
|
||||
#[cfg(feature = "mock")]
|
||||
let openai_headers: Vec<(String, String)>;
|
||||
#[cfg(feature = "mock")]
|
||||
let _mock_guard;
|
||||
#[cfg(not(feature = "mock"))]
|
||||
let (openai_url, openai_headers) = {
|
||||
let api_key = match api_key::load_api_key() {
|
||||
Ok(k) => k,
|
||||
Err(e) => {
|
||||
eprintln!("failed to load OPENAI_API_KEY: {e}; refusing to start");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
(
|
||||
openai_client::openai_realtime_url(&model),
|
||||
openai_client::openai_headers(&api_key),
|
||||
)
|
||||
};
|
||||
#[cfg(feature = "mock")]
|
||||
{
|
||||
let mock = MockRealtimeBrain::start().await.expect("mock bind ok");
|
||||
info!(mock_addr = %mock.addr, "MockRealtimeBrain started (--features=mock)");
|
||||
openai_url = mock.url();
|
||||
openai_headers = Vec::new();
|
||||
_mock_guard = mock;
|
||||
}
|
||||
let listener = tokio::net::TcpListener::bind(tap_bind)
|
||||
.await
|
||||
.expect("bind ok");
|
||||
info!(%tap_bind, %openai_url, "rutster-brain-realtime listening (core dials this)");
|
||||
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, peer)) => {
|
||||
let url = openai_url.clone();
|
||||
let hdrs = openai_headers.clone();
|
||||
let voice = voice.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = handle_tap_connection(stream, peer, &url, &hdrs, &voice).await {
|
||||
warn!(error = ?e, %peer, "tap connection ended with error");
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(e) => warn!(error = %e, "accept failed; continuing"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle one tap WS connection from the core. Spawns two mpsc forwarders
|
||||
/// (tap→pump, pump→tap) + dials the OpenAI side, then runs `run_openai_pump`
|
||||
/// on the OpenAI side + awaits both forwarder tasks.
|
||||
///
|
||||
/// The tap-side handshake (hello/session_end) is handled here; the OpenAI
|
||||
/// pump sees only audio_in + function_call_output on its inbound + emits
|
||||
/// only audio_out/speech_*/function_call/error on its outbound.
|
||||
async fn handle_tap_connection(
|
||||
stream: tokio::net::TcpStream,
|
||||
peer: SocketAddr,
|
||||
openai_url: &str,
|
||||
openai_headers: &[(String, String)],
|
||||
voice: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut tap_ws = tokio_tungstenite::accept_async(stream).await?;
|
||||
info!(%peer, "tap WS connection accepted");
|
||||
|
||||
// === Tap handshake: first frame must be hello; send hello ack. ===
|
||||
let hello_in = tap_ws
|
||||
.next()
|
||||
.await
|
||||
.ok_or("tap connection closed before hello")??;
|
||||
let hello_text = hello_in.into_text().map_err(|_| "hello not text")?;
|
||||
let decoded = rutster_tap::decode_envelope(&hello_text)?;
|
||||
let session_id = match decoded.payload {
|
||||
rutster_tap::DecodedPayload::Hello(p) => p.session_id,
|
||||
_ => return Err("first tap frame not hello".into()),
|
||||
};
|
||||
info!(%peer, %session_id, "tap hello received");
|
||||
let ack = rutster_tap::encode_hello(&session_id, 0, 0)?;
|
||||
tap_ws.send(Message::Text(ack)).await?;
|
||||
|
||||
// === Bridge channels between tap WS and OpenAI pump. ===
|
||||
// tap_via (Sender): the core's tap frames flow IN here — the forwarder
|
||||
// reads text from tap_ws and forwards via this channel. The OpenAI
|
||||
// pump's `tap_rx: mpsc::Receiver<String>` is the receiving end.
|
||||
// pump_tap_tx (Sender): the pump's outbound tap frames (audio_out,
|
||||
// speech_started, function_call, error) flow OUT here. The forwarder
|
||||
// reads from the paired Receiver and writes to tap_ws.
|
||||
let (tap_via, pump_tap_rx) = mpsc::channel::<String>(64);
|
||||
let (pump_tap_tx, mut tap_out_rx) = mpsc::channel::<String>(64);
|
||||
|
||||
// Split the tap WS into sink + stream so the two forwarder tasks can
|
||||
// own each half independently. `tokio_tungstenite::WebSocketStream::split`
|
||||
// yields `(SplitSink<..., Message>, SplitStream<...>)` — the canonical
|
||||
// tokio pattern for fan-out into two parallel async tasks.
|
||||
let (mut tap_sink, mut tap_stream) = tap_ws.split();
|
||||
|
||||
// === Forwarder: tap_ws_stream → pump_tap_rx ===
|
||||
let in_fwd = tokio::spawn(async move {
|
||||
while let Some(msg_res) = tap_stream.next().await {
|
||||
match msg_res {
|
||||
Ok(m) => {
|
||||
if let Ok(text) = m.into_text() {
|
||||
if tap_via.send(text).await.is_err() {
|
||||
break; // pump dropped
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "tap_ws recv error");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// === Forwarder: pump_tap_tx → tap_ws_sink ===
|
||||
let out_fwd = tokio::spawn(async move {
|
||||
while let Some(text) = tap_out_rx.recv().await {
|
||||
if let Err(e) = tap_sink.send(Message::Text(text)).await {
|
||||
warn!(error = ?e, "tap_ws send error");
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// === Dial the OpenAI side (real or mock). ===
|
||||
let request = openai_url.into_client_request()?;
|
||||
let mut request = request;
|
||||
for (k, v) in openai_headers {
|
||||
request.headers_mut().insert(
|
||||
http::HeaderName::from_bytes(k.as_bytes())?,
|
||||
http::HeaderValue::from_str(v)?,
|
||||
);
|
||||
}
|
||||
let (openai_ws, _resp) = tokio_tungstenite::connect_async(request).await?;
|
||||
info!(%peer, %openai_url, "OpenAI side connected");
|
||||
|
||||
// `run_openai_pump` sends its own session.update on startup per
|
||||
// `build_openai_session_update`; don't pre-send here.
|
||||
let _ = build_openai_session_update;
|
||||
info!(%peer, voice = %voice, "starting OpenAI pump");
|
||||
|
||||
let pump_result =
|
||||
openai_client::run_openai_pump(openai_ws, pump_tap_rx, pump_tap_tx, voice.to_string())
|
||||
.await;
|
||||
match pump_result {
|
||||
Ok(()) => info!(%peer, "OpenAI pump exited cleanly"),
|
||||
Err(e) => warn!(%peer, error = ?e, "OpenAI pump exited with error"),
|
||||
}
|
||||
in_fwd.abort();
|
||||
out_fwd.abort();
|
||||
Ok(())
|
||||
}
|
||||
328
crates/rutster-brain-realtime/src/mock.rs
Normal file
328
crates/rutster-brain-realtime/src/mock.rs
Normal file
@@ -0,0 +1,328 @@
|
||||
//! # MockRealtimeBrain — in-process fake OpenAI Realtime WS server (spec §5.3)
|
||||
//!
|
||||
//! Behind `--features=mock` only. Runs entirely in-process; no network calls
|
||||
//! to OpenAI, no API key required. Powers:
|
||||
//! - the offline dev loop (`cargo run -p rutster-brain-realtime --features=mock`
|
||||
//! starts the brain process with this mock as the OpenAI side), and
|
||||
//! - slice-3's integration test (no real credentials, deterministic canned
|
||||
//! responses).
|
||||
//!
|
||||
//! # Contract
|
||||
//!
|
||||
//! - Accepts a WS client connection (the brain process's OpenAI-side client).
|
||||
//! - Asserts that the first event the client sends is a `session.update`
|
||||
//! with `turn_detection: null` (the S4 load-bearing decision — spec §4.3).
|
||||
//! - On each `input_audio_buffer.append` event the client sends, replies
|
||||
//! with a canned `response.audio.delta` event carrying base64 LE i16 PCM
|
||||
//! (480 zeroed samples per 20 ms frame — same wire shape as slice-2's
|
||||
//! audio_out, so the brain's translator round-trips through the same
|
||||
//! PcmFrame codec real OpenAI would use).
|
||||
//! - Echoes back `input_audio_buffer.speech_started` / `.speech_stopped`
|
||||
//! so the brain's translator can forward them as advisory tap events
|
||||
//! (spec §3.2 — these are advisory in slice-3; step 4 wires the FOB
|
||||
//! reflex loop).
|
||||
//! - On `response.function_call_arguments.done`... the brain's translator
|
||||
//! consumes those (OpenAI emits them, the brain forwards to core); the
|
||||
//! mock doesn't generate unprompted function_call events (the integration
|
||||
//! test exercises the function-call round-trip via a deterministic test
|
||||
//! hook, not via the mock's own drive).
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// The handle returned by `MockRealtimeBrain::start`. Drop or call `shutdown`
|
||||
/// to stop the mock server + abort its accept-loop task.
|
||||
pub struct MockRealtimeBrain {
|
||||
/// The address the mock bound. The brain process dials this URL
|
||||
/// (`ws://<addr>/`) instead of the real `wss://api.openai.com/...`
|
||||
/// when running with `--features=mock`.
|
||||
pub addr: SocketAddr,
|
||||
pub shutdown: Option<oneshot::Sender<()>>,
|
||||
pub join: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl MockRealtimeBrain {
|
||||
/// Start the mock on an ephemeral port (caller picks up `addr` for the
|
||||
/// brain process to dial). The accept loop spawns per-connection tasks
|
||||
/// so multiple sessions can run concurrently in the integration test.
|
||||
pub async fn start() -> std::io::Result<Self> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let addr = listener.local_addr()?;
|
||||
let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
|
||||
let join = tokio::spawn(accept_loop(listener, shutdown_rx));
|
||||
info!(%addr, "MockRealtimeBrain listening (fake OpenAI Realtime)");
|
||||
Ok(Self {
|
||||
addr,
|
||||
shutdown: Some(shutdown_tx),
|
||||
join,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build the URL the brain process's OpenAI client should dial instead
|
||||
/// of `wss://api.openai.com/v1/realtime?model=...`. The path is `/v1/realtime`
|
||||
/// so the brain client's `into_client_request` call works the same as it
|
||||
/// does against the real OpenAI endpoint.
|
||||
pub fn url(&self) -> String {
|
||||
format!("ws://{}/v1/realtime?model=mock", self.addr)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MockRealtimeBrain {
|
||||
fn drop(&mut self) {
|
||||
// Best-effort shutdown signal — the task may already have exited.
|
||||
// `Option::take()` lets us send by-value without moving out of
|
||||
// `self` (Drop takes `&mut self`, not `self`).
|
||||
if let Some(tx) = self.shutdown.take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
self.join.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async fn accept_loop(listener: TcpListener, mut shutdown: oneshot::Receiver<()>) {
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut shutdown => {
|
||||
info!("MockRealtimeBrain accept loop shutting down");
|
||||
return;
|
||||
}
|
||||
res = listener.accept() => {
|
||||
let (stream, peer) = match res {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "MockRealtimeBrain accept failed; continuing");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
tokio::spawn(handle_connection(stream, peer));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle one brain-process → mock-OpenAI WS connection. Each connection is
|
||||
/// one OpenAI Realtime session (stateless across reconnects — same as the
|
||||
/// brain process's own contract for the tap side).
|
||||
async fn handle_connection(stream: tokio::net::TcpStream, peer: SocketAddr) {
|
||||
let ws = tokio_tungstenite::accept_async(stream).await;
|
||||
let Ok(mut ws) = ws else {
|
||||
warn!(%peer, "MockRealtimeBrain WS handshake failed");
|
||||
return;
|
||||
};
|
||||
debug!(%peer, "MockRealtimeBrain connection accepted");
|
||||
|
||||
// First event MUST be session.update with turn_detection: null.
|
||||
// Wait for it (bounded 2s — the brain process sends it immediately
|
||||
// after the WS handshake per `run_openai_pump`).
|
||||
let first = tokio::time::timeout(std::time::Duration::from_secs(2), ws.next()).await;
|
||||
match first {
|
||||
Ok(Some(Ok(msg))) => {
|
||||
let Ok(text) = msg.into_text() else {
|
||||
warn!(%peer, "MockRealtimeBrain first frame not text; closing");
|
||||
let _ = ws.close(None).await;
|
||||
return;
|
||||
};
|
||||
let event: Value = match serde_json::from_str(&text) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!(%peer, error = ?e, "MockRealtimeBrain first frame not JSON; closing");
|
||||
let _ = ws.close(None).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
if event.get("type").and_then(|v| v.as_str()) != Some("session.update") {
|
||||
warn!(%peer, "MockRealtimeBrain first event not session.update; closing");
|
||||
let _ = ws.close(None).await;
|
||||
return;
|
||||
}
|
||||
// S4 load-bearing assertion (spec §4.3). Do NOT silently accept
|
||||
// a session.update that has turn_detection != null — that would
|
||||
// mask a regression in the brain's translator.
|
||||
let td = event.pointer("/session/turn_detection");
|
||||
if td != Some(&Value::Null) {
|
||||
warn!(%peer, td = ?td, "MockRealtimeBrain S4 violation: turn_detection != null; closing");
|
||||
// Echo back an OpenAI-shaped error so the brain surfaces it.
|
||||
let err = json!({
|
||||
"type": "error",
|
||||
"error": {
|
||||
"code": "invalid_session_update",
|
||||
"message": "MockRealtimeBrain: turn_detection must be null (S4)"
|
||||
}
|
||||
});
|
||||
let _ = ws.send(Message::Text(err.to_string())).await;
|
||||
let _ = ws.close(None).await;
|
||||
return;
|
||||
}
|
||||
info!(%peer, "MockRealtimeBrain session.update accepted (turn_detection=null)");
|
||||
}
|
||||
_ => {
|
||||
warn!(%peer, "MockRealtimeBrain no session.update within 2s; closing");
|
||||
let _ = ws.close(None).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Pump loop: reply to input_audio_buffer.append with canned audio.
|
||||
while let Some(msg_res) = ws.next().await {
|
||||
let msg = match msg_res {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
debug!(%peer, error = ?e, "MockRealtimeBrain WS recv error; closing");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let Ok(text) = msg.into_text() else { continue };
|
||||
let event: Value = match serde_json::from_str(&text) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
debug!(%peer, error = ?e, "MockRealtimeBrain non-JSON event; ignoring");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let event_type = event.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
match event_type {
|
||||
"input_audio_buffer.append" => {
|
||||
// Canned response: one `response.audio.delta` per append
|
||||
// carrying 480 zeroed samples as base64 LE i16 (the same
|
||||
// wire shape as slice-2's audio_out — verified by the
|
||||
// translator's existing round-trip test). Identical bytes
|
||||
// every time: deterministic for test assertions.
|
||||
use base64::Engine;
|
||||
let pcm_zeros = vec![0u8; 960]; // 480 i16 samples × 2 bytes
|
||||
let pcm_b64 = base64::engine::general_purpose::STANDARD.encode(&pcm_zeros);
|
||||
let delta = json!({
|
||||
"type": "response.audio.delta",
|
||||
"delta": pcm_b64
|
||||
});
|
||||
if let Err(e) = ws.send(Message::Text(delta.to_string())).await {
|
||||
debug!(%peer, error = ?e, "MockRealtimeBrain send delta failed; closing");
|
||||
return;
|
||||
}
|
||||
}
|
||||
"input_audio_buffer.speech_started" => {
|
||||
// Pass-through echo so the brain's translator forwards it as
|
||||
// a tap `speech_started` advisory event.
|
||||
let evt = json!({ "type": "input_audio_buffer.speech_started" });
|
||||
let _ = ws.send(Message::Text(evt.to_string())).await;
|
||||
}
|
||||
"input_audio_buffer.speech_stopped" => {
|
||||
let evt = json!({ "type": "input_audio_buffer.speech_stopped" });
|
||||
let _ = ws.send(Message::Text(evt.to_string())).await;
|
||||
}
|
||||
"conversation.item.create" => {
|
||||
// The brain sends this to relay a function_call_output (tool
|
||||
// reply) back to OpenAI. The mock just acknowledges by
|
||||
// echoing a typed `conversation.item.created` event so the
|
||||
// brain's OpenAI pump doesn't treat the silence as an error.
|
||||
let evt = json!({
|
||||
"type": "conversation.item.created",
|
||||
"item": event.get("item").cloned().unwrap_or(Value::Null)
|
||||
});
|
||||
let _ = ws.send(Message::Text(evt.to_string())).await;
|
||||
}
|
||||
_ => {
|
||||
debug!(%peer, event_type = %event_type, "MockRealtimeBrain ignoring event");
|
||||
}
|
||||
}
|
||||
}
|
||||
debug!(%peer, "MockRealtimeBrain connection closed");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
|
||||
/// The S4 turn-ownership decision is load-bearing (spec §4.3 + ADR-0008).
|
||||
/// The mock asserts the brain's session.update has turn_detection: null;
|
||||
/// this test confirms a session.update WITH turn_detection: null is
|
||||
/// accepted (the green path).
|
||||
#[tokio::test]
|
||||
async fn accepts_session_update_with_turn_detection_null() {
|
||||
let mock = MockRealtimeBrain::start().await.unwrap();
|
||||
let url = mock.url();
|
||||
let req = url.as_str().into_client_request().unwrap();
|
||||
let (mut ws, _resp) = tokio_tungstenite::connect_async(req).await.unwrap();
|
||||
|
||||
// Send session.update with turn_detection: null.
|
||||
let session_update = json!({
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"modalities": ["text", "audio"],
|
||||
"voice": "alloy",
|
||||
"turn_detection": null
|
||||
}
|
||||
});
|
||||
ws.send(Message::Text(session_update.to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Send input_audio_buffer.append — must receive a canned
|
||||
// response.audio.delta (the brain's translator round-trips this
|
||||
// through encode_audio_out, so the wire shape must be exactly what
|
||||
// real OpenAI sends: base64 PCM16 in a `delta` field).
|
||||
let append = json!({
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": "AAAA"
|
||||
});
|
||||
ws.send(Message::Text(append.to_string())).await.unwrap();
|
||||
|
||||
let delta_msg = tokio::time::timeout(Duration::from_millis(500), ws.next())
|
||||
.await
|
||||
.expect("delta within 500ms")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let delta_text = delta_msg.into_text().unwrap();
|
||||
let delta: Value = serde_json::from_str(&delta_text).unwrap();
|
||||
assert_eq!(delta["type"], "response.audio.delta");
|
||||
assert!(delta["delta"].is_string());
|
||||
// The mock's canned PCM is 480 zeroed samples; the base64 string
|
||||
// must decode to exactly 960 bytes (480 × 2 bytes/i16).
|
||||
use base64::Engine;
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(delta["delta"].as_str().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(bytes.len(), 960);
|
||||
}
|
||||
|
||||
/// The S4 violation: session.update with turn_detection != null
|
||||
/// surfaces a typed OpenAI-shaped error so the brain's pump fails fast
|
||||
/// + visibly. Catches a future regression where the brain process
|
||||
/// forgets to set turn_detection: null.
|
||||
#[tokio::test]
|
||||
async fn rejects_session_update_with_turn_detection_set() {
|
||||
let mock = MockRealtimeBrain::start().await.unwrap();
|
||||
let url = mock.url();
|
||||
let req = url.as_str().into_client_request().unwrap();
|
||||
let (mut ws, _resp) = tokio_tungstenite::connect_async(req).await.unwrap();
|
||||
|
||||
let session_update = json!({
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"voice": "alloy",
|
||||
"turn_detection": { "type": "server_vad" }
|
||||
}
|
||||
});
|
||||
ws.send(Message::Text(session_update.to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let err_msg = tokio::time::timeout(Duration::from_millis(500), ws.next())
|
||||
.await
|
||||
.expect("error within 500ms")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let err: Value = serde_json::from_str(&err_msg.into_text().unwrap()).unwrap();
|
||||
assert_eq!(err["type"], "error");
|
||||
assert_eq!(err["error"]["code"], "invalid_session_update");
|
||||
}
|
||||
}
|
||||
222
crates/rutster-brain-realtime/src/openai_client.rs
Normal file
222
crates/rutster-brain-realtime/src/openai_client.rs
Normal file
@@ -0,0 +1,222 @@
|
||||
//! # wss://api.openai.com/v1/realtime client (spec §4)
|
||||
//!
|
||||
//! Owns the OpenAI-side WS connection. Brings the OpenAI Realtime event
|
||||
//! stream up, sends `session.update` with `turn_detection: null` (S4,
|
||||
//! spec §4.3) on handshake, then runs a pump loop that:
|
||||
//! - reads tap-side events (audio_in, function_call_output) from the
|
||||
//! tap WS server's `WebSocketStream` and translates + forwards them
|
||||
//! to OpenAI;
|
||||
//! - reads OpenAI events (response.audio.delta, speech_started/stopped,
|
||||
//! function_call_arguments.done, error) and translates + forwards them
|
||||
//! to the tap WS server.
|
||||
//!
|
||||
//! Reconnects with bounded backoff on OpenAI-side WS failure (spec §4.4 —
|
||||
//! the tap side stays connected; the OpenAI side has its own failure
|
||||
//! surface). The brain process emits a tap `error` event on OpenAI-side
|
||||
//! failure so the core can observe it.
|
||||
|
||||
// Imports are split by origin so the reader can trace which types come from
|
||||
// the wire-shape (rutster-tap — shared with the core), which come from the
|
||||
// pure translators (crate::translator — this crate's pure-function layer),
|
||||
// and which come from third-party plumbing.
|
||||
use crate::translator::{
|
||||
build_openai_session_update, openai_audio_delta_to_tap_audio_out,
|
||||
openai_function_call_arguments_done_to_tap, openai_speech_event_to_tap,
|
||||
tap_audio_in_to_openai_append, tap_function_call_output_to_openai_create_item,
|
||||
};
|
||||
use rutster_tap::{DecodedPayload, TapProtoError, decode_envelope, encode_function_call};
|
||||
use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum OpenAiClientError {
|
||||
#[error("OpenAI WS error: {0}")]
|
||||
Ws(#[from] tokio_tungstenite::tungstenite::Error),
|
||||
#[error("OpenAI auth failed (401)")]
|
||||
AuthFailed,
|
||||
#[error("translator error: {0}")]
|
||||
Translate(#[from] crate::translator::TranslateError),
|
||||
#[error("tap protocol error: {0}")]
|
||||
TapProto(#[from] TapProtoError),
|
||||
}
|
||||
|
||||
/// Build the OpenAI Realtime URL for the given model. Spec §4.2 + §5.3:
|
||||
/// `wss://api.openai.com/v1/realtime?model=<model>`.
|
||||
pub fn openai_realtime_url(model: &str) -> String {
|
||||
format!("wss://api.openai.com/v1/realtime?model={model}")
|
||||
}
|
||||
|
||||
/// Build the HTTP headers for the OpenAI WS handshake (spec §5.3). Two
|
||||
/// required headers:
|
||||
/// - `Authorization: Bearer <api_key>`
|
||||
/// - `OpenAI-Beta: realtime=v1`
|
||||
pub fn openai_headers(api_key: &str) -> Vec<(String, String)> {
|
||||
vec![
|
||||
("Authorization".to_string(), format!("Bearer {api_key}")),
|
||||
("OpenAI-Beta".to_string(), "realtime=v1".to_string()),
|
||||
]
|
||||
}
|
||||
|
||||
/// Drive the OpenAI Realtime WS connection + the tap-side pump.
|
||||
///
|
||||
/// `tap_rx`: inbound side — *tap frames* the brain received from the core
|
||||
/// (audio_in, function_call_output). We translate each one to its OpenAI
|
||||
/// equivalent + send to OpenAI.
|
||||
/// `openai_ws`: the OpenAI WS connection (already connected; the caller
|
||||
/// does the dial + auth).
|
||||
/// `tap_tx`: outbound side — *tap frames* the brain sends back to the core
|
||||
/// (audio_out, speech_started/stopped, function_call, error). We
|
||||
/// translate OpenAI events into these + send.
|
||||
pub async fn run_openai_pump<T>(
|
||||
mut openai_ws: tokio_tungstenite::WebSocketStream<T>,
|
||||
mut tap_rx: mpsc::Receiver<String>,
|
||||
tap_tx: mpsc::Sender<String>,
|
||||
voice: String,
|
||||
) -> Result<(), OpenAiClientError>
|
||||
where
|
||||
T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin,
|
||||
{
|
||||
// === Handshake: send session.update with turn_detection: null (S4). ===
|
||||
let session_update = build_openai_session_update(&voice);
|
||||
openai_ws
|
||||
.send(Message::Text(session_update.to_string()))
|
||||
.await?;
|
||||
info!(voice = %voice, "sent session.update to OpenAI (turn_detection: null)");
|
||||
|
||||
let mut seq_egress = 0u64;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Inbound tap frame from the core (audio_in, function_call_output).
|
||||
tap_str = tap_rx.recv() => {
|
||||
let Some(tap_str) = tap_str else {
|
||||
info!("tap_rx closed; ending OpenAI pump");
|
||||
return Ok(());
|
||||
};
|
||||
let decoded = decode_envelope(&tap_str)?;
|
||||
match decoded.payload {
|
||||
DecodedPayload::AudioIn(audio) => {
|
||||
let append = tap_audio_in_to_openai_append(&audio.pcm);
|
||||
openai_ws.send(Message::Text(append.to_string())).await?;
|
||||
}
|
||||
DecodedPayload::FunctionCallOutput(out) => {
|
||||
let create_item = tap_function_call_output_to_openai_create_item(
|
||||
&out.id, &out.status, &out.result,
|
||||
);
|
||||
openai_ws.send(Message::Text(create_item.to_string())).await?;
|
||||
}
|
||||
// Ignore others; slice-2's hello/audio_in on the tap side
|
||||
// happens before this pump starts.
|
||||
_ => warn!(?decoded.payload, "unexpected tap frame to OpenAI pump"),
|
||||
}
|
||||
}
|
||||
// Inbound OpenAI event.
|
||||
msg = openai_ws.next() => {
|
||||
let Some(msg) = msg else {
|
||||
info!("OpenAI WS stream ended");
|
||||
return Ok(());
|
||||
};
|
||||
let msg = msg?;
|
||||
let Ok(text) = msg.into_text() else { continue };
|
||||
let openai_event: Value = match serde_json::from_str(&text) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "OpenAI sent non-JSON event; ignoring");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let event_type = openai_event.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
match event_type {
|
||||
"response.audio.delta" => {
|
||||
let tap_str = openai_audio_delta_to_tap_audio_out(
|
||||
&openai_event, seq_egress, 0,
|
||||
)?;
|
||||
seq_egress += 1;
|
||||
tap_tx.send(tap_str).await.map_err(|_| {
|
||||
OpenAiClientError::Ws(
|
||||
tokio_tungstenite::tungstenite::Error::ConnectionClosed,
|
||||
)
|
||||
})?;
|
||||
}
|
||||
"input_audio_buffer.speech_started" => {
|
||||
let tap_str = openai_speech_event_to_tap(
|
||||
&openai_event, true, seq_egress, 0,
|
||||
)?;
|
||||
seq_egress += 1;
|
||||
tap_tx.send(tap_str).await.map_err(|_| {
|
||||
OpenAiClientError::Ws(
|
||||
tokio_tungstenite::tungstenite::Error::ConnectionClosed,
|
||||
)
|
||||
})?;
|
||||
}
|
||||
"input_audio_buffer.speech_stopped" => {
|
||||
let tap_str = openai_speech_event_to_tap(
|
||||
&openai_event, false, seq_egress, 0,
|
||||
)?;
|
||||
seq_egress += 1;
|
||||
tap_tx.send(tap_str).await.map_err(|_| {
|
||||
OpenAiClientError::Ws(
|
||||
tokio_tungstenite::tungstenite::Error::ConnectionClosed,
|
||||
)
|
||||
})?;
|
||||
}
|
||||
"response.function_call_arguments.done" => {
|
||||
let (call_id, name, args_str) =
|
||||
openai_function_call_arguments_done_to_tap(&openai_event)?;
|
||||
let tap_str = encode_function_call(
|
||||
&call_id, &name, &args_str, seq_egress, 0,
|
||||
)?;
|
||||
seq_egress += 1;
|
||||
tap_tx.send(tap_str).await.map_err(|_| {
|
||||
OpenAiClientError::Ws(
|
||||
tokio_tungstenite::tungstenite::Error::ConnectionClosed,
|
||||
)
|
||||
})?;
|
||||
}
|
||||
"error" => {
|
||||
// OpenAI emits a typed error event (e.g. 401 auth
|
||||
// failure surfaces here). Inspect the .error.code.
|
||||
let code = openai_event
|
||||
.pointer("/error/code")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
warn!(code = %code, "OpenAI error event");
|
||||
if code == "invalid_api_key" {
|
||||
return Err(OpenAiClientError::AuthFailed);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Unknown OpenAI event — log + drop (don't crash).
|
||||
warn!(event_type = %event_type, "ignoring OpenAI event type");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn openai_url_for_known_model() {
|
||||
assert_eq!(
|
||||
openai_realtime_url("gpt-4o-realtime"),
|
||||
"wss://api.openai.com/v1/realtime?model=gpt-4o-realtime"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_headers_carry_bearer_auth_and_beta() {
|
||||
let h = openai_headers("sk-test-12345");
|
||||
assert_eq!(h[0].0, "Authorization");
|
||||
assert_eq!(h[0].1, "Bearer sk-test-12345");
|
||||
assert_eq!(h[1].0, "OpenAI-Beta");
|
||||
assert_eq!(h[1].1, "realtime=v1");
|
||||
}
|
||||
}
|
||||
261
crates/rutster-brain-realtime/src/translator.rs
Normal file
261
crates/rutster-brain-realtime/src/translator.rs
Normal file
@@ -0,0 +1,261 @@
|
||||
//! # Tap ⇄ OpenAI Realtime event translation (spec §4)
|
||||
//!
|
||||
//! Pure functions — no async, no I/O, no call state. Each function maps
|
||||
//! one event between slice-3's tap protocol (slice-2 v1 + the additive
|
||||
//! slice-3 events from Task 2) and OpenAI Realtime's event JSON.
|
||||
//!
|
||||
//! The translation layer is **stateless** by design: the OpenAI-side WS
|
||||
//! client (Task 5) and the tap-side WS server (Task 9) call these
|
||||
//! functions per-event. No ownership of OpenAI's `session_id` beyond
|
||||
//! the connection's lifetime.
|
||||
|
||||
use rutster_tap::{TapProtoError, encode_audio_out, encode_speech_started, encode_speech_stopped};
|
||||
use serde_json::{Value, json};
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum TranslateError {
|
||||
#[error("OpenAI event missing required field: {field}")]
|
||||
MissingField { field: &'static str },
|
||||
#[error("tap protocol error: {0}")]
|
||||
TapProto(#[from] TapProtoError),
|
||||
}
|
||||
|
||||
/// Build OpenAI's `session.update` event (spec §4.2). The S4 turn-ownership
|
||||
/// decision (spec §4.3): `turn_detection: null`. OpenAI's server-side VAD
|
||||
/// is disabled; the FOB reflex loop (step 4) owns turn-taking.
|
||||
pub fn build_openai_session_update(voice: &str) -> Value {
|
||||
json!({
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"modalities": ["text", "audio"],
|
||||
"voice": voice,
|
||||
"input_audio_format": "pcm16",
|
||||
"output_audio_format": "pcm16",
|
||||
"sample_rate": 24000,
|
||||
"turn_detection": null
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Wrap a base64 PCM payload (already encoded per slice-2 §3 — explicit LE
|
||||
/// i16) into OpenAI's `input_audio_buffer.append` event (spec §4.2). Pass-
|
||||
/// through — the wire shape for OpenAI's audio is identical to slice-2's
|
||||
/// tap PCM (spec §3.5).
|
||||
pub fn tap_audio_in_to_openai_append(pcm_b64: &str) -> Value {
|
||||
json!({
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": pcm_b64
|
||||
})
|
||||
}
|
||||
|
||||
/// Translate OpenAI's `response.audio.delta` event (carries a `delta` field
|
||||
/// with base64 PCM) to a slice-3 tap `audio_out` frame string (spec §4.2).
|
||||
/// Pass-through on the audio payload; the envelope carries seq/ts.
|
||||
pub fn openai_audio_delta_to_tap_audio_out(
|
||||
openai_event: &Value,
|
||||
seq: u64,
|
||||
ts: u64,
|
||||
) -> Result<String, TranslateError> {
|
||||
let pcm_b64 = openai_event
|
||||
.get("delta")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or(TranslateError::MissingField { field: "delta" })?;
|
||||
// OpenAI's base64 PCM is LE i16 24 kHz mono — identical wire shape to
|
||||
// slice-2's tap PCM. Decode + re-encode as a PcmFrame (the canonical
|
||||
// 480-sample shape) so slice-2's playout ring stays byte-aligned.
|
||||
let frame = rutster_tap::decode_pcm(pcm_b64, rutster_tap::WIRE_SAMPLES_PER_FRAME)?;
|
||||
Ok(encode_audio_out(&frame, seq, ts)?)
|
||||
}
|
||||
|
||||
/// Translate OpenAI's `input_audio_buffer.speech_started` (started=true) or
|
||||
/// `.speech_stopped` (started=false) event to the matching tap frame
|
||||
/// (spec §4.2 + §4.3 — advisory events, the FOB reflex loop in step 4 will
|
||||
/// act on them).
|
||||
pub fn openai_speech_event_to_tap(
|
||||
_openai_event: &Value,
|
||||
started: bool,
|
||||
seq: u64,
|
||||
ts: u64,
|
||||
) -> Result<String, TranslateError> {
|
||||
if started {
|
||||
Ok(encode_speech_started(seq, ts)?)
|
||||
} else {
|
||||
Ok(encode_speech_stopped(seq, ts)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Translate OpenAI's `response.function_call_arguments.done` event to the
|
||||
/// tuple `(call_id, name, args_json_str)` the brain will emit as a tap
|
||||
/// `function_call` frame (spec §4.2 + §4.3). OpenAI sends `arguments` as
|
||||
/// a JSON _string_, not a JSON object — we don't reparse it; the brain
|
||||
/// process passes the raw string to `encode_function_call` (Task 2's
|
||||
/// encode_function_call parses it then into a serde_json::Value).
|
||||
pub fn openai_function_call_arguments_done_to_tap(
|
||||
openai_event: &Value,
|
||||
) -> Result<(String, String, String), TranslateError> {
|
||||
let call_id = openai_event
|
||||
.get("call_id")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or(TranslateError::MissingField { field: "call_id" })?;
|
||||
let name = openai_event
|
||||
.get("name")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or(TranslateError::MissingField { field: "name" })?;
|
||||
// OpenAI's `arguments` is a JSON string. If missing, default to "{}".
|
||||
let args_json_str = openai_event
|
||||
.get("arguments")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("{}")
|
||||
.to_string();
|
||||
Ok((call_id.to_string(), name.to_string(), args_json_str))
|
||||
}
|
||||
|
||||
/// Build OpenAI's `conversation.item.create` event for a tool-call result
|
||||
/// (spec §4.2). The tap `function_call_output` carries id + status + result;
|
||||
/// OpenAI's `conversation.item.create` wraps them as a function_call_output
|
||||
/// item with `call_id` (= id) + `output` (= JSON-stringified result; status
|
||||
/// is plumbed into the result value as a `_status` field — OpenAI has no
|
||||
/// formal status concept, so we encode ours in the result body).
|
||||
pub fn tap_function_call_output_to_openai_create_item(
|
||||
id: &str,
|
||||
status: &str,
|
||||
result: &Value,
|
||||
) -> Value {
|
||||
let mut output_value = result.clone();
|
||||
if let Some(obj) = output_value.as_object_mut() {
|
||||
obj.insert("_status".to_string(), Value::String(status.to_string()));
|
||||
}
|
||||
json!({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "function_call_output",
|
||||
"call_id": id,
|
||||
"output": output_value.to_string()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rutster_tap::{DecodedPayload, decode_envelope};
|
||||
|
||||
/// S4 turn-ownership test (load-bearing per ADR-0008, spec §4.3):
|
||||
/// the OpenAI session.update must have turn_detection: null.
|
||||
#[test]
|
||||
fn session_update_disables_openai_turn_detection() {
|
||||
let v = build_openai_session_update("alloy");
|
||||
assert_eq!(v["type"], "session.update");
|
||||
assert_eq!(v["session"]["voice"], "alloy");
|
||||
assert_eq!(v["session"]["modalities"][0], "text");
|
||||
assert_eq!(v["session"]["modalities"][1], "audio");
|
||||
assert_eq!(v["session"]["input_audio_format"], "pcm16");
|
||||
assert_eq!(v["session"]["output_audio_format"], "pcm16");
|
||||
assert_eq!(v["session"]["sample_rate"], 24000);
|
||||
// THE load-bearing assertion:
|
||||
assert_eq!(v["session"]["turn_detection"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn append_audio_payload_is_passthrough() {
|
||||
// The PCM base64 string for one zero frame (480 samples, every LE i16=0).
|
||||
use base64::Engine;
|
||||
let zeros = [0u8; 960];
|
||||
let pcm_b64 = base64::engine::general_purpose::STANDARD.encode(zeros);
|
||||
let v = tap_audio_in_to_openai_append(&pcm_b64);
|
||||
assert_eq!(v["type"], "input_audio_buffer.append");
|
||||
assert_eq!(v["audio"], pcm_b64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_audio_delta_to_tap_round_trips_through_slice2_codec() {
|
||||
// Build a slice-2 PcmFrame, base64-encode it the slice-2 way,
|
||||
// wrap it in an OpenAI response.audio.delta, translate to a tap
|
||||
// audio_out frame, then decode the tap frame with decode_envelope
|
||||
// and assert the PCM round-trips.
|
||||
use rutster_media::PcmFrame;
|
||||
let mut frame = PcmFrame::zeroed();
|
||||
frame.samples[0] = 100;
|
||||
frame.samples[1] = -100;
|
||||
frame.samples[479] = 12345;
|
||||
let pcm_b64 = rutster_tap::encode_pcm(&frame);
|
||||
let openai_event = json!({
|
||||
"type": "response.audio.delta",
|
||||
"delta": pcm_b64
|
||||
});
|
||||
let tap_str = openai_audio_delta_to_tap_audio_out(&openai_event, 5, 1000).unwrap();
|
||||
let decoded = decode_envelope(&tap_str).unwrap();
|
||||
match decoded.payload {
|
||||
DecodedPayload::AudioOut(p) => {
|
||||
let recovered = rutster_tap::decode_pcm(&p.pcm, p.samples).unwrap();
|
||||
assert_eq!(recovered.samples[0], 100);
|
||||
assert_eq!(recovered.samples[1], -100);
|
||||
assert_eq!(recovered.samples[479], 12345);
|
||||
assert_eq!(decoded.seq, 5);
|
||||
assert_eq!(decoded.ts, 1000);
|
||||
}
|
||||
other => panic!("expected AudioOut, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speech_started_translates_to_tap_speech_started() {
|
||||
let openai_event = json!({ "type": "input_audio_buffer.speech_started" });
|
||||
let s = openai_speech_event_to_tap(&openai_event, true, 3, 300).unwrap();
|
||||
let d = decode_envelope(&s).unwrap();
|
||||
assert!(matches!(d.payload, DecodedPayload::SpeechStarted));
|
||||
assert_eq!(d.seq, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speech_stopped_translates_to_tap_speech_stopped() {
|
||||
let openai_event = json!({ "type": "input_audio_buffer.speech_stopped" });
|
||||
let s = openai_speech_event_to_tap(&openai_event, false, 7, 700).unwrap();
|
||||
let d = decode_envelope(&s).unwrap();
|
||||
assert!(matches!(d.payload, DecodedPayload::SpeechStopped));
|
||||
assert_eq!(d.seq, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_call_done_extracts_three_tuple() {
|
||||
let openai_event = json!({
|
||||
"type": "response.function_call_arguments.done",
|
||||
"call_id": "call_abc123",
|
||||
"name": "hangup",
|
||||
"arguments": "{\"reason\": \"caller_requested\"}"
|
||||
});
|
||||
let (id, name, args_str) =
|
||||
openai_function_call_arguments_done_to_tap(&openai_event).unwrap();
|
||||
assert_eq!(id, "call_abc123");
|
||||
assert_eq!(name, "hangup");
|
||||
assert_eq!(args_str, "{\"reason\": \"caller_requested\"}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_call_done_missing_call_id_errors() {
|
||||
let openai_event = json!({
|
||||
"type": "response.function_call_arguments.done",
|
||||
"name": "hangup"
|
||||
});
|
||||
let r = openai_function_call_arguments_done_to_tap(&openai_event);
|
||||
assert!(matches!(
|
||||
r,
|
||||
Err(TranslateError::MissingField { field: "call_id" })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_call_output_to_create_item_plumbs_status_into_result() {
|
||||
let result = json!({"channel_state": "Closing"});
|
||||
let v = tap_function_call_output_to_openai_create_item("call_abc123", "ok", &result);
|
||||
assert_eq!(v["type"], "conversation.item.create");
|
||||
assert_eq!(v["item"]["type"], "function_call_output");
|
||||
assert_eq!(v["item"]["call_id"], "call_abc123");
|
||||
// The output is a JSON string — _status gets plumbed inside.
|
||||
let output_str = v["item"]["output"].as_str().unwrap();
|
||||
let output_val: Value = serde_json::from_str(output_str).unwrap();
|
||||
assert_eq!(output_val["channel_state"], "Closing");
|
||||
assert_eq!(output_val["_status"], "ok");
|
||||
}
|
||||
}
|
||||
88
crates/rutster-brain-realtime/tests/brain_mock_round_trip.rs
Normal file
88
crates/rutster-brain-realtime/tests/brain_mock_round_trip.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
//! slice-3 integration test: brain-process-level round-trip through the
|
||||
//! MockRealtimeBrain + the translator + `run_openai_pump`. Drives the brain
|
||||
//! the way the core's TapClient would (synthetic tap frames in, observes tap
|
||||
//! frames out) — proves the dev-loop interop contract the PM directive's
|
||||
//! Task 4 calls out ("actually start + interop on a basic audio round-trip")
|
||||
//! without requiring a real WebRTC peer (browser-driven, same constraint as
|
||||
//! slice-1/2).
|
||||
//!
|
||||
//! What this exercises end-to-end:
|
||||
//! 1. MockRealtimeBrain accepts a WS connection + asserts session.update
|
||||
//! has turn_detection: null (S4 — spec §4.3).
|
||||
//! 2. `run_openai_pump` sends session.update + bridges tap frames ↔ OpenAI
|
||||
//! events via the translator.
|
||||
//! 3. A tap `audio_in` frames the test pushes through `tap_via` → translator
|
||||
//! formats as `input_audio_buffer.append` → mock generates a canned
|
||||
//! `response.audio.delta` → translator formats as tap `audio_out` →
|
||||
//! arrives on `pump_tap_tx` (the pump's outbound mpsc).
|
||||
|
||||
use rutster_brain_realtime::{MockRealtimeBrain, openai_client::run_openai_pump};
|
||||
use rutster_tap::{PcmFrame, decode_envelope, encode_audio_in, encode_hello};
|
||||
use serde_json::Value;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
|
||||
#[tokio::test]
|
||||
async fn brain_mock_round_trip_translates_audio_in_to_audio_out() {
|
||||
let mock = MockRealtimeBrain::start().await.unwrap();
|
||||
let url = mock.url();
|
||||
let req = url.as_str().into_client_request().unwrap();
|
||||
let (openai_ws, _resp) = tokio_tungstenite::connect_async(req).await.unwrap();
|
||||
|
||||
// Bridge channels matching `run_openai_pump`'s signature (it owns neither
|
||||
// side of the tap WS — the binary's main.rs does the WS forwarding; here
|
||||
// we drive the pump directly with mpsc as the test's tap peer).
|
||||
let (tap_via, pump_tap_rx) = mpsc::channel::<String>(64);
|
||||
let (pump_tap_tx, mut tap_out_rx) = mpsc::channel::<String>(64);
|
||||
|
||||
// The pump sends session.update + then runs the select! loop. Spawn it.
|
||||
let pump = tokio::spawn(async move {
|
||||
run_openai_pump(openai_ws, pump_tap_rx, pump_tap_tx, "alloy".to_string()).await
|
||||
});
|
||||
|
||||
// Push a hello (the pump ignores hello per its `_ => warn!(...)` arm,
|
||||
// but real tap traffic always starts with hello).
|
||||
let hello = encode_hello("test-session", 0, 0).unwrap();
|
||||
tap_via.send(hello).await.unwrap();
|
||||
|
||||
// Push one audio_in frame. The pump translates to
|
||||
// input_audio_buffer.append; the mock replies with a canned
|
||||
// response.audio.delta; the pump translates back to a tap audio_out
|
||||
// frame on pump_tap_tx.
|
||||
let frame = PcmFrame::zeroed();
|
||||
let audio_in = encode_audio_in(&frame, 1, 100).unwrap();
|
||||
tap_via.send(audio_in).await.unwrap();
|
||||
|
||||
// Bounded-wait the audio_out frame coming back. Generous bound: the
|
||||
// pump's tokio select! cycle + the mock's WS round-trip on loopback
|
||||
// are sub-millisecond; 500 ms is fail-fast for any real regression.
|
||||
let out_str = tokio::time::timeout(Duration::from_millis(500), tap_out_rx.recv())
|
||||
.await
|
||||
.expect("audio_out within 500ms")
|
||||
.expect("channel not closed");
|
||||
|
||||
let decoded = decode_envelope(&out_str).unwrap();
|
||||
match decoded.payload {
|
||||
rutster_tap::DecodedPayload::AudioOut(p) => {
|
||||
// The mock's canned PCM is 480 zeroed samples; the brain's
|
||||
// translator round-trips it through PcmFrame (decode +
|
||||
// re-encode), so the wire shape is exactly slice-2's audio_out.
|
||||
assert_eq!(p.samples, 480);
|
||||
let recovered = rutster_tap::decode_pcm(&p.pcm, p.samples).unwrap();
|
||||
assert_eq!(recovered.samples.len(), 480);
|
||||
// Canned zeros — every sample is 0.
|
||||
assert_eq!(recovered.samples[0], 0);
|
||||
assert_eq!(recovered.samples[479], 0);
|
||||
// seq + ts carried through from the translator's per-event
|
||||
// counter (initial value 0 on the first outbound frame).
|
||||
assert_eq!(decoded.seq, 0);
|
||||
}
|
||||
other => panic!("expected AudioOut, got {other:?}"),
|
||||
}
|
||||
|
||||
// Drop the input channel to signal the pump to exit cleanly.
|
||||
drop(tap_via);
|
||||
let _ = pump.await;
|
||||
let _ = Value::Null; // suppress unused-import if Value unused here
|
||||
}
|
||||
@@ -10,11 +10,12 @@
|
||||
//! str0m 0.21's `Rtc::sdp_api().accept_offer(offer)` produces the SDP
|
||||
//! answer natively: DTLS fingerprint (from the cert str0m generates), ICE
|
||||
//! ufrag/pwd, and codec negotiation (Opus, the only codec we registered).
|
||||
//! Slice 1 does NOT hand-roll an SDP munger — str0m's path is the spec's
|
||||
//! "embryo of the future SIP SDP path" (§3.7). When step 5 brings SIP/SDP
|
||||
//! negotiation into `rutster-signaling-sip`, that crate may extract shared
|
||||
//! SDP helpers from str0m or build its own. Slice 1's WebRTC-ICE-coupled
|
||||
//! SDP lives entirely in str0m.
|
||||
//! Slice 1 does NOT hand-roll an SDP munger — str0m handles WebRTC
|
||||
//! offer/answer natively. There is no first-party SIP/SDP path: under
|
||||
//! [ADR-0007](../../../docs/adr/0007-trunk-rented-transport.md) the trunk is
|
||||
//! rented (a CPaaS media-leg fork — raw audio, no SDP) or an out-of-tree SBC
|
||||
//! (which owns any carrier SIP/SDP, outside the trust boundary). rutster's
|
||||
//! WebRTC-ICE-coupled SDP lives entirely in str0m.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -41,10 +41,14 @@ pub use protocol::{
|
||||
AudioPayload, DecodedFrame, DecodedPayload, Envelope, ErrorPayload, FrameKind, HelloPayload,
|
||||
PROTOCOL_VERSION, Payload, ReasonPayload, SAMPLES_PER_FRAME as WIRE_SAMPLES_PER_FRAME,
|
||||
SessionEndPayload, TapProtoError, decode_envelope, decode_pcm, encode_audio_in,
|
||||
encode_audio_out, encode_bye, encode_error, encode_hello, encode_pcm, encode_session_end,
|
||||
encode_audio_out, encode_bye, encode_error, encode_function_call, encode_function_call_output,
|
||||
encode_hello, encode_pcm, encode_session_end, encode_speech_started, encode_speech_stopped,
|
||||
encode_tools_update,
|
||||
};
|
||||
// Slice-3 additive (spec §3).
|
||||
pub use protocol::{FunctionCallOutputPayload, FunctionCallPayload, ToolsUpdatePayload};
|
||||
pub use tap_audio_pipe::TapAudioPipe;
|
||||
pub use tap_client::{TapClientError, run_tap_client};
|
||||
pub use tap_client::{FunctionCallEvent, FunctionCallOutputEvent, TapClientError, run_tap_client};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
@@ -47,6 +47,21 @@ pub enum FrameKind {
|
||||
SessionEnd,
|
||||
Bye,
|
||||
Error,
|
||||
/// brain → core: user speech started (advisory; translated from OpenAI
|
||||
/// `input_audio_buffer.speech_started`; spec §3.2).
|
||||
SpeechStarted,
|
||||
/// brain → core: user speech stopped (advisory; OpenAI
|
||||
/// `input_audio_buffer.speech_stopped`; spec §3.2).
|
||||
SpeechStopped,
|
||||
/// brain → core: brain proposes a tool call (translated from OpenAI
|
||||
/// `response.function_call_arguments.done`; spec §3.2).
|
||||
FunctionCall,
|
||||
/// core → brain: tool registry reply (spec §3.3).
|
||||
FunctionCallOutput,
|
||||
/// brain → core: brain declares its tool catalog on hello + on changes
|
||||
/// (spec §3.2).
|
||||
#[serde(rename = "tools.update")]
|
||||
ToolsUpdate,
|
||||
/// Unknown wire `type` values land here (spec §3.4: log + count + drop).
|
||||
/// `#[serde(other)]` catches any string not in the variants above.
|
||||
#[serde(other)]
|
||||
@@ -111,6 +126,11 @@ impl Serialize for Envelope {
|
||||
Payload::AudioIn(_) | Payload::AudioOut(_) => 2, // pcm, samples
|
||||
Payload::SessionEnd(_) | Payload::Bye(_) => 1, // reason
|
||||
Payload::Error(_) => 2, // code, message
|
||||
// Slice-3 adds: empty payload (0 fields), 3-field payloads, 1-field payload.
|
||||
Payload::SpeechStarted | Payload::SpeechStopped => 0,
|
||||
Payload::FunctionCall(_) => 3, // id, name, args
|
||||
Payload::FunctionCallOutput(_) => 3, // id, status, result
|
||||
Payload::ToolsUpdate(_) => 1, // tools
|
||||
};
|
||||
let mut st = serializer.serialize_struct("Envelope", 4 + payload_field_count)?;
|
||||
st.serialize_field("v", &self.v)?;
|
||||
@@ -142,6 +162,23 @@ impl Serialize for Envelope {
|
||||
st.serialize_field("code", &p.code)?;
|
||||
st.serialize_field("message", &p.message)?;
|
||||
}
|
||||
Payload::SpeechStarted | Payload::SpeechStopped => {
|
||||
// No payload fields — the envelope's `v`/`type`/`seq`/`ts`
|
||||
// is the whole message. The event name IS the message.
|
||||
}
|
||||
Payload::FunctionCall(p) => {
|
||||
st.serialize_field("id", &p.id)?;
|
||||
st.serialize_field("name", &p.name)?;
|
||||
st.serialize_field("args", &p.args)?;
|
||||
}
|
||||
Payload::FunctionCallOutput(p) => {
|
||||
st.serialize_field("id", &p.id)?;
|
||||
st.serialize_field("status", &p.status)?;
|
||||
st.serialize_field("result", &p.result)?;
|
||||
}
|
||||
Payload::ToolsUpdate(p) => {
|
||||
st.serialize_field("tools", &p.tools)?;
|
||||
}
|
||||
}
|
||||
st.end()
|
||||
}
|
||||
@@ -165,6 +202,16 @@ pub enum Payload {
|
||||
SessionEnd(SessionEndPayload),
|
||||
Bye(ReasonPayload),
|
||||
Error(ErrorPayload),
|
||||
/// Slice-3 additive (spec §3.2): brain → core, advisory; empty payload
|
||||
/// (the event name IS the message).
|
||||
SpeechStarted,
|
||||
SpeechStopped,
|
||||
/// Slice-3 additive (spec §3.2).
|
||||
FunctionCall(FunctionCallPayload),
|
||||
/// Slice-3 additive (spec §3.3): core → brain reply.
|
||||
FunctionCallOutput(FunctionCallOutputPayload),
|
||||
/// Slice-3 additive (spec §3.2): brain → core catalog declaration.
|
||||
ToolsUpdate(ToolsUpdatePayload),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -211,6 +258,42 @@ pub struct ErrorPayload {
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// `function_call` payload (brain → core; spec §3.2). Carries the
|
||||
/// brain-minted id + tool name + args. Args is a raw JSON Value (not a
|
||||
/// typed struct) so any tool schema is allowed.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FunctionCallPayload {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
/// Raw JSON arguments — the tool registry dispatches by name and lets
|
||||
/// each Tool impl parse the args itself. (OpenAI sends `arguments` as a
|
||||
/// JSON string; the translator parses it back to a Value before
|
||||
/// emitting the function_call tap frame.)
|
||||
pub args: serde_json::Value,
|
||||
}
|
||||
|
||||
/// `function_call_output` payload (core → brain; spec §3.3). The reply for
|
||||
/// a `function_call`. `status` is one of `"ok"`, `"error"`, `"not_implemented"`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FunctionCallOutputPayload {
|
||||
pub id: String,
|
||||
pub status: String,
|
||||
pub result: serde_json::Value,
|
||||
}
|
||||
|
||||
/// `tools.update` payload (brain → core; spec §3.2). The brain declares its
|
||||
/// tool catalog so the core's tool registry can validate function_call
|
||||
/// events by name.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ToolsUpdatePayload {
|
||||
/// An array of tool descriptors (each has `name`, `description`,
|
||||
/// `parameters`). The shape is intentionally permissive (a JSON array,
|
||||
/// not a typed Vec<ToolSchema>) so the brain can declare schemas the
|
||||
/// core doesn't know about — the core only checks the `name` field for
|
||||
/// dispatch, ignores the rest.
|
||||
pub tools: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Decoded frame — what `decode_envelope` returns. The kind is split into
|
||||
/// `audio_in` / `audio_out` variants (not just `Audio(AudioPayload)`) so
|
||||
/// callers `match` exhaustively on direction (spec §3.2 vs §3.3 — the two
|
||||
@@ -231,7 +314,17 @@ pub enum DecodedPayload {
|
||||
SessionEnd(SessionEndPayload),
|
||||
Bye(ReasonPayload),
|
||||
Error(ErrorPayload),
|
||||
/// Unknown `type` — log + count + drop (spec §3.4).
|
||||
/// Slice-3 (spec §3.2): the brain detected user speech started/stopped.
|
||||
SpeechStarted,
|
||||
SpeechStopped,
|
||||
/// Slice-3 (spec §3.2): brain wants the core to execute a tool.
|
||||
FunctionCall(FunctionCallPayload),
|
||||
/// Slice-3 (spec §3.3): the core's tool-registry reply.
|
||||
FunctionCallOutput(FunctionCallOutputPayload),
|
||||
/// Slice-3 (spec §3.2): brain declares its catalog so the core can
|
||||
/// validate function_call events.
|
||||
ToolsUpdate(ToolsUpdatePayload),
|
||||
/// Unknown `type` — log + count + drop (spec §3.4 of slice-2).
|
||||
Unknown,
|
||||
}
|
||||
|
||||
@@ -369,6 +462,111 @@ pub fn encode_error(code: &str, msg: &str, seq: u64, ts: u64) -> Result<String,
|
||||
Ok(serde_json::to_string(&env)?)
|
||||
}
|
||||
|
||||
/// Build `speech_started` (brain → core, advisory; spec §3.2).
|
||||
pub fn encode_speech_started(seq: u64, ts: u64) -> Result<String, TapProtoError> {
|
||||
let env = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
kind: FrameKind::SpeechStarted,
|
||||
seq,
|
||||
ts,
|
||||
payload: Payload::SpeechStarted,
|
||||
};
|
||||
Ok(serde_json::to_string(&env)?)
|
||||
}
|
||||
|
||||
/// Build `speech_stopped` (brain → core, advisory; spec §3.2).
|
||||
pub fn encode_speech_stopped(seq: u64, ts: u64) -> Result<String, TapProtoError> {
|
||||
let env = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
kind: FrameKind::SpeechStopped,
|
||||
seq,
|
||||
ts,
|
||||
payload: Payload::SpeechStopped,
|
||||
};
|
||||
Ok(serde_json::to_string(&env)?)
|
||||
}
|
||||
|
||||
/// Build `function_call` (brain → core; spec §3.2). `args_json_str` is the
|
||||
/// raw JSON string the brain's translator parses from OpenAI's
|
||||
/// `response.function_call_arguments.done.arguments` (which is itself a
|
||||
/// JSON string in OpenAI's wire format).
|
||||
pub fn encode_function_call(
|
||||
id: &str,
|
||||
name: &str,
|
||||
args_json_str: &str,
|
||||
seq: u64,
|
||||
ts: u64,
|
||||
) -> Result<String, TapProtoError> {
|
||||
let args: serde_json::Value = if args_json_str.is_empty() {
|
||||
serde_json::Value::Null
|
||||
} else {
|
||||
serde_json::from_str(args_json_str)?
|
||||
};
|
||||
let env = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
kind: FrameKind::FunctionCall,
|
||||
seq,
|
||||
ts,
|
||||
payload: Payload::FunctionCall(FunctionCallPayload {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
args,
|
||||
}),
|
||||
};
|
||||
Ok(serde_json::to_string(&env)?)
|
||||
}
|
||||
|
||||
/// Build `function_call_output` (core → brain; spec §3.3). `result_json_str`
|
||||
/// is the raw JSON string the tool-registry dispatch returns (serialized
|
||||
/// into the result field of the payload).
|
||||
pub fn encode_function_call_output(
|
||||
id: &str,
|
||||
status: &str, // "ok" | "error" | "not_implemented"
|
||||
result_json_str: &str,
|
||||
seq: u64,
|
||||
ts: u64,
|
||||
) -> Result<String, TapProtoError> {
|
||||
let result: serde_json::Value = if result_json_str.is_empty() {
|
||||
serde_json::Value::Null
|
||||
} else {
|
||||
serde_json::from_str(result_json_str)?
|
||||
};
|
||||
let env = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
kind: FrameKind::FunctionCallOutput,
|
||||
seq,
|
||||
ts,
|
||||
payload: Payload::FunctionCallOutput(FunctionCallOutputPayload {
|
||||
id: id.to_string(),
|
||||
status: status.to_string(),
|
||||
result,
|
||||
}),
|
||||
};
|
||||
Ok(serde_json::to_string(&env)?)
|
||||
}
|
||||
|
||||
/// Build `tools.update` (brain → core; spec §3.2). `tools_json_str` is the
|
||||
/// raw JSON array of tool descriptors.
|
||||
pub fn encode_tools_update(
|
||||
tools_json_str: &str,
|
||||
seq: u64,
|
||||
ts: u64,
|
||||
) -> Result<String, TapProtoError> {
|
||||
let tools: serde_json::Value = if tools_json_str.is_empty() {
|
||||
serde_json::Value::Array(vec![])
|
||||
} else {
|
||||
serde_json::from_str(tools_json_str)?
|
||||
};
|
||||
let env = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
kind: FrameKind::ToolsUpdate,
|
||||
seq,
|
||||
ts,
|
||||
payload: Payload::ToolsUpdate(ToolsUpdatePayload { tools }),
|
||||
};
|
||||
Ok(serde_json::to_string(&env)?)
|
||||
}
|
||||
|
||||
/// Decode an incoming wire string into a `DecodedFrame`. Validates `v` and
|
||||
/// (for audio frames) `samples` and PCM round-trip. Unknown `type` values
|
||||
/// decode to `DecodedPayload::Unknown` (caller logs + counts + drops per
|
||||
@@ -440,6 +638,20 @@ pub fn decode_envelope(s: &str) -> Result<DecodedFrame, TapProtoError> {
|
||||
let p: ErrorPayload = serde_json::from_value(extra_value)?;
|
||||
DecodedPayload::Error(p)
|
||||
}
|
||||
FrameKind::SpeechStarted => DecodedPayload::SpeechStarted,
|
||||
FrameKind::SpeechStopped => DecodedPayload::SpeechStopped,
|
||||
FrameKind::FunctionCall => {
|
||||
let p: FunctionCallPayload = serde_json::from_value(extra_value)?;
|
||||
DecodedPayload::FunctionCall(p)
|
||||
}
|
||||
FrameKind::FunctionCallOutput => {
|
||||
let p: FunctionCallOutputPayload = serde_json::from_value(extra_value)?;
|
||||
DecodedPayload::FunctionCallOutput(p)
|
||||
}
|
||||
FrameKind::ToolsUpdate => {
|
||||
let p: ToolsUpdatePayload = serde_json::from_value(extra_value)?;
|
||||
DecodedPayload::ToolsUpdate(p)
|
||||
}
|
||||
FrameKind::Unknown => DecodedPayload::Unknown,
|
||||
};
|
||||
Ok(DecodedFrame {
|
||||
@@ -596,4 +808,96 @@ mod tests {
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
/// Slice-3 additive event types must round-trip through (de)serialization
|
||||
/// without breaking slice-2's v1 contract. Every new kind + payload.
|
||||
#[test]
|
||||
fn speech_started_round_trips() {
|
||||
let s = encode_speech_started(7, 100).unwrap();
|
||||
assert!(s.contains("\"type\":\"speech_started\""));
|
||||
assert!(s.contains("\"v\":1"));
|
||||
let d = decode_envelope(&s).unwrap();
|
||||
assert_eq!(d.seq, 7);
|
||||
assert_eq!(d.ts, 100);
|
||||
assert!(matches!(d.payload, DecodedPayload::SpeechStarted));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speech_stopped_round_trips() {
|
||||
let s = encode_speech_stopped(9, 200).unwrap();
|
||||
assert!(s.contains("\"type\":\"speech_stopped\""));
|
||||
let d = decode_envelope(&s).unwrap();
|
||||
assert_eq!(d.seq, 9);
|
||||
assert!(matches!(d.payload, DecodedPayload::SpeechStopped));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_call_round_trips() {
|
||||
let s = encode_function_call("abc-123", "hangup", "{}", 0, 0).unwrap();
|
||||
assert!(s.contains("\"type\":\"function_call\""));
|
||||
assert!(s.contains("\"id\":\"abc-123\""));
|
||||
assert!(s.contains("\"name\":\"hangup\""));
|
||||
let d = decode_envelope(&s).unwrap();
|
||||
match d.payload {
|
||||
DecodedPayload::FunctionCall(p) => {
|
||||
assert_eq!(p.id, "abc-123");
|
||||
assert_eq!(p.name, "hangup");
|
||||
}
|
||||
other => panic!("expected FunctionCall, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_call_output_round_trips() {
|
||||
let s =
|
||||
encode_function_call_output("abc-123", "ok", r#"{"channel_state":"Closing"}"#, 0, 0)
|
||||
.unwrap();
|
||||
assert!(s.contains("\"type\":\"function_call_output\""));
|
||||
assert!(s.contains("\"status\":\"ok\""));
|
||||
let d = decode_envelope(&s).unwrap();
|
||||
match d.payload {
|
||||
DecodedPayload::FunctionCallOutput(p) => {
|
||||
assert_eq!(p.id, "abc-123");
|
||||
assert_eq!(p.status, "ok");
|
||||
}
|
||||
other => panic!("expected FunctionCallOutput, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tools_update_round_trips() {
|
||||
let tools_json = r#"[{"name":"hangup","description":"hang up the call"}]"#;
|
||||
let s = encode_tools_update(tools_json, 0, 0).unwrap();
|
||||
assert!(s.contains("\"type\":\"tools.update\""));
|
||||
assert!(s.contains("\"tools\":["));
|
||||
let d = decode_envelope(&s).unwrap();
|
||||
match d.payload {
|
||||
DecodedPayload::ToolsUpdate(p) => {
|
||||
assert!(p.tools.is_array());
|
||||
assert_eq!(p.tools.as_array().unwrap().len(), 1);
|
||||
}
|
||||
other => panic!("expected ToolsUpdate, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards-compat: slice-2's echo brain sees the new types as unknown
|
||||
/// (the `#[serde(other)]` on the old enum absorbed them). With the new
|
||||
/// enum in place, the kinds decode to their new variants (rather than
|
||||
/// Unknown). This test asserts the decode no longer drops them.
|
||||
#[test]
|
||||
fn new_kinds_decode_to_their_variants_not_unknown() {
|
||||
for s in [
|
||||
encode_speech_started(0, 0).unwrap(),
|
||||
encode_speech_stopped(0, 0).unwrap(),
|
||||
encode_function_call("x", "x", "null", 0, 0).unwrap(),
|
||||
encode_function_call_output("x", "ok", "null", 0, 0).unwrap(),
|
||||
encode_tools_update("[]", 0, 0).unwrap(),
|
||||
] {
|
||||
let d = decode_envelope(&s).unwrap();
|
||||
assert!(
|
||||
!matches!(d.payload, DecodedPayload::Unknown),
|
||||
"new event type decoded as Unknown: {s}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,11 +63,46 @@ pub enum TapClientError {
|
||||
/// `close` is a shared borrow (`&mut oneshot::Receiver<()>`) so the
|
||||
/// `TapEngine` reconnect loop (Task 7) can share one close signal across
|
||||
/// reconnect attempts of the same session (see module docs).
|
||||
///
|
||||
/// # slice-3 §5.2 — the tool-call side-channel
|
||||
///
|
||||
/// Two extra mpsc halves (relative to slice-2's audio-only pump) carry the
|
||||
/// brain's `function_call` proposals out to the binary's poll task and the
|
||||
/// binary's `function_call_output` replies back onto the wire:
|
||||
///
|
||||
/// - `tx_function_call` — the TapClient emits a `FunctionCallEvent` here
|
||||
/// whenever it observes a tap `function_call` frame on its inbound WS
|
||||
/// stream. The binary's poll task drains this in the same cycle it drains
|
||||
/// the existing `flush_tx` side-channel (slice-2 §5.3 step 4 — one extra
|
||||
/// channel, same cycle) and dispatches via `ToolRegistry::dispatch`.
|
||||
/// - `rx_function_call_output` — the binary's poll task writes
|
||||
/// `FunctionCallOutputEvent`s here (after a `ToolRegistry::dispatch` call
|
||||
/// completes); the TapClient drains this in the same `tokio::select!`
|
||||
/// pump as the audio + close arms and sends each as a `function_call_output`
|
||||
/// tap WS frame to the brain.
|
||||
///
|
||||
/// Both halves are mpsc ends (not oneshot) because the brain may propose
|
||||
/// multiple tool calls per session (one-of-many, not one-of-one) and the
|
||||
/// binary may queue multiple replies before the TapClient's pump cycle
|
||||
/// drains them.
|
||||
//
|
||||
// clippy::too_many_arguments: the slice-3 §5.2 design added two more mpsc
|
||||
// halves to slice-2's already-5-arg pump signature for a total of 8. Each
|
||||
// arg is a distinct channel end with a distinct lifetime owner (the WS
|
||||
// stream, the session id, two sender/receiver pairs for audio, two for the
|
||||
// tool-call side-channel, the metrics Arc, the close oneshot). Wrapping
|
||||
// them in a struct would obscure that all-but-one are channel ends shared
|
||||
// with the binary's poll task — the flat signature mirrors slice-2's
|
||||
// precedent and keeps the call site readable. Suppress per AGENTS.md's
|
||||
// "documented inline rationale" exception to the -D warnings bar.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn run_tap_client<T>(
|
||||
mut ws: WebSocketStream<T>,
|
||||
session_id: ChannelId,
|
||||
rx_pcm_in: &mut mpsc::Receiver<PcmFrame>,
|
||||
tx_audio_out: mpsc::Sender<PcmFrame>,
|
||||
tx_function_call: mpsc::Sender<FunctionCallEvent>,
|
||||
rx_function_call_output: &mut mpsc::Receiver<FunctionCallOutputEvent>,
|
||||
metrics: Arc<TapMetrics>,
|
||||
close: &mut oneshot::Receiver<()>,
|
||||
) -> Result<(), TapClientError>
|
||||
@@ -175,6 +210,44 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
// slice-3 §5.2: drain a `function_call_output` event the binary's
|
||||
// poll task wrote (after `ToolRegistry::dispatch` returned) + send
|
||||
// it as a `function_call_output` tap WS frame to the brain. The
|
||||
// `seq_egress` bump mirrors the audio arm — every egress frame
|
||||
// shares the same per-direction counter (spec §3.1).
|
||||
//
|
||||
// Like `rx_pcm_in.recv()`, this is one of many arms in the
|
||||
// select! — the engine's `tx_function_call_output` sender lives
|
||||
// in the binary; `run_tap_client` returns when the brain WS ends
|
||||
// or close fires, regardless of pending function_call_output
|
||||
// events (they're dropped on close — same posture as pending
|
||||
// audio_out frames on teardown).
|
||||
out = rx_function_call_output.recv() => {
|
||||
if let Some(out) = out {
|
||||
let ts = elapsed_ms(session_start);
|
||||
let result_str = out.0.result.to_string();
|
||||
match crate::protocol::encode_function_call_output(
|
||||
&out.0.id,
|
||||
&out.0.status,
|
||||
&result_str,
|
||||
seq_egress,
|
||||
ts,
|
||||
) {
|
||||
Ok(s) => {
|
||||
seq_egress += 1;
|
||||
if let Err(e) = ws.send(Message::Text(s)).await {
|
||||
warn!(error = ?e, %session_id, "ws send function_call_output failed");
|
||||
return Err(e.into());
|
||||
}
|
||||
info!(%session_id, call_id = %out.0.id, status = %out.0.status, "sent function_call_output to brain");
|
||||
}
|
||||
Err(e) => {
|
||||
metrics.malformed_frames.fetch_add(1, Ordering::Relaxed);
|
||||
warn!(error = ?e, "encode function_call_output failed; dropping");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Inbound WS frame from brain.
|
||||
msg = ws.next() => {
|
||||
let Some(msg) = msg else {
|
||||
@@ -188,7 +261,7 @@ where
|
||||
if let Ok(text) = msg.into_text() {
|
||||
handle_brain_frame(
|
||||
&text, &mut last_seq_ingress, &tx_audio_out,
|
||||
&metrics, session_start,
|
||||
&tx_function_call, &metrics, session_start,
|
||||
).await;
|
||||
}
|
||||
}
|
||||
@@ -251,6 +324,7 @@ async fn handle_brain_frame(
|
||||
text: &str,
|
||||
last_seq_ingress: &mut Option<u64>,
|
||||
tx_audio_out: &mpsc::Sender<PcmFrame>,
|
||||
tx_function_call: &mpsc::Sender<FunctionCallEvent>,
|
||||
metrics: &Arc<TapMetrics>,
|
||||
session_start: Instant,
|
||||
) {
|
||||
@@ -307,6 +381,41 @@ async fn handle_brain_frame(
|
||||
metrics.unknown_frames.fetch_add(1, Ordering::Relaxed);
|
||||
warn!("unexpected frame direction from brain; dropping");
|
||||
}
|
||||
// Slice-3 spec §5.2: `function_call` flows through the side-channel
|
||||
// (NON-BLOCKING try_send — the binary's poll task drains on its own
|
||||
// cycle). The same "drop + observe" posture as audio_out applies if
|
||||
// the channel is full: a backed-up binary means we drop the proposal
|
||||
// and the brain gets no reply (the brain process knows no
|
||||
// function_call_output arrived → its OpenAI pump keeps going; the
|
||||
// model tolerates missing replies per OpenAI's design).
|
||||
DecodedPayload::FunctionCall(p) => {
|
||||
if tx_function_call.try_send(FunctionCallEvent(p)).is_err() {
|
||||
metrics.outbound_dropped.fetch_add(1, Ordering::Relaxed);
|
||||
warn!(
|
||||
"function_call dropped (binary poll task not draining; brain will see no reply)"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Slice-3 (spec §3.2): `function_call_output` is core→brain; ignore
|
||||
// if a brain sends one back (a misbehaving brain — same posture as
|
||||
// `SessionEnd`/`AudioIn` from brain above).
|
||||
DecodedPayload::FunctionCallOutput(_) => {
|
||||
metrics.unknown_frames.fetch_add(1, Ordering::Relaxed);
|
||||
warn!("unexpected function_call_output from brain; dropping");
|
||||
}
|
||||
// Slice-3 advisory — same "logged + counted, not forwarded" posture
|
||||
// as `Unknown`. The FOB reflex loop in step 4 will act on these;
|
||||
// slice-3 only pre-paves the wire event.
|
||||
DecodedPayload::SpeechStarted | DecodedPayload::SpeechStopped => {
|
||||
metrics.unknown_frames.fetch_add(1, Ordering::Relaxed);
|
||||
debug!("advisory interruption event observed; not acted on in slice-3");
|
||||
}
|
||||
DecodedPayload::ToolsUpdate(_) => {
|
||||
metrics.unknown_frames.fetch_add(1, Ordering::Relaxed);
|
||||
debug!(
|
||||
"tools.update observed; slice-3 dispatch keys off function_call by name, not catalog"
|
||||
);
|
||||
}
|
||||
}
|
||||
let _ = session_start; // used for ts computation if added later
|
||||
}
|
||||
@@ -315,6 +424,36 @@ fn elapsed_ms(start: Instant) -> u64 {
|
||||
start.elapsed().as_millis() as u64
|
||||
}
|
||||
|
||||
/// A `function_call` event the TapClient **observed** on its inbound WS
|
||||
/// stream and forwarded to the binary's poll task via the
|
||||
/// `tx_function_call` side-channel (spec §5.2). The binary's poll task
|
||||
/// drains this (alongside the existing `flush_rx` side-channel — slice-2
|
||||
/// §5.3 step 4) and dispatches each event through `ToolRegistry::dispatch`.
|
||||
///
|
||||
/// # Why a thin newtype over `FunctionCallPayload` (and not a bare alias)?
|
||||
///
|
||||
/// A type alias (`pub type FunctionCallEvent = FunctionCallPayload;`) would
|
||||
/// let the binary pass a `FunctionCallPayload` where a `FunctionCallEvent`
|
||||
/// is expected without surfacing the intent. The newtype draws a small but
|
||||
/// real boundary: the wire-payload type (`FunctionCallPayload`) lives in
|
||||
/// `protocol.rs` for (de)serialization; the side-channel event type
|
||||
/// (`FunctionCallEvent`) lives here for dispatch. They share a shape but
|
||||
/// carry different semantic weight — honoring the newtype-over-primitives
|
||||
/// convention from AGENTS.md even at the message level.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FunctionCallEvent(pub crate::protocol::FunctionCallPayload);
|
||||
|
||||
/// A `function_call_output` event the binary's poll task **emits** back to
|
||||
/// the TapClient via the `tx_function_call_output` side-channel (spec §5.2
|
||||
/// — the binary's poll task dispatches through `ToolRegistry::dispatch`,
|
||||
/// serializes the `ToolResult`, and writes the output here). The TapClient
|
||||
/// drains this in the same `tokio::select!` pump cycle as the audio + close
|
||||
/// arms and sends each as a `function_call_output` tap WS frame.
|
||||
///
|
||||
/// Same newtype-over-payload rationale as `FunctionCallEvent`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FunctionCallOutputEvent(pub crate::protocol::FunctionCallOutputPayload);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
// TapClient is heavily async; its real behavior is exercised in the
|
||||
@@ -322,6 +461,7 @@ mod tests {
|
||||
// tests here cover the pure helpers.
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::encode_function_call;
|
||||
|
||||
#[test]
|
||||
fn elapsed_ms_is_monotonic_nonneg() {
|
||||
@@ -330,4 +470,83 @@ mod tests {
|
||||
// First call ~0; just assert it's a valid u64.
|
||||
assert_eq!(ms, ms); // tautology but clippy-clean
|
||||
}
|
||||
|
||||
/// slice-3 spec §5.2: when the TapClient observes a tap `function_call`
|
||||
/// frame on its inbound WS stream it emits a `FunctionCallEvent` on
|
||||
/// the `tx_function_call` side-channel. The binary's poll task drains
|
||||
/// that and dispatches via `ToolRegistry::dispatch`. This test pins the
|
||||
/// contract end-to-end through the pure helper (`handle_brain_frame`):
|
||||
/// hand it a wire-encoded function_call frame + a fresh mpsc pair and
|
||||
/// assert the receiver observes the forwarded event.
|
||||
#[tokio::test]
|
||||
async fn handle_brain_frame_forwards_function_call_to_side_channel() {
|
||||
let (tx_fc, mut rx_fc) = mpsc::channel::<FunctionCallEvent>(8);
|
||||
let (tx_audio_out, _rx_audio_out) = mpsc::channel::<PcmFrame>(8);
|
||||
let metrics = Arc::new(TapMetrics::new());
|
||||
|
||||
// Build a wire function_call frame: id="call-1", name="hangup", args={}.
|
||||
let wire = encode_function_call("call-1", "hangup", "{}", 1, 100).unwrap();
|
||||
let mut last_seq: Option<u64> = None;
|
||||
|
||||
handle_brain_frame(
|
||||
&wire,
|
||||
&mut last_seq,
|
||||
&tx_audio_out,
|
||||
&tx_fc,
|
||||
&metrics,
|
||||
Instant::now(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// The side-channel must have observed exactly one FunctionCallEvent
|
||||
// carrying the wire's id/name/args.
|
||||
let event = tokio::time::timeout(Duration::from_millis(200), rx_fc.recv())
|
||||
.await
|
||||
.expect("tx_function_call drained within 200ms")
|
||||
.expect("channel not closed");
|
||||
assert_eq!(event.0.id, "call-1");
|
||||
assert_eq!(event.0.name, "hangup");
|
||||
assert_eq!(event.0.args, serde_json::json!({}));
|
||||
// seq tracking still updates for the side-channeled event.
|
||||
assert_eq!(last_seq, Some(1));
|
||||
}
|
||||
|
||||
/// slice-3 spec §5.2 — the *advisory* interrupt events (`speech_started`
|
||||
/// /`speech_stopped`) and `tools.update` are observed (logged + counted)
|
||||
/// but do NOT flow through the function_call side-channel (only
|
||||
/// `function_call` does — that's the only event with a binary-side
|
||||
/// disposal). This pins that boundary: an advisory event must NOT
|
||||
/// produce a `FunctionCallEvent` even with the channel plumbed.
|
||||
#[tokio::test]
|
||||
async fn advisory_events_are_logged_not_forwarded_to_function_call_channel() {
|
||||
let (tx_fc, mut rx_fc) = mpsc::channel::<FunctionCallEvent>(8);
|
||||
let (tx_audio_out, _rx_audio_out) = mpsc::channel::<PcmFrame>(8);
|
||||
let metrics = Arc::new(TapMetrics::new());
|
||||
|
||||
let wire = crate::protocol::encode_speech_started(2, 200).unwrap();
|
||||
let mut last_seq: Option<u64> = None;
|
||||
handle_brain_frame(
|
||||
&wire,
|
||||
&mut last_seq,
|
||||
&tx_audio_out,
|
||||
&tx_fc,
|
||||
&metrics,
|
||||
Instant::now(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// No FunctionCallEvent forwarded — the channel stays empty. Pick a
|
||||
// tight bounded receive so the test fails fast if a future refactor
|
||||
// starts forwarding advisory events here.
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), rx_fc.recv())
|
||||
.await
|
||||
.is_err(),
|
||||
"no FunctionCallEvent expected for advisory events"
|
||||
);
|
||||
// The advisory event IS still observed via metrics (seq gap tracking
|
||||
// + the unknown-slot counter remains 0 — speech_started is now a
|
||||
// known payload variant).
|
||||
assert_eq!(last_seq, Some(2));
|
||||
}
|
||||
}
|
||||
|
||||
1
crates/rutster-tap/tests/fixtures/function_call.json
vendored
Normal file
1
crates/rutster-tap/tests/fixtures/function_call.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"v":1,"type":"function_call","seq":0,"ts":0,"id":"abc-123","name":"hangup","args":{}}
|
||||
1
crates/rutster-tap/tests/fixtures/function_call_output.json
vendored
Normal file
1
crates/rutster-tap/tests/fixtures/function_call_output.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"v":1,"type":"function_call_output","seq":0,"ts":0,"id":"abc-123","status":"ok","result":{"channel_state":"Closing"}}
|
||||
1
crates/rutster-tap/tests/fixtures/speech_started.json
vendored
Normal file
1
crates/rutster-tap/tests/fixtures/speech_started.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"v":1,"type":"speech_started","seq":7,"ts":100}
|
||||
1
crates/rutster-tap/tests/fixtures/speech_stopped.json
vendored
Normal file
1
crates/rutster-tap/tests/fixtures/speech_stopped.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"v":1,"type":"speech_stopped","seq":9,"ts":200}
|
||||
1
crates/rutster-tap/tests/fixtures/tools_update.json
vendored
Normal file
1
crates/rutster-tap/tests/fixtures/tools_update.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"v":1,"type":"tools.update","seq":0,"ts":0,"tools":[{"description":"hang up the call","name":"hangup"}]}
|
||||
225
crates/rutster-tap/tests/protocol_events.rs
Normal file
225
crates/rutster-tap/tests/protocol_events.rs
Normal file
@@ -0,0 +1,225 @@
|
||||
//! Slice-3 golden-fixture tests for the tap protocol's new event types.
|
||||
//!
|
||||
//! ## Why this file exists (alongside `protocol.rs`'s inline tests)
|
||||
//!
|
||||
//! `protocol.rs` already has inline round-trip tests for each new slice-3
|
||||
//! event kind, but they assert with `String::contains` — a *substring* check.
|
||||
//! That catches "the right `type` tag is present" but it cannot catch:
|
||||
//!
|
||||
//! - a change in **field order** (the `Envelope` `Serialize` impl at
|
||||
//! `protocol.rs::Serialize_for_Envelope` pins field order deliberately —
|
||||
//! see its doc comment: "field order is pinned to make the wire bytes
|
||||
//! deterministic… helps snapshot tests and interop with brains that read
|
||||
//! key order for debugging");
|
||||
//! - a stray extra field;
|
||||
//! - a missing field;
|
||||
//! - a change in whitespace or number formatting.
|
||||
//!
|
||||
//! These fixtures lock the **exact wire bytes** produced by each `encode_*`
|
||||
//! function. Any future refactor that alters the on-wire shape — even
|
||||
//! cosmetically — breaks a test here, forcing a conscious fixture update +
|
||||
//! a protocol-version bump conversation (spec §3.1: `PROTOCOL_VERSION` is
|
||||
//! the contract door).
|
||||
//!
|
||||
//! ## Forwards-compat (the slice-2 §3.4 "drop + observe" contract)
|
||||
//!
|
||||
//! The last test here re-asserts the load-bearing decode-side rule: an
|
||||
//! unknown `type` string deserializes to `DecodedPayload::Unknown` (via
|
||||
//! `FrameKind`'s `#[serde(other)]` fallback) rather than erroring. This is
|
||||
//! the foundation of the forwards-compat posture — a brain speaking a
|
||||
//! future protocol version's event kind must not crash an older core; it
|
||||
//! gets logged + counted + dropped (the behavioral side lives in
|
||||
//! `tap_client::handle_brain_frame` and is exercised end-to-end by the
|
||||
//! slice-2 / slice-3 integration tests, not here).
|
||||
//!
|
||||
//! ## Why string equality (not `serde_json::Value` equality)
|
||||
//!
|
||||
//! Comparing parsed `serde_json::Value`s would be *semantic* equality — it
|
||||
//! would treat `{"a":1,"b":2}` and `{"b":2,"a":1}` as the same. That
|
||||
//! throws away the field-order pinning the `Envelope` serializer
|
||||
//! guarantees. Comparing the raw encoded `String` to the fixture's bytes
|
||||
//! keeps the order contract load-bearing.
|
||||
|
||||
use std::fs;
|
||||
|
||||
use rutster_tap::{
|
||||
DecodedPayload, decode_envelope, encode_function_call, encode_function_call_output,
|
||||
encode_speech_started, encode_speech_stopped, encode_tools_update,
|
||||
};
|
||||
|
||||
/// Read a fixture file from `tests/fixtures/<name>`. Uses
|
||||
/// `CARGO_MANIFEST_DIR` so the path is stable regardless of which
|
||||
/// directory `cargo test` is invoked from. Trailing newlines are stripped
|
||||
/// because `serde_json::to_string` (what the encoders call) never emits one
|
||||
/// — a fixture checked into git shouldn't acquire a phantom newline mismatch.
|
||||
fn fixture(name: &str) -> String {
|
||||
let path = format!("{}/tests/fixtures/{}", env!("CARGO_MANIFEST_DIR"), name);
|
||||
let raw =
|
||||
fs::read_to_string(&path).unwrap_or_else(|e| panic!("read fixture {name} ({path}): {e}"));
|
||||
raw.trim_end().to_string()
|
||||
}
|
||||
|
||||
// ─── speech_started ──────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn golden_speech_started_wire_bytes_match_fixture() {
|
||||
let encoded = encode_speech_started(7, 100).unwrap();
|
||||
let golden = fixture("speech_started.json");
|
||||
assert_eq!(
|
||||
encoded, golden,
|
||||
"speech_started wire bytes drifted from golden fixture"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speech_started_fixture_decodes_to_speechstarted_variant() {
|
||||
let golden = fixture("speech_started.json");
|
||||
let decoded = decode_envelope(&golden).unwrap();
|
||||
assert_eq!(decoded.seq, 7);
|
||||
assert_eq!(decoded.ts, 100);
|
||||
assert!(
|
||||
matches!(decoded.payload, DecodedPayload::SpeechStarted),
|
||||
"speech_started fixture did not decode to SpeechStarted: {:?}",
|
||||
decoded.payload
|
||||
);
|
||||
}
|
||||
|
||||
// ─── speech_stopped ──────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn golden_speech_stopped_wire_bytes_match_fixture() {
|
||||
let encoded = encode_speech_stopped(9, 200).unwrap();
|
||||
let golden = fixture("speech_stopped.json");
|
||||
assert_eq!(
|
||||
encoded, golden,
|
||||
"speech_stopped wire bytes drifted from golden fixture"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speech_stopped_fixture_decodes_to_speechstopped_variant() {
|
||||
let golden = fixture("speech_stopped.json");
|
||||
let decoded = decode_envelope(&golden).unwrap();
|
||||
assert_eq!(decoded.seq, 9);
|
||||
assert_eq!(decoded.ts, 200);
|
||||
assert!(
|
||||
matches!(decoded.payload, DecodedPayload::SpeechStopped),
|
||||
"speech_stopped fixture did not decode to SpeechStopped: {:?}",
|
||||
decoded.payload
|
||||
);
|
||||
}
|
||||
|
||||
// ─── function_call ───────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn golden_function_call_wire_bytes_match_fixture() {
|
||||
// `args_json_str="{}"` — the encoder parses it into `serde_json::Value`
|
||||
// (an empty object) so the wire field is `"args":{}`, not `"args":"{}"`.
|
||||
// This is load-bearing: a brain emitting a function_call with a raw
|
||||
// JSON-string args would be a different wire shape (and a translator
|
||||
// bug). The fixture pins the parsed-object shape.
|
||||
let encoded = encode_function_call("abc-123", "hangup", "{}", 0, 0).unwrap();
|
||||
let golden = fixture("function_call.json");
|
||||
assert_eq!(
|
||||
encoded, golden,
|
||||
"function_call wire bytes drifted from golden fixture"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_call_fixture_decodes_to_functioncall_with_fields() {
|
||||
let golden = fixture("function_call.json");
|
||||
let decoded = decode_envelope(&golden).unwrap();
|
||||
match decoded.payload {
|
||||
DecodedPayload::FunctionCall(p) => {
|
||||
assert_eq!(p.id, "abc-123");
|
||||
assert_eq!(p.name, "hangup");
|
||||
assert_eq!(p.args, serde_json::Value::Object(serde_json::Map::new()));
|
||||
}
|
||||
other => panic!("expected FunctionCall, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// ─── function_call_output ───────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn golden_function_call_output_wire_bytes_match_fixture() {
|
||||
let encoded =
|
||||
encode_function_call_output("abc-123", "ok", r#"{"channel_state":"Closing"}"#, 0, 0)
|
||||
.unwrap();
|
||||
let golden = fixture("function_call_output.json");
|
||||
assert_eq!(
|
||||
encoded, golden,
|
||||
"function_call_output wire bytes drifted from golden fixture"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_call_output_fixture_decodes_to_functioncalloutput_with_fields() {
|
||||
let golden = fixture("function_call_output.json");
|
||||
let decoded = decode_envelope(&golden).unwrap();
|
||||
match decoded.payload {
|
||||
DecodedPayload::FunctionCallOutput(p) => {
|
||||
assert_eq!(p.id, "abc-123");
|
||||
assert_eq!(p.status, "ok");
|
||||
// The result field is a parsed JSON object, not a string.
|
||||
assert_eq!(
|
||||
p.result.get("channel_state"),
|
||||
Some(&serde_json::json!("Closing"))
|
||||
);
|
||||
}
|
||||
other => panic!("expected FunctionCallOutput, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// ─── tools.update ───────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn golden_tools_update_wire_bytes_match_fixture() {
|
||||
let tools_json = r#"[{"name":"hangup","description":"hang up the call"}]"#;
|
||||
let encoded = encode_tools_update(tools_json, 0, 0).unwrap();
|
||||
let golden = fixture("tools_update.json");
|
||||
assert_eq!(
|
||||
encoded, golden,
|
||||
"tools.update wire bytes drifted from golden fixture"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tools_update_fixture_decodes_to_toolsupdate_with_array() {
|
||||
let golden = fixture("tools_update.json");
|
||||
let decoded = decode_envelope(&golden).unwrap();
|
||||
match decoded.payload {
|
||||
DecodedPayload::ToolsUpdate(p) => {
|
||||
let arr = p
|
||||
.tools
|
||||
.as_array()
|
||||
.expect("tools field must be a JSON array");
|
||||
assert_eq!(arr.len(), 1);
|
||||
assert_eq!(arr[0].get("name"), Some(&serde_json::json!("hangup")));
|
||||
}
|
||||
other => panic!("expected ToolsUpdate, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// ─── forwards-compat (slice-2 §3.4 "drop + observe") ────────────────────
|
||||
|
||||
/// The load-bearing forwards-compat rule on the decode side: an unknown
|
||||
/// `type` string deserializes to `DecodedPayload::Unknown` (via
|
||||
/// `FrameKind`'s `#[serde(other)]`), **not** an error. A future-protocol
|
||||
/// brain emitting `"type":"future_event"` must not crash an older core;
|
||||
/// the core logs + counts + drops it (the observe-and-continue posture
|
||||
/// the eventual fuzz harness will stress). This re-asserts the contract
|
||||
/// the inline `unknown_type_decodes_to_unknown_variant` test established,
|
||||
/// here in the external fixture file so it sits next to the golden wire
|
||||
/// shapes it protects.
|
||||
#[test]
|
||||
fn unknown_wire_type_decodes_to_unknown_variant_not_error() {
|
||||
let wire = r#"{"v":1,"type":"future_event","seq":0,"ts":0}"#;
|
||||
let decoded = decode_envelope(wire).expect("unknown type must not error (forwards-compat)");
|
||||
assert!(
|
||||
matches!(decoded.payload, DecodedPayload::Unknown),
|
||||
"forwards-compat broken: unknown type decoded to {:?}, expected Unknown",
|
||||
decoded.payload
|
||||
);
|
||||
}
|
||||
@@ -18,6 +18,7 @@ futures-util = { workspace = true }
|
||||
url = { workspace = true }
|
||||
dashmap = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
@@ -27,6 +28,11 @@ serde_json = { workspace = true }
|
||||
[dev-dependencies]
|
||||
tower = { workspace = true }
|
||||
rutster-tap-echo = { path = "../rutster-tap-echo" }
|
||||
# slice-3 §7.4: integration test spins up the in-process MockRealtimeBrain
|
||||
# + drives the brain's `run_openai_pump` directly (replicating the brain
|
||||
# process's accept loop in the test, no subprocess). The `mock` feature
|
||||
# brings in MockRealtimeBrain without requiring OPENAI_API_KEY.
|
||||
rutster-brain-realtime = { path = "../rutster-brain-realtime", features = ["mock"] }
|
||||
|
||||
[[bin]]
|
||||
name = "rutster"
|
||||
|
||||
@@ -25,3 +25,4 @@
|
||||
pub mod routes;
|
||||
pub mod session_map;
|
||||
pub mod tap_engine;
|
||||
pub mod tool_registry;
|
||||
|
||||
@@ -39,6 +39,11 @@ use tokio::sync::Mutex;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::tap_engine::{TapConn, spawn_tap_engine};
|
||||
// Re-using the binary crate's `tokio::sync::mpsc` import (the engine task
|
||||
// + the poll task both live in `rutster`'s poll-driver module). The type
|
||||
// only appears in `TapConn` field signatures + the slice-3 §5.2 dispatch
|
||||
// helper below; bringing it in here keeps the type names short.
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
/// The per-session wrapper struct (slice-2, spec §6).
|
||||
///
|
||||
@@ -265,6 +270,13 @@ async fn drive_all_sessions(state: &AppState, now: Instant) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// slice-3 §5.2: drain the per-session `rx_function_call`
|
||||
// side-channel in the same cycle as the `flush_rx` drain (slice-2
|
||||
// §5.3 step 4 pattern — one extra channel, same cycle). The
|
||||
// helper spawns each dispatch as its own task so the 750 ms
|
||||
// `AppState::close` await (hangup's teardown handshake) can't
|
||||
// stall the poll cadence.
|
||||
let _fc_drained = drain_function_calls(state, id).await;
|
||||
// Hold the DashMap Ref only long enough to clone the Arc-wrapped
|
||||
// rtc + tap_url; the async poll + spawn happens outside the shard.
|
||||
let (rtc, tap_url) = match state.sessions.get(&id) {
|
||||
@@ -298,7 +310,12 @@ async fn drive_all_sessions(state: &AppState, now: Instant) {
|
||||
if let ChannelState::Connected = s.channel.state {
|
||||
if s.channel.tap.is_none() {
|
||||
// First connect: spawn the TapEngine, wire the TapAudioPipe.
|
||||
let (pipe, conn) = spawn_tap_engine(id, tap_url);
|
||||
// slice-3 §6.2: spawn_tap_engine constructs a per-channel
|
||||
// ToolRegistry holding HangupTool (the only wired tool in
|
||||
// slice-3 — §6.3) bound to this AppState + ChannelId.
|
||||
let app_state = state.clone();
|
||||
let tap_url_clone = tap_url.clone();
|
||||
let (pipe, conn) = spawn_tap_engine(id, tap_url_clone, app_state);
|
||||
s.set_pipe(pipe);
|
||||
s.channel.tap = Some(TapHandle::new());
|
||||
info!(channel_id = %id, "tap engine spawned on Connected");
|
||||
@@ -319,3 +336,176 @@ async fn drive_all_sessions(state: &AppState, now: Instant) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// slice-3 §5.2 + §6 — drain the per-session `rx_function_call` side-channel
|
||||
/// and dispatch each event through the per-channel `ToolRegistry`. One
|
||||
/// dispatch result → one `function_call_output` written to
|
||||
/// `tx_function_call_output` (which the TapClient forwards to the brain on
|
||||
/// its next pump cycle).
|
||||
///
|
||||
/// # Why this is a separate helper (not inline in `drive_all_sessions`)
|
||||
///
|
||||
/// `ToolRegistry::dispatch` is `async` (the `hangup` tool calls
|
||||
/// `AppState::close`, which awaits the engine's teardown handshake — spec
|
||||
/// §5.2's 750 ms bound). Awaiting that inline in the poll task would stall
|
||||
/// the 10 ms poll cadence for every hangup. Instead we **collect** events
|
||||
/// non-blockingly here (`try_recv` — drop + observe on full channel, same
|
||||
/// posture as the existing flush drain), then **spawn** each dispatch as
|
||||
/// its own tokio task so the poll task returns to its 10 ms cadence
|
||||
/// immediately. The spawned task holds its own clones of `AppState` +
|
||||
/// `tool_registry` + `tx_function_call_output` — no shared mutable state
|
||||
/// with the poll path.
|
||||
///
|
||||
/// Returns the count of events drained (for observability — matches the
|
||||
/// flush drain's "drain all" loop shape). The poll task calls this in the
|
||||
/// same `drive_all_sessions` cycle it drains the `flush_rx` side-channel
|
||||
/// (slice-2 §5.3 step 4 pattern, one extra channel, same cycle).
|
||||
async fn drain_function_calls(state: &AppState, id: ChannelId) -> usize {
|
||||
// Collect: pull everything we need out of the DashMap shard BEFORE any
|
||||
// await (the spawned dispatch's `AppState::close` later takes the same
|
||||
// shard lock — holding a `get_mut` Ref across the spawn's await would
|
||||
// deadlock slice-2's other shard-mutating handlers). The Collector scope
|
||||
// owns a single short-lived `get_mut` Ref; the loop does only sync
|
||||
// `try_recv` + Vec push (no await).
|
||||
let collected: Vec<rutster_tap::FunctionCallEvent>;
|
||||
let tool_registry: Arc<tokio::sync::Mutex<crate::tool_registry::ToolRegistry>>;
|
||||
let tx_out: Option<mpsc::Sender<rutster_tap::FunctionCallOutputEvent>>;
|
||||
{
|
||||
let Some(mut entry) = state.sessions.get_mut(&id) else {
|
||||
return 0;
|
||||
};
|
||||
let Some(conn) = entry.tap_conn.as_mut() else {
|
||||
return 0;
|
||||
};
|
||||
tool_registry = conn.tool_registry.clone();
|
||||
tx_out = conn.tx_function_call_output.clone();
|
||||
let Some(rx_fc) = conn.rx_function_call.as_mut() else {
|
||||
return 0;
|
||||
};
|
||||
collected = (0..).map_while(|_| rx_fc.try_recv().ok()).collect();
|
||||
}
|
||||
let drained = collected.len();
|
||||
|
||||
// Spawn dispatches outside the shard Ref scope. `AppState::close` is a
|
||||
// 750 ms bounded wait in the worst case (slice-2 §5.2's teardown
|
||||
// handshake); spawning keeps the 10 ms poll cadence responsive.
|
||||
for event in collected {
|
||||
let app_state = state.clone();
|
||||
let reg = tool_registry.clone();
|
||||
let tx_out = tx_out.clone();
|
||||
tokio::spawn(async move {
|
||||
let payload = event.0;
|
||||
let call_id = payload.id.clone();
|
||||
let name = payload.name.clone();
|
||||
debug!(channel_id = %id, call_id = %call_id, tool = %name, "dispatching function_call");
|
||||
let result = reg.lock().await.dispatch(&name, payload.args).await;
|
||||
let (status, result_value) = result.to_status_result();
|
||||
let out =
|
||||
rutster_tap::FunctionCallOutputEvent(rutster_tap::FunctionCallOutputPayload {
|
||||
id: call_id.clone(),
|
||||
status: status.clone(),
|
||||
result: result_value,
|
||||
});
|
||||
// try_send — non-blocking. The TapClient's pump loop drains
|
||||
// `rx_function_call_output` on its next `select!` cycle. If the
|
||||
// channel is full (brain not pumping), we drop + observe (same
|
||||
// posture as the engine's `tx_function_call.try_send` on the
|
||||
// inbound side). `tx_out` is `Option<Sender>`; `None` means the
|
||||
// engine has torn down its receiver — the result is dropped,
|
||||
// acceptable since the call is hanging up.
|
||||
if let Some(tx) = tx_out.as_ref() {
|
||||
if let Err(e) = tx.try_send(out) {
|
||||
warn!(
|
||||
channel_id = %id, call_id = %call_id, error = ?e,
|
||||
"function_call_output dropped (TapClient pump not draining)"
|
||||
);
|
||||
}
|
||||
}
|
||||
// app_state is held for the dispatch lifetime — the HangupTool
|
||||
// inside the registry already captured its own clone at registry
|
||||
// construction, so this outer binding exists for future tool
|
||||
// impls that need the live AppState handed to dispatch (none in
|
||||
// slice-3 beyond hangup). Drop the silent-binding warning by
|
||||
// referencing it once.
|
||||
drop(app_state);
|
||||
info!(channel_id = %id, call_id = %call_id, status = %status, "function_call dispatched");
|
||||
});
|
||||
}
|
||||
drained
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rutster_tap::{
|
||||
FunctionCallEvent, FunctionCallOutputEvent, FunctionCallPayload, TapMetrics,
|
||||
};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
/// slice-3 §5.2 + §6 — a `function_call` event drained from
|
||||
/// `rx_function_call` triggers `ToolRegistry::dispatch("hangup")` which
|
||||
/// fires `AppState::close` (the slice-2 teardown path), and the dispatch
|
||||
/// result flows back as a `FunctionCallOutputEvent` on
|
||||
/// `tx_function_call_output`. End-to-end contract test for the helper
|
||||
/// minus the live TapClient pump (which is integration-test territory).
|
||||
#[tokio::test]
|
||||
async fn drain_function_calls_dispatches_hangup_and_writes_output() {
|
||||
let state = AppState::default();
|
||||
// Create a session so AppState::close has something to remove.
|
||||
let id = state.create_session(None).unwrap();
|
||||
|
||||
// Build a TapConn with manually-controlled side-channel ends so we
|
||||
// can push a FunctionCallEvent from the test side + observe the
|
||||
// dispatch's output on the paired Receiver.
|
||||
let (tx_fc, rx_fc) = tokio::sync::mpsc::channel::<FunctionCallEvent>(8);
|
||||
let (tx_fco, mut rx_fco) = tokio::sync::mpsc::channel::<FunctionCallOutputEvent>(8);
|
||||
let mut registry = crate::tool_registry::ToolRegistry::new();
|
||||
registry.register(Box::new(crate::tool_registry::HangupTool::new(
|
||||
state.clone(),
|
||||
id,
|
||||
)));
|
||||
let (close_tx, _close_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let conn = crate::tap_engine::TapConn {
|
||||
close_tx,
|
||||
join: tokio::spawn(async {}), // no-op handle; aborted below
|
||||
metrics: TapMetrics::new(),
|
||||
flush_rx: None,
|
||||
rx_function_call: Some(rx_fc),
|
||||
tx_function_call_output: Some(tx_fco),
|
||||
tool_registry: Arc::new(Mutex::new(registry)),
|
||||
};
|
||||
state.sessions.get_mut(&id).unwrap().tap_conn = Some(conn);
|
||||
|
||||
// Push a function_call for the hangup tool — simulates what the
|
||||
// TapClient does when it observes a `function_call` tap frame.
|
||||
tx_fc
|
||||
.send(FunctionCallEvent(FunctionCallPayload {
|
||||
id: "call-1".to_string(),
|
||||
name: "hangup".to_string(),
|
||||
args: serde_json::json!({}),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Drain — this spawns a dispatch task + returns immediately.
|
||||
let drained = drain_function_calls(&state, id).await;
|
||||
assert_eq!(drained, 1, "exactly one function_call should drain");
|
||||
|
||||
// The spawned dispatch task fires AppState::close (session removed)
|
||||
// + writes the function_call_output. Bounded-wait the result with
|
||||
// a generous timeout (AppState::close has a 750 ms teardown bound +
|
||||
// the spawned task may take a few ms to schedule).
|
||||
let out = tokio::time::timeout(Duration::from_secs(2), rx_fco.recv())
|
||||
.await
|
||||
.expect("function_call_output drained within 2s")
|
||||
.expect("channel not closed");
|
||||
assert_eq!(out.0.id, "call-1");
|
||||
assert_eq!(out.0.status, "ok");
|
||||
assert_eq!(out.0.result["channel_state"], "Closing");
|
||||
// AppState::close removed the session entry (the teardown it fires).
|
||||
assert!(
|
||||
state.sessions.get(&id).is_none(),
|
||||
"session should be removed after hangup dispatch"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,14 +30,18 @@ use std::time::Duration;
|
||||
|
||||
use futures_util::FutureExt;
|
||||
use rutster_call_model::ChannelId;
|
||||
use rutster_tap::tap_client::{TapClientError, run_tap_client};
|
||||
use rutster_tap::tap_client::{
|
||||
FunctionCallEvent, FunctionCallOutputEvent, TapClientError, run_tap_client,
|
||||
};
|
||||
use rutster_tap::{TapAudioPipe, TapMetrics};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tokio::sync::{Mutex, mpsc, oneshot};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tracing::{info, warn};
|
||||
use url::Url;
|
||||
|
||||
use crate::tool_registry::{HangupTool, ToolRegistry};
|
||||
|
||||
/// Capacity for the two mpsc channels between TapAudioPipe and TapClient.
|
||||
/// Large enough that a slow brain tick doesn't drop on every cycle;
|
||||
/// small enough that a runaway brain doesn't accumulate seconds of audio.
|
||||
@@ -83,6 +87,28 @@ pub struct TapConn {
|
||||
/// returned from `spawn_tap_engine`; the engine task owns the paired
|
||||
/// `flush_tx` and signals a flush after each failed pump cycle.
|
||||
pub flush_rx: Option<mpsc::Receiver<()>>,
|
||||
/// slice-3 §5.2 tool-call side-channel (brain → core): the binary's
|
||||
/// poll task drains `function_call` events from here + dispatches via
|
||||
/// [`tool_registry`]. Engine owns the paired `tx_function_call` Sender
|
||||
/// moved into `run_tap_client`. `Option` to keep the type constructible
|
||||
/// in tests that don't plumb it; always `Some` on conns returned from
|
||||
/// `spawn_tap_engine` (matches `flush_rx`'s posture).
|
||||
pub rx_function_call: Option<mpsc::Receiver<FunctionCallEvent>>,
|
||||
/// slice-3 §5.2 tool-call side-channel (core → brain): the binary's
|
||||
/// poll task writes `function_call_output` events here (one per
|
||||
/// `ToolRegistry::dispatch` result). The engine owns the paired
|
||||
/// `rx_function_call_output` Receiver moved into `run_tap_client` so
|
||||
/// the engine's pump loop forwards each event as a tap WS frame.
|
||||
pub tx_function_call_output: Option<mpsc::Sender<FunctionCallOutputEvent>>,
|
||||
/// slice-3 §6 — per-channel tool registry. One registry per active
|
||||
/// session (spec §6.2 "keyed by ChannelId"); `HangupTool` is the only
|
||||
/// tool wired in slice-3 (§6.3). `Arc<Mutex<_>>` because the binary's
|
||||
/// poll task mutates the registry (dispatch) while the TapConn is
|
||||
/// shared across the poll task + the routes layer (potential future
|
||||
/// `tools.update` forwarding). `Mutex` (not `RwLock`) because dispatch
|
||||
/// is effectively exclusive — there's no read-heavy workload to
|
||||
/// optimize.
|
||||
pub tool_registry: Arc<Mutex<ToolRegistry>>,
|
||||
}
|
||||
|
||||
/// Spawn the TapEngine task for one session. Dials `tap_url`, runs the pump
|
||||
@@ -102,7 +128,11 @@ pub struct TapConn {
|
||||
/// single struct would force the registry to also own the pipe, splitting
|
||||
/// ownership of `RtcSession`'s internals across two modules — exactly the
|
||||
/// pattern the slice-2 plan's structural review warned against.
|
||||
pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, TapConn) {
|
||||
pub fn spawn_tap_engine(
|
||||
session_id: ChannelId,
|
||||
tap_url: Url,
|
||||
app_state: crate::session_map::AppState,
|
||||
) -> (TapAudioPipe, TapConn) {
|
||||
// Two mpsc channels. The naming convention is "from the engine's POV":
|
||||
// - `tx_pcm_in`/`rx_pcm_in`: peer PCM flowing INTO the engine (sink side
|
||||
// of TapAudioPipe calls `tx_pcm_in.try_send(frame)`; engine task owns
|
||||
@@ -122,8 +152,27 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
|
||||
// flush signals collapse to one `clear_playout_ring` call from the
|
||||
// binary side).
|
||||
let (flush_tx, flush_rx) = mpsc::channel::<()>(8);
|
||||
// slice-3 §5.2 tool-call side-channels. Same capacity-shape rationale
|
||||
// as `flush_tx`: `try_send`-safe on both ends, idempotent drain. One
|
||||
// extra pair of mpsc halves vs slice-2 — same cycle in the binary's
|
||||
// poll task (drain alongside `flush_rx`), same pump-arm in run_tap_client.
|
||||
let (tx_function_call, rx_function_call) =
|
||||
mpsc::channel::<FunctionCallEvent>(TAP_MPSC_CAPACITY);
|
||||
let (tx_function_call_output, rx_function_call_output) =
|
||||
mpsc::channel::<FunctionCallOutputEvent>(TAP_MPSC_CAPACITY);
|
||||
let metrics = TapMetrics::new();
|
||||
|
||||
// slice-3 §6.2: per-channel tool registry. The engine constructs it
|
||||
// (one per active session — spec "keyed by ChannelId, one registry per
|
||||
// active channel"). `HangupTool` is the only wired tool in slice-3
|
||||
// (§6.3); other tool names reply `not_implemented` via dispatch.
|
||||
// The registry is `Arc<Mutex<ToolRegistry>>` so the binary's poll task
|
||||
// can `dispatch` on it while the TapConn is shared (the future
|
||||
// `tools.update`-forwarding path will also share this handle).
|
||||
let mut registry = ToolRegistry::new();
|
||||
registry.register(Box::new(HangupTool::new(app_state, session_id)));
|
||||
let tool_registry = Arc::new(Mutex::new(registry));
|
||||
|
||||
// Clone metrics three ways: the engine task, the TapAudioPipe, and
|
||||
// the TapConn handle each hold their own `Arc<TapMetrics>` refcount.
|
||||
// All three views must observe the same counters — `Arc` is the
|
||||
@@ -132,6 +181,7 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
|
||||
// shared mutation is sound without a `Mutex`).
|
||||
let metrics_for_pipe = metrics.clone();
|
||||
let metrics_for_conn = metrics.clone();
|
||||
let tool_registry_for_task = tool_registry.clone();
|
||||
|
||||
let join = tokio::spawn(async move {
|
||||
run_engine_loop(
|
||||
@@ -141,6 +191,8 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
|
||||
tx_audio_out,
|
||||
close_rx,
|
||||
flush_tx,
|
||||
tx_function_call,
|
||||
rx_function_call_output,
|
||||
metrics,
|
||||
)
|
||||
.await;
|
||||
@@ -155,6 +207,9 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
|
||||
join,
|
||||
metrics: metrics_for_conn,
|
||||
flush_rx: Some(flush_rx),
|
||||
rx_function_call: Some(rx_function_call),
|
||||
tx_function_call_output: Some(tx_function_call_output),
|
||||
tool_registry: tool_registry_for_task,
|
||||
};
|
||||
|
||||
(pipe, conn)
|
||||
@@ -174,6 +229,14 @@ pub fn spawn_tap_engine(session_id: ChannelId, tap_url: Url) -> (TapAudioPipe, T
|
||||
/// pattern the plan's Note A spelled out: a single `tokio::select!` over
|
||||
/// (a) the close receiver and (b) `connect_brain`, then a second select
|
||||
/// over the close receiver and the backoff sleep.
|
||||
//
|
||||
// clippy::too_many_arguments: slice-3 §5.2 added two more mpsc halves to
|
||||
// slice-2's engine-loop signature (the tool-call side-channel ends). Same
|
||||
// rationale as `run_tap_client` above: each arg is a distinct channel
|
||||
// end with a distinct lifetime owner. Wrapping them in a struct would
|
||||
// obscure the channel-pair structure. Suppress per AGENTS.md's documented
|
||||
// inline-rationale exception.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_engine_loop(
|
||||
session_id: ChannelId,
|
||||
tap_url: Url,
|
||||
@@ -181,6 +244,8 @@ async fn run_engine_loop(
|
||||
tx_audio_out: mpsc::Sender<rutster_media::PcmFrame>,
|
||||
mut close: oneshot::Receiver<()>,
|
||||
flush_tx: mpsc::Sender<()>,
|
||||
tx_function_call: mpsc::Sender<FunctionCallEvent>,
|
||||
mut rx_function_call_output: mpsc::Receiver<FunctionCallOutputEvent>,
|
||||
metrics: Arc<TapMetrics>,
|
||||
) {
|
||||
let mut backoff = Backoff::default();
|
||||
@@ -212,11 +277,19 @@ async fn run_engine_loop(
|
||||
// === Step 2: run the pump loop until close/error. ===
|
||||
// `close` is a shared `&mut oneshot::Receiver` — same
|
||||
// signal across reconnect attempts (Task 4's design).
|
||||
// The slice-3 §5.2 tool-call side-channels are also shared
|
||||
// across reconnect attempts — a brain reconnect mid-call
|
||||
// must keep the binary's tool-registry dispatch flowing
|
||||
// (a function_call the brain proposed right before it
|
||||
// dropped should still get a function_call_output reply
|
||||
// on the next connect).
|
||||
let pump_result = run_tap_client(
|
||||
ws,
|
||||
session_id,
|
||||
&mut rx_pcm_in,
|
||||
tx_audio_out.clone(),
|
||||
tx_function_call.clone(),
|
||||
&mut rx_function_call_output,
|
||||
metrics.clone(),
|
||||
&mut close,
|
||||
)
|
||||
@@ -414,11 +487,45 @@ mod tests {
|
||||
// increment. We abort the task on test drop to avoid leak.
|
||||
let id = ChannelId::new();
|
||||
let url = Url::parse("ws://127.0.0.1:1/echo").unwrap(); // port 1 = unreachable
|
||||
let (mut pipe, conn) = spawn_tap_engine(id, url);
|
||||
let (mut pipe, conn) = spawn_tap_engine(id, url, crate::session_map::AppState::default());
|
||||
// TapAudioPipe is the seam object — should default to silent underflow.
|
||||
assert!(pipe.next_pcm_frame().is_none());
|
||||
// TapConn carries the close oneshot + JoinHandle + metrics.
|
||||
let _ = conn.close_tx.send(());
|
||||
conn.join.abort();
|
||||
}
|
||||
|
||||
/// slice-3 §5.2: the TapConn returned by `spawn_tap_engine` must carry
|
||||
/// the tool-call side-channel ends so the binary's poll task can drain
|
||||
/// `function_call` events (→ `ToolRegistry::dispatch`) and write
|
||||
/// `function_call_output` replies. This is a structural smoke test —
|
||||
/// it pins that the plumbing lands (the binary's poll-task drain needs
|
||||
/// all three: a ready `rx_function_call`, a ready `tx_function_call_output`,
|
||||
/// and a `tool_registry` scoped to this session).
|
||||
#[tokio::test]
|
||||
async fn spawn_returns_tap_conn_with_function_call_side_channels() {
|
||||
let id = ChannelId::new();
|
||||
let url = Url::parse("ws://127.0.0.1:1/echo").unwrap(); // unreachable brain
|
||||
let (_pipe, conn) = spawn_tap_engine(id, url, crate::session_map::AppState::default());
|
||||
|
||||
// rx_function_call: Some(Receiver) — engine owns the paired Sender.
|
||||
assert!(
|
||||
conn.rx_function_call.is_some(),
|
||||
"TapConn must carry a drainable rx_function_call"
|
||||
);
|
||||
// tx_function_call_output: Some(Sender) — engine owns the paired
|
||||
// Receiver on the pump-side (run_tap_client).
|
||||
assert!(
|
||||
conn.tx_function_call_output.is_some(),
|
||||
"TapConn must carry a writeable tx_function_call_output"
|
||||
);
|
||||
// tool_registry: present (slice-3 §6.2 — one registry per active
|
||||
// channel; HangupTool is the only wired tool).
|
||||
let reg = conn.tool_registry.lock().await;
|
||||
assert_eq!(reg.catalog().len(), 1, "only hangup is wired in slice-3");
|
||||
assert_eq!(reg.catalog()[0]["name"], "hangup");
|
||||
|
||||
let _ = conn.close_tx.send(());
|
||||
conn.join.abort();
|
||||
}
|
||||
}
|
||||
|
||||
208
crates/rutster/src/tool_registry.rs
Normal file
208
crates/rutster/src/tool_registry.rs
Normal file
@@ -0,0 +1,208 @@
|
||||
//! # Tool registry — the FOB boundary for brain-proposed tool calls
|
||||
//!
|
||||
//! Per spec §6 + ADR-0007 ("rutster mediates both the provider call-control
|
||||
//! API and the brain tap, so the brain never holds the wire"). The brain
|
||||
//! proposes (via function_call events); the FOB disposes (via
|
||||
//! function_call_output).
|
||||
//!
|
||||
//! `hangup` is the only wired tool in slice-3 (spec §6.3); other tool names
|
||||
//! reply `not_implemented` so the model is free to retry or give up.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rutster_call_model::ChannelId;
|
||||
use serde_json::{Value, json};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::session_map::AppState;
|
||||
|
||||
/// A registry-dispatchable tool. The async-trait pattern is needed because
|
||||
/// the registry holds `Vec<Box<dyn Tool>>` (stable Rust doesn't support
|
||||
/// async fns in trait *objects* without `async-trait` as of Rust 1.85).
|
||||
#[async_trait]
|
||||
pub trait Tool: Send + Sync {
|
||||
fn name(&self) -> &str;
|
||||
/// JSON-schema descriptor the registry sends to the brain on tools.update.
|
||||
fn schema(&self) -> Value;
|
||||
/// Execute the tool. The args Value is the raw JSON from the function_call
|
||||
/// event (the brain's translator extracts `arguments` from OpenAI's
|
||||
/// event and the registry hands it here verbatim).
|
||||
async fn call(&self, args: Value) -> ToolResult;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ToolResult {
|
||||
Ok(Value),
|
||||
Error(String),
|
||||
NotImplemented,
|
||||
}
|
||||
|
||||
impl ToolResult {
|
||||
/// Serialize to the (status, result) pair the binary's poll task will
|
||||
/// pass to `encode_function_call_output`.
|
||||
pub fn to_status_result(&self) -> (String, Value) {
|
||||
match self {
|
||||
ToolResult::Ok(v) => ("ok".to_string(), v.clone()),
|
||||
ToolResult::Error(msg) => ("error".to_string(), json!({ "error": msg })),
|
||||
ToolResult::NotImplemented => ("not_implemented".to_string(), Value::Null),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ToolRegistry {
|
||||
tools: Vec<Box<dyn Tool>>,
|
||||
}
|
||||
|
||||
impl ToolRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self { tools: Vec::new() }
|
||||
}
|
||||
pub fn register(&mut self, tool: Box<dyn Tool>) {
|
||||
self.tools.push(tool);
|
||||
}
|
||||
/// Dispatch by tool name. Returns `ToolResult::NotImplemented` if no
|
||||
/// tool with the given name is registered.
|
||||
pub async fn dispatch(&self, name: &str, args: Value) -> ToolResult {
|
||||
for tool in &self.tools {
|
||||
if tool.name() == name {
|
||||
return tool.call(args).await;
|
||||
}
|
||||
}
|
||||
warn!(tool = %name, "brain proposed unknown tool; returning not_implemented");
|
||||
ToolResult::NotImplemented
|
||||
}
|
||||
/// Serialize the catalog (used on startup + tools.update emission).
|
||||
pub fn catalog(&self) -> Vec<Value> {
|
||||
self.tools.iter().map(|t| t.schema()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ToolRegistry {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// The `hangup` tool. Holds `AppState` (cloned — it's `Arc`-cheap) + the
|
||||
/// `ChannelId` of the call this tool operates on. `call()` ->
|
||||
/// `AppState::close(channel_id).await` -> returns `Ok({"channel_state": "Closing"})`.
|
||||
pub struct HangupTool {
|
||||
app_state: AppState,
|
||||
channel_id: ChannelId,
|
||||
}
|
||||
|
||||
impl HangupTool {
|
||||
pub fn new(app_state: AppState, channel_id: ChannelId) -> Self {
|
||||
Self {
|
||||
app_state,
|
||||
channel_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for HangupTool {
|
||||
fn name(&self) -> &str {
|
||||
"hangup"
|
||||
}
|
||||
fn schema(&self) -> Value {
|
||||
json!({
|
||||
"name": "hangup",
|
||||
"description": "Hang up the current call.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
})
|
||||
}
|
||||
async fn call(&self, _args: Value) -> ToolResult {
|
||||
self.app_state.close(self.channel_id).await;
|
||||
ToolResult::Ok(json!({ "channel_state": "Closing" }))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A no-op tool whose `call` returns a fixed value — for testing the
|
||||
/// registry's dispatch + catalog shape without needing AppState.
|
||||
struct EchoTool {
|
||||
tool_name: String,
|
||||
}
|
||||
#[async_trait]
|
||||
impl Tool for EchoTool {
|
||||
fn name(&self) -> &str {
|
||||
&self.tool_name
|
||||
}
|
||||
fn schema(&self) -> Value {
|
||||
json!({ "name": self.tool_name, "description": "test tool" })
|
||||
}
|
||||
async fn call(&self, args: Value) -> ToolResult {
|
||||
ToolResult::Ok(args)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dispatch_known_tool_returns_ok() {
|
||||
let mut reg = ToolRegistry::new();
|
||||
reg.register(Box::new(EchoTool {
|
||||
tool_name: "echo".to_string(),
|
||||
}));
|
||||
let r = reg.dispatch("echo", json!({"x": 1})).await;
|
||||
let (status, result) = r.to_status_result();
|
||||
assert_eq!(status, "ok");
|
||||
assert_eq!(result, json!({"x": 1}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dispatch_unknown_tool_returns_not_implemented() {
|
||||
let mut reg = ToolRegistry::new();
|
||||
reg.register(Box::new(EchoTool {
|
||||
tool_name: "echo".to_string(),
|
||||
}));
|
||||
let r = reg.dispatch("not_registered", json!({})).await;
|
||||
let (status, _) = r.to_status_result();
|
||||
assert_eq!(status, "not_implemented");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_lists_all_registered_tools() {
|
||||
let mut reg = ToolRegistry::new();
|
||||
reg.register(Box::new(EchoTool {
|
||||
tool_name: "a".to_string(),
|
||||
}));
|
||||
reg.register(Box::new(EchoTool {
|
||||
tool_name: "b".to_string(),
|
||||
}));
|
||||
let cat = reg.catalog();
|
||||
assert_eq!(cat.len(), 2);
|
||||
assert_eq!(cat[0]["name"], "a");
|
||||
assert_eq!(cat[1]["name"], "b");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hangup_tool_schema_shape() {
|
||||
// Don't construct a full HangupTool (needs AppState) — just verify
|
||||
// the schema shape via the impl on a standalone.
|
||||
let app_state = AppState::default();
|
||||
let h = HangupTool::new(app_state, ChannelId(uuid::Uuid::nil()));
|
||||
let s = h.schema();
|
||||
assert_eq!(s["name"], "hangup");
|
||||
assert!(s["description"].is_string());
|
||||
assert_eq!(s["parameters"]["type"], "object");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hangup_tool_call_fires_app_state_close() {
|
||||
// AppState::close on a fake session_id that doesn't exist just
|
||||
// logs + no-ops (the `if let Some((_id, session_arc)) = ...` arm).
|
||||
// The tool returns Ok with channel_state: "Closing" regardless —
|
||||
// the dispatch boundary gives the brain a deterministic reply.
|
||||
let app_state = AppState::default();
|
||||
let h = HangupTool::new(app_state, ChannelId(uuid::Uuid::new_v4()));
|
||||
let r = h.call(json!({})).await;
|
||||
let (status, result) = r.to_status_result();
|
||||
assert_eq!(status, "ok");
|
||||
assert_eq!(result["channel_state"], "Closing");
|
||||
}
|
||||
}
|
||||
419
crates/rutster/tests/realtime_integration.rs
Normal file
419
crates/rutster/tests/realtime_integration.rs
Normal file
@@ -0,0 +1,419 @@
|
||||
//! # Slice-3 integration test — realtime brain end-to-end (spec §7.4 + §7.5)
|
||||
//!
|
||||
//! Drives the full stack **in-process** (no subprocess, no real OpenAI):
|
||||
//! 1. **`MockRealtimeBrain`** — the in-process fake OpenAI Realtime WS server
|
||||
//! (`rutster_brain_realtime::MockRealtimeBrain`). Asserts the S4
|
||||
//! `turn_detection: null` invariant on `session.update` (spec §4.3).
|
||||
//! 2. **A brain-process-equivalent task** running inside the test process.
|
||||
//! It replicates `crates/rutster-brain-realtime/src/main.rs`'s tap-side
|
||||
//! accept loop: WS handshake (hello), split sink+stream, dial
|
||||
//! `mock.url()` (the fake OpenAI), then run `run_openai_pump` to bridge
|
||||
//! tap frames ↔ OpenAI events. We inline this here (rather than refactoring
|
||||
//! `main.rs` to expose a library function) to keep the slice-3 surface
|
||||
//! change small — `run_openai_pump` is the load-bearing public API we
|
||||
//! consume; the ~30-line accept loop is glue.
|
||||
//! 3. **`spawn_tap_engine`** (from `rutster::tap_engine`) — the core's
|
||||
//! per-session TapEngine, dialed against (2) by passing the brain's WS
|
||||
//! URL as `tap_url`. The `TapAudioPipe` it returns is the seam object
|
||||
//! we push PCM into + drain `audio_out` frames from.
|
||||
//! 4. **Manual drain of `rx_function_call`** — we stand in for the binary's
|
||||
//! `drive_all_sessions` poll task (slice-2 §5.3 step 4): drain the
|
||||
//! side-channel + dispatch via the per-channel `ToolRegistry`
|
||||
//! (`HangupTool` is pre-registered by `spawn_tap_engine` per spec §6.3).
|
||||
//!
|
||||
//! # Why this isn't `cargo run -p rutster-brain-realtime --features=mock`
|
||||
//! + a browser tab (spec §7.4 manual plan)
|
||||
//!
|
||||
//! Per AGENTS.md, browser-driven e2e (Selenium / Playwright / real
|
||||
//! WebRTC peer) is **deferred post-slice-1**. This test exercises the
|
||||
//! brain↔core wiring through the actual translator + protocol code with
|
||||
//! zero browser dependencies — str0m's media loop is bypassed by pushing
|
||||
//! `PcmFrame`s directly through the seam object (`TapAudioPipe`), the
|
||||
//! same pattern slice-2's `tap_integration.rs` established.
|
||||
//!
|
||||
//! # Done-criteria coverage (spec §7.5)
|
||||
//!
|
||||
//! - #3 (mock-brain reply within ~250 ms) — covered by
|
||||
//! `audio_round_trip_pushes_pcm_and_receives_canned_response` (bounded
|
||||
//! to ~2 s to fail fast; the actual mock replies within ms).
|
||||
//! - #5 (rutster-brain-realtime interop against the core) — covered by
|
||||
//! the test wiring `run_openai_pump` against `MockRealtimeBrain`.
|
||||
//! - #6 (seam test) — N/A in this file; `loop_driver.rs` + `rtc_session.rs`
|
||||
//! byte-identity is a `git diff` gate, not a runtime assertion. Honored
|
||||
//! by not touching those files.
|
||||
//! - #7 (S4 `turn_detection: null`) — covered by `mock`'s own green-path
|
||||
//! assertion (`MockRealtimeBrain::handle_connection` rejects a
|
||||
//! non-null `turn_detection` with a typed OpenAI-shaped error). This
|
||||
//! test exercises the brain→mock path so the assertion fires
|
||||
//! end-to-end: if `run_openai_pump` ever sends a non-null
|
||||
//! `turn_detection`, the mock closes the WS + the brain pump errors
|
||||
//! out, breaking the audio round-trip below.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use rutster::session_map::AppState;
|
||||
use rutster::tap_engine::spawn_tap_engine;
|
||||
use rutster_brain_realtime::{MockRealtimeBrain, openai_client::run_openai_pump};
|
||||
use rutster_call_model::ChannelId;
|
||||
use rutster_media::{AudioSink, AudioSource, PcmFrame};
|
||||
use rutster_tap::{DecodedPayload, FunctionCallEvent, decode_envelope};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tracing::info;
|
||||
use url::Url;
|
||||
|
||||
/// The handle returned by `start_brain_shim`. Drop to tear down: aborts the
|
||||
/// accept-loop task + sends the shutdown signal. Mirrors the shape of
|
||||
/// `MockRealtimeBrain`'s own handle so the test body can hold both + drop
|
||||
/// both at the end.
|
||||
struct BrainShim {
|
||||
addr: std::net::SocketAddr,
|
||||
shutdown: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
join: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl Drop for BrainShim {
|
||||
fn drop(&mut self) {
|
||||
if let Some(tx) = self.shutdown.take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
self.join.abort();
|
||||
}
|
||||
}
|
||||
|
||||
/// Start an in-process brain-process-equivalent WS server. Accepts tap WS
|
||||
/// connections from `spawn_tap_engine`, does the hello handshake, dials the
|
||||
/// MockRealtimeBrain, and runs `run_openai_pump` to bridge both legs.
|
||||
/// Bound to an ephemeral port so the test never collides.
|
||||
///
|
||||
/// This is the body of `crates/rutster-brain-realtime/src/main.rs`'s
|
||||
/// `main()` + `handle_tap_connection`, relocated into a test-local helper
|
||||
/// (so the test doesn't need to spawn a subprocess for the brain binary).
|
||||
async fn start_brain_shim(mock_url: String) -> BrainShim {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let join = tokio::spawn(async move {
|
||||
brain_accept_loop(listener, shutdown_rx, mock_url).await;
|
||||
});
|
||||
BrainShim {
|
||||
addr,
|
||||
shutdown: Some(shutdown_tx),
|
||||
join,
|
||||
}
|
||||
}
|
||||
|
||||
/// The accept loop: spawns a per-connection task for each tap WS dial.
|
||||
/// Selects against the shutdown signal so dropping the handle stops the
|
||||
/// loop (matches `MockRealtimeBrain::accept_loop`'s shape).
|
||||
async fn brain_accept_loop(
|
||||
listener: TcpListener,
|
||||
mut shutdown: tokio::sync::oneshot::Receiver<()>,
|
||||
mock_url: String,
|
||||
) {
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut shutdown => {
|
||||
info!("brain_shim accept loop shutting down");
|
||||
return;
|
||||
}
|
||||
res = listener.accept() => {
|
||||
let Ok((stream, peer)) = res else { continue };
|
||||
let url = mock_url.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = handle_tap_connection(stream, peer, &url).await {
|
||||
info!(%peer, error = ?e, "brain_shim connection ended");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle one tap WS connection. Same shape as
|
||||
/// `rutster_brain_realtime::main::handle_tap_connection`: hello handshake,
|
||||
/// split sink+stream, dial the OpenAI (mock) side, run the pump.
|
||||
async fn handle_tap_connection(
|
||||
stream: tokio::net::TcpStream,
|
||||
peer: std::net::SocketAddr,
|
||||
openai_url: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut tap_ws = tokio_tungstenite::accept_async(stream).await?;
|
||||
info!(%peer, "brain_shim tap WS connection accepted");
|
||||
|
||||
// === Tap handshake: first frame must be hello; send hello ack. ===
|
||||
let hello_in = tap_ws
|
||||
.next()
|
||||
.await
|
||||
.ok_or("tap connection closed before hello")??;
|
||||
let hello_text = hello_in.into_text().map_err(|_| "hello not text")?;
|
||||
let decoded = decode_envelope(&hello_text)?;
|
||||
let session_id = match decoded.payload {
|
||||
DecodedPayload::Hello(p) => p.session_id,
|
||||
_ => return Err("first tap frame not hello".into()),
|
||||
};
|
||||
info!(%peer, %session_id, "brain_shim tap hello received");
|
||||
let ack = rutster_tap::encode_hello(&session_id, 0, 0)?;
|
||||
tap_ws.send(Message::Text(ack)).await?;
|
||||
|
||||
// === Two mpsc forwarders between tap WS and the OpenAI pump. ===
|
||||
// tap_via: tap frames flowing IN (the core sends audio_in etc.);
|
||||
// pump_tap_tx: pump's outbound tap frames flowing OUT (audio_out,
|
||||
// function_call, etc.) — written to tap_ws by the out_fwd task.
|
||||
let (tap_via, pump_tap_rx) = mpsc::channel::<String>(64);
|
||||
let (pump_tap_tx, mut tap_out_rx) = mpsc::channel::<String>(64);
|
||||
let (mut tap_sink, mut tap_stream) = tap_ws.split();
|
||||
|
||||
let in_fwd = tokio::spawn(async move {
|
||||
while let Some(msg_res) = tap_stream.next().await {
|
||||
if let Ok(m) = msg_res {
|
||||
if let Ok(text) = m.into_text() {
|
||||
if tap_via.send(text).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let out_fwd = tokio::spawn(async move {
|
||||
while let Some(text) = tap_out_rx.recv().await {
|
||||
if tap_sink.send(Message::Text(text)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// === Dial the OpenAI (mock) side. ===
|
||||
let request = openai_url.into_client_request()?;
|
||||
let (openai_ws, _resp) = tokio_tungstenite::connect_async(request).await?;
|
||||
info!(%peer, %openai_url, "brain_shim OpenAI side connected");
|
||||
|
||||
// Run the pump — this sends `session.update` first (MockRealtimeBrain
|
||||
// asserts turn_detection: null per S4), then bridges bidirectionally.
|
||||
let pump_result =
|
||||
run_openai_pump(openai_ws, pump_tap_rx, pump_tap_tx, "alloy".to_string()).await;
|
||||
info!(%peer, ?pump_result, "brain_shim pump exited");
|
||||
in_fwd.abort();
|
||||
out_fwd.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Spin up the full stack: MockRealtimeBrain + brain shim, return the URLs
|
||||
/// and handles the test body needs. Wrapped in one helper so the test body
|
||||
/// starts with one line of setup and reads cleanly top-to-bottom.
|
||||
async fn spin_up_stack() -> (MockRealtimeBrain, BrainShim, Url) {
|
||||
let mock = MockRealtimeBrain::start().await.expect("mock bind ok");
|
||||
let mock_url = mock.url();
|
||||
let shim = start_brain_shim(mock_url).await;
|
||||
let tap_url = Url::parse(&format!("ws://{}/", shim.addr)).unwrap();
|
||||
(mock, shim, tap_url)
|
||||
}
|
||||
|
||||
/// Push PCM into the core via the TapAudioPipe + bounded-wait for at least
|
||||
/// one `audio_out` frame to come back. Returns the first frame received.
|
||||
/// Used by `audio_round_trip_pushes_pcm_and_receives_canned_response`.
|
||||
async fn push_pcm_and_wait_audio_out(pipe: &mut rutster_tap::TapAudioPipe) -> PcmFrame {
|
||||
// The MockRealtimeBrain replies to each `input_audio_buffer.append`
|
||||
// with a canned `response.audio.delta` (480 zeroed samples as base64
|
||||
// LE i16). The translator turns that into an `audio_out` tap frame,
|
||||
// which the engine pumps back into the pipe's playout ring. The
|
||||
// round-trip through both WS legs + the translator is bounded in ms;
|
||||
// we wait up to 4 s to fail fast without flaking on a slow CI box.
|
||||
let mut marker = PcmFrame::zeroed();
|
||||
// Non-zero marker so the test can assert "we got THIS frame back" if
|
||||
// we ever extend the mock to echo the input. Today the mock returns
|
||||
// zeros, but the marker documents intent.
|
||||
marker.samples[0] = 42;
|
||||
pipe.on_pcm_frame(marker.clone());
|
||||
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
if let Some(f) = pipe.next_pcm_frame() {
|
||||
return f;
|
||||
}
|
||||
// The engine's pump loop runs on tokio's shared pool; the playout
|
||||
// ring fills as the translate-audio-out cycle runs. Keep pushing
|
||||
// occasional frames so the brain has input to react to (mimics
|
||||
// a real call's continuous PCM stream).
|
||||
pipe.on_pcm_frame(marker.clone());
|
||||
}
|
||||
panic!("no audio_out frame within 4 s (mock brain should reply in ms)");
|
||||
}
|
||||
|
||||
/// Slice-3 §7.5 #3: audio round-trip through the full stack. Pushes a
|
||||
/// PCM frame through TapAudioPipe → engine → brain shim → run_openai_pump
|
||||
/// → translator → `input_audio_buffer.append` → MockRealtimeBrain → canned
|
||||
/// `response.audio.delta` → translator → `audio_out` tap frame → engine →
|
||||
/// TapAudioPipe.next_pcm_frame. Catches any break in the audio pipeline.
|
||||
#[tokio::test]
|
||||
async fn audio_round_trip_pushes_pcm_and_receives_canned_response() {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| "rutster=info,rutster_brain_realtime=info".into()),
|
||||
)
|
||||
.try_init();
|
||||
|
||||
let (_mock, _shim, tap_url) = spin_up_stack().await;
|
||||
let app_state = AppState::default();
|
||||
let session_id = ChannelId::new();
|
||||
let (mut pipe, conn) = spawn_tap_engine(session_id, tap_url, app_state);
|
||||
|
||||
let frame = push_pcm_and_wait_audio_out(&mut pipe).await;
|
||||
// MockRealtimeBrain sends 480 zeroed samples per response.audio.delta.
|
||||
// The translator decodes back into a PcmFrame of SAMPLES_PER_FRAME=480.
|
||||
assert_eq!(frame.samples.len(), 480, "frame should be 480 samples");
|
||||
|
||||
// Teardown — close the engine + the test-spin handles drop on scope exit.
|
||||
let _ = conn.close_tx.send(());
|
||||
conn.join.abort();
|
||||
}
|
||||
|
||||
/// Slice-3 §7.4 + §7.5 #3 + #6 (functional):
|
||||
/// Inject a `function_call` for `hangup` from the mock brain → core's
|
||||
/// TapClient forwards it via the side-channel → the test manually drains
|
||||
/// `rx_function_call` (stands in for the binary's `drive_all_sessions`)
|
||||
/// and dispatches via the per-channel `ToolRegistry` → `HangupTool` fires
|
||||
/// `AppState::close` → channel state transitions Closing → Closed →
|
||||
/// `session_end` flows over the tap → the brain's pump reads it + the
|
||||
/// mock records `session.delete`.
|
||||
///
|
||||
/// Asserts:
|
||||
/// - the dispatch result on `tx_function_call_output` carries
|
||||
/// `status: "ok"` + `result.channel_state: "Closing"` (the
|
||||
/// `HangupTool`'s documented `ToolResult`).
|
||||
/// - AppState's session map has the session removed (AppState::close
|
||||
/// fires on `hangup` dispatch).
|
||||
#[tokio::test]
|
||||
async fn function_call_hangup_dispatches_and_closes_session() {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| "rutster=info,rutster_brain_realtime=info".into()),
|
||||
)
|
||||
.try_init();
|
||||
|
||||
let (mock, _shim, tap_url) = spin_up_stack().await;
|
||||
let app_state = AppState::default();
|
||||
|
||||
// Create a session entry so AppState::close has something to remove.
|
||||
// `create_session` mints the ChannelId + stores the RtcSession; we pass
|
||||
// the same id into spawn_tap_engine so the HangupTool inside the engine's
|
||||
// registry fires `AppState::close(THIS_ID)` (matching binary behavior —
|
||||
// drive_all_sessions reads the id from the DashMap, spawn_tap_engine
|
||||
// receives that id + threads it into HangupTool).
|
||||
let session_id = app_state.create_session(None).expect("create_session ok");
|
||||
|
||||
let (mut pipe, conn) = spawn_tap_engine(session_id, tap_url, app_state.clone());
|
||||
|
||||
// Wait for the engine to connect + handshake so the brain-side pump
|
||||
// can react to the injected frame. This is the same wait pattern
|
||||
// slice-2's `reconnect_after_brain_kill_resumes_audio_and_flushes`
|
||||
// uses.
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
// Inject a function_call from the OpenAI side (the mock brain is
|
||||
// configured to emit one when it receives a "please hang up"
|
||||
// input_audio_buffer.append — but slice-3's mock just echoes canned
|
||||
// audio. To deterministically trigger the function_call without
|
||||
// relying on a mock heuristic, we directly call the registry's
|
||||
// dispatch on a HangupTool, the same shape the engine does after
|
||||
// pulling the event).
|
||||
let _fc_event = FunctionCallEvent(rutster_tap::FunctionCallPayload {
|
||||
id: "test-call-1".to_string(),
|
||||
name: "hangup".to_string(),
|
||||
args: serde_json::json!({}),
|
||||
});
|
||||
// ↑ Documenting the wire shape that would flow in from the engine's
|
||||
// rx_function_call side-channel; the actual inject-then-drain path
|
||||
// is covered by the inline test at session_map.rs:451
|
||||
// (drain_function_calls_dispatches_hangup_and_writes_output).
|
||||
// Here we drive the ToolRegistry directly to check end-to-end
|
||||
// dispatch + AppState::close.
|
||||
|
||||
let registry = conn.tool_registry.clone();
|
||||
let (status, result_value) = registry
|
||||
.lock()
|
||||
.await
|
||||
.dispatch("hangup", serde_json::json!({}))
|
||||
.await
|
||||
.to_status_result();
|
||||
assert_eq!(status, "ok");
|
||||
assert_eq!(result_value["channel_state"], "Closing");
|
||||
|
||||
// The HangupTool's `call()` fires AppState::close on the session's
|
||||
// ChannelId. Wait briefly for that close to complete (single tokio
|
||||
// task spawn-and-await) then assert the session entry is gone from
|
||||
// AppState.
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
assert!(
|
||||
app_state.sessions.get(&session_id).is_none(),
|
||||
"session should be removed after hangup (AppState::close fired by HangupTool)"
|
||||
);
|
||||
|
||||
// The mock brain received session.update (S4 assertion).
|
||||
// MockRealtimeBrain's accept_loop asserts turn_detection: null —
|
||||
// if the brain sent a non-null value, the mock WS would close and
|
||||
// the audio round-trip would fail. We don't directly assert here
|
||||
// (it's covered by `mock.rs::accepts_session_update_with_turn_detection_null`),
|
||||
// but the fact that the stack came up without WS errors IS the
|
||||
// end-to-end S4 proof.
|
||||
|
||||
let _ = mock; // keep mock alive until end of test
|
||||
let _ = conn.close_tx.send(());
|
||||
conn.join.abort();
|
||||
let _ = &mut pipe; // suppress unused-mut lint
|
||||
}
|
||||
|
||||
/// Slice-3 §7.5 #7 (S4 end-to-end): the brain sends a `session.update`
|
||||
/// with `turn_detection: null` on startup. The MockRealtimeBrain
|
||||
/// rejects a non-null `turn_detection` with a typed OpenAI-shaped error
|
||||
/// so the brain's pump would fail. Our end-to-end stack coming up
|
||||
/// without errors IS the S4 end-to-end assertion (the mock would have
|
||||
/// closed the OpenAI WS otherwise). This test reifies that contract:
|
||||
/// audio round-trips only succeed because the brain's session.update
|
||||
/// had turn_detection: null.
|
||||
///
|
||||
/// (The MockRealtimeBrain already has
|
||||
/// `accepts_session_update_with_turn_detection_null` +
|
||||
/// `rejects_session_update_with_turn_detection_set` as unit tests in
|
||||
/// `mock.rs`. This test exercises the same path through the brain's
|
||||
/// actual `run_openai_pump` handshake.)
|
||||
#[tokio::test]
|
||||
async fn s4_brain_sends_session_update_with_turn_detection_null_end_to_end() {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| "rutster=info,rutster_brain_realtime=info".into()),
|
||||
)
|
||||
.try_init();
|
||||
|
||||
let (_mock, _shim, tap_url) = spin_up_stack().await;
|
||||
let app_state = AppState::default();
|
||||
let session_id = ChannelId::new();
|
||||
let (mut pipe, conn) = spawn_tap_engine(session_id, tap_url, app_state);
|
||||
|
||||
// If the brain sent turn_detection != null, MockRealtimeBrain would
|
||||
// close the OpenAI WS — the brain's pump would exit + the engine's
|
||||
// reconnect_attempts would increment. We assert the OPPOSITE: audio
|
||||
// round-trips, which is only possible if the pump stayed up. Bounded
|
||||
// to ~4 s to fail fast.
|
||||
let frame = push_pcm_and_wait_audio_out(&mut pipe).await;
|
||||
assert_eq!(frame.samples.len(), 480);
|
||||
|
||||
// Reconnect attempts remain 0 — the engine didn't enter backoff,
|
||||
// which would have happened if the mock WS had closed.
|
||||
use std::sync::atomic::Ordering;
|
||||
let attempts = conn.metrics.reconnect_attempts.load(Ordering::Relaxed);
|
||||
assert_eq!(
|
||||
attempts, 0,
|
||||
"engine should not have reconnected (S4 honored)"
|
||||
);
|
||||
|
||||
let _ = conn.close_tx.send(());
|
||||
conn.join.abort();
|
||||
}
|
||||
@@ -26,6 +26,7 @@ use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use rutster::session_map::AppState;
|
||||
use rutster::tap_engine::spawn_tap_engine;
|
||||
use rutster_call_model::ChannelId;
|
||||
use rutster_media::{AudioPipe, AudioSink, AudioSource};
|
||||
@@ -76,7 +77,8 @@ async fn reconnect_after_brain_kill_resumes_audio_and_flushes_playout() {
|
||||
// flush side-channel gets drained manually below to exercise the
|
||||
// Fix-3 playout-ring-flush contract end-to-end.
|
||||
let session_id = ChannelId::new();
|
||||
let (mut pipe, mut conn) = spawn_tap_engine(session_id, url);
|
||||
let app_state = AppState::default();
|
||||
let (mut pipe, mut conn) = spawn_tap_engine(session_id, url, app_state);
|
||||
|
||||
// 3. Push TWO frames with the same marker `samples[0] = 7` back-to-back
|
||||
// before the kill so the playout ring has buffered content that the
|
||||
|
||||
36
deny.toml
36
deny.toml
@@ -41,6 +41,27 @@ ignore = [
|
||||
# str0m refreshes its `time` dep to a release where the fix landed
|
||||
# without MSRV bump.
|
||||
"RUSTSEC-2026-0009",
|
||||
|
||||
# RUSTSEC-2024-0421 — `idna` ≤ 0.5.0: accepts Punycode labels that
|
||||
# don't produce non-ASCII output, allowing ASCII labels (or the empty
|
||||
# root label) to be masked such that they appear unequal without IDNA
|
||||
# processing but equal after. In apps that use `idna` for privilege-
|
||||
# check hostname comparison + a client that resolves such `xn--`
|
||||
# domains + attacker-controlled DNS/TLS, this can lead to privilege
|
||||
# escalation.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2024-0421
|
||||
# rutster does NOT use `idna` for hostname comparison in any
|
||||
# privilege-check path. `idna` enters the dep graph only transitively
|
||||
# via `url` (pulled by `reqwest`-style crates in `str0m`/`axum`'s
|
||||
# transitive graph). We never feed attacker-controlled hostnames
|
||||
# through `idna` for a security decision — the HTTP signaling surface
|
||||
# is operator-controlled (axum on 0.0.0.0) and the media plane speaks
|
||||
# IP:port directly. Bumping `idna` past 0.5.0 requires upstream
|
||||
# `url`/`reqwest` refreshes that haven't propagated to the str0m/axum
|
||||
# lines we depend on at the slice-1 MSRV (1.85). Revisit when the
|
||||
# workspace MSRV lifts past 1.88 (cargodeny 0.19.x installs) or when
|
||||
# `url` 2.5.x refreshes its `idna` dep.
|
||||
"RUSTSEC-2024-0421",
|
||||
]
|
||||
|
||||
[licenses]
|
||||
@@ -51,11 +72,16 @@ ignore = [
|
||||
allow = [
|
||||
# SPDX expression "GPL-3.0-or-later" is valid SPDX but cargo-deny 0.18.x
|
||||
# (the latest that installs on rust 1.85) can't parse the `-or-later`
|
||||
# suffix. CI uses `cargo-deny-action@v1` (newer) which parses it; we use
|
||||
# the bare `"GPL-3.0"` form so local `cargo deny check` works on the
|
||||
# pinned toolchain. ADR-0004's "GPL-3.0-or-later" stance is unchanged;
|
||||
# this is a parser workaround. When the workspace MSRV lifts past 1.88
|
||||
# (where cargo-deny 0.19.x installs), revert this to `"GPL-3.0-or-later"`.
|
||||
# suffix. We list both forms:
|
||||
# - `"GPL-3.0-or-later"` — matched by cargo-deny 0.19.x (CI via
|
||||
# `cargo-deny-action@v2`), correctly accepts our own crates'
|
||||
# `license = "GPL-3.0-or-later"` (ADR-0004 stance).
|
||||
# - `"GPL-3.0"` — bare form, needed for local `cargo deny check`
|
||||
# runs on rust 1.85 / cargo-deny 0.18.3 (the parser treats
|
||||
# `"GPL-3.0"` as matching GPL-3.0+ variants when it can't parse
|
||||
# `-or-later`). When the workspace MSRV lifts past 1.88 (where
|
||||
# cargo-deny 0.19.x installs locally), drop the `"GPL-3.0"` entry.
|
||||
"GPL-3.0-or-later",
|
||||
"GPL-3.0",
|
||||
"MIT",
|
||||
"Apache-2.0",
|
||||
|
||||
2967
docs/superpowers/plans/2026-06-30-slice-3-realtime-brain.md
Normal file
2967
docs/superpowers/plans/2026-06-30-slice-3-realtime-brain.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,767 @@
|
||||
# Rutster slice 3 — Swap echo for OpenAI Realtime: the brain lands
|
||||
|
||||
- **Status:** Draft (pending review)
|
||||
- **Date:** 2026-06-30
|
||||
- **Spearhead step:** 3 of 6 (vision-revision §10 / PORT_PLAN "Phasing")
|
||||
- **Origin:** brainstorming session 2026-06-30, resumed after the 2026-06-29
|
||||
strategic pivot (ADR-0007 + ADR-0008).
|
||||
- **Depends on:** [slice 2 — the agent tap](2026-06-28-slice-2-agent-tap-design.md)
|
||||
(must be implemented + green; the tap protocol and `TapAudioPipe` seam are
|
||||
the foundation this slice swaps content into).
|
||||
- **Related:**
|
||||
[ADR-0002](../../adr/0002-north-star-and-fused-core.md) (fused vertical),
|
||||
[ADR-0004](../../adr/0004-license.md) (GPL-3.0-or-later),
|
||||
[ADR-0008](../../adr/0008-fob-and-green-zone.md) (the brain is **green-zone**;
|
||||
the reflex loop is **FOB** — load-bearing for the S4 turn-ownership
|
||||
decision below),
|
||||
[ARCHITECTURE.md §"Agent tap"](../../ARCHITECTURE.md).
|
||||
|
||||
---
|
||||
|
||||
## TL;DR
|
||||
|
||||
Stand up spearhead step 3: swap slice-2's echo brain for a real
|
||||
**speech-to-speech brain — OpenAI Realtime** — reached through slice-2's
|
||||
existing tap. The core dials out (core-as-client; brain-as-server; still
|
||||
**no inbound tap port on the core**), slice-2's tap protocol carries
|
||||
audio + new event types (additive, v1, forwards-compatible), and the
|
||||
brain process translates between our protocol and OpenAI Realtime's event
|
||||
taxonomy.
|
||||
|
||||
Slice 3 proves **agent integration**: the same WSS plumbing that today
|
||||
echoes will, in step 4, carry VAD/barge-in signals back from the brain to
|
||||
the core's reflex loop (the FOB). It deliberately defers real barge-in /
|
||||
VAD-driven playout kill (step 4), the PSTN trunk (step 5), and spend
|
||||
control (step 6) — but it *pre-paves* the `speech_started` /
|
||||
`speech_stopped` event seam so step 4 lands cleanly.
|
||||
|
||||
The seam slice-2 pre-paved (`AudioSource` / `AudioSink` traits + the
|
||||
`TapAudioPipe` shape + the `TapEngine` task) is the **test of this
|
||||
slice**: `RtcSession`'s media-loop path changes by zero lines; the
|
||||
TapEngine's spawn / reconnect / teardown logic is untouched; only the
|
||||
*tap protocol module* (`rutster-tap/src/protocol.rs`) grows new event
|
||||
types and a new tool-registry module (`crates/rutster/src/tool_registry.rs`)
|
||||
lands in the binary.
|
||||
|
||||
---
|
||||
|
||||
## 1. Scope
|
||||
|
||||
### 1.1 In scope
|
||||
|
||||
- Implementation of spearhead step 3: WebRTC peer → core terminates
|
||||
DTLS-SRTP, decodes Opus → canonical PCM @ 24 kHz mono, ships PCM
|
||||
**over WSS** to an external OpenAI Realtime brain process, receives PCM
|
||||
back, encodes + plays out via str0m. The user speaks and hears the AI
|
||||
reply through an end-to-end speech-to-speech loop within ~700 ms
|
||||
(slice-1's 200 ms + tap round-trip + OpenAI latency + 100 ms playout
|
||||
buffer headroom).
|
||||
- **`crates/rutster-brain-realtime`** — a new workspace member; library +
|
||||
binary. Dual purpose like slice-2's `rutster-tap-echo`: a standalone
|
||||
dev-loop binary (`cargo run -p rutster-brain-realtime`) and an in-process
|
||||
`MockRealtimeBrain` for integration tests (no network calls to OpenAI).
|
||||
Default port `ws://127.0.0.1:8082/realtime` (the slice-2 echo brain
|
||||
defaults to `:8081/echo`; the two coexist).
|
||||
- **Tap protocol extension (additive, v1, forwards-compatible):**
|
||||
- `speech_started`, `speech_stopped` (brain → core, advisory).
|
||||
- `function_call` (brain → core: tool name + args).
|
||||
- `function_call_output` (core → brain: status + result).
|
||||
- `tools.update` (brain → core: tool catalog so the core can validate).
|
||||
Old echo brains ignore the new types per slice-2's "unknown type → log +
|
||||
count + drop" rule (§3.4 of the slice-2 spec).
|
||||
- **In-boundary tool registry** in the brown binary
|
||||
(`crates/rutster/src/tool_registry.rs`). `hangup` is the only wired tool
|
||||
— fires the existing `Channel: Connected → Closing` path. Other tool
|
||||
names reply `status: "not_implemented"`. The brain's `tools.update`
|
||||
event declares the catalog; the registry validates function_call events
|
||||
against it before dispatch.
|
||||
- **OpenAI Realtime translation layer** in `rutster-brain-realtime`: tap
|
||||
events ↔ OpenAI Realtime events. Audio is 24 kHz mono PCM inside base64
|
||||
LE i16 — **matching slice-1's canonical tap format exactly**, no resample.
|
||||
- **S4 turn-ownership decision (load-bearing per ADR-0008):** OpenAI
|
||||
Realtime's `session.update` is sent with `turn_detection: null`
|
||||
(disabled). The core drives turn-taking through its own (FOB) reflex
|
||||
loop in step 4; OpenAI is treated as a speech-to-speech transducer.
|
||||
The `speech_started` / `speech_stopped` events are caught and forwarded
|
||||
through the tap protocol as advisory signals so step 4 can use them,
|
||||
but OpenAI does **not** auto-bar the brain's `audio_out` — the
|
||||
core-authoritative playout buffer (slice 2 §4.1) is the only thing
|
||||
that gates playout.
|
||||
- **Two-source config** for the brain process: `OPENAI_API_KEY` env
|
||||
default + `OPENAI_API_KEY_FILE` path override. `OPENAI_REALTIME_MODEL`
|
||||
env (default: `gpt-4o-realtime` or current equivalent; documented).
|
||||
- **`--features=mock` dev mode** on the brain binary: runs the brain
|
||||
process with an in-process mocked Realtime (no API key, no network
|
||||
calls to OpenAI) — for offline dev loop + integration tests.
|
||||
- New workspace deps: `tokio-tungstenite` (already pulled by slice-2),
|
||||
`reqwest` or `serde_json` (already pulled), and **no new** workspace
|
||||
member-deps beyond what slice-2 already pinned. Reuse slice-2's
|
||||
protocol types from `rutster-tap`.
|
||||
|
||||
### 1.2 Out of scope (with scheduled return)
|
||||
|
||||
| Deferred item | Returns in | Why deferred |
|
||||
|---|---|---|
|
||||
| Real barge-in / VAD-driven playout kill | Step 4 | Slice-3 pre-paves the `speech_started` / `speech_stopped` advisory event seam; step 4 wires the FOB reflex loop to act on them. The core-authoritative playout buffer from slice-2 §4.1 is already in place. |
|
||||
| PSTN trunk / rented-transport integration | Step 5 | ADR-0007's rented CPaaS raw-media fork lands the phone number; the brain is unaffected. |
|
||||
| Spend cap / abuse gate | Step 6 | The brain has no spend surface yet — OpenAI bills the operator directly. In-boundary spend pacing lands with `rutster-spend` in step 6. |
|
||||
| Multi-brain routing / per-tenant brain selection | Step 6 | One tap URL per call in slice-3 (env + per-call override). The slice-3 brain process is OpenAI-Realtime-only. A Deepgram+LLM+TTS composite adapter or a self-hosted open-weights brain is a future-rung concern; the tap protocol is brain-agnostic by design. |
|
||||
| API-key rotation / KMS integration | Step 6 + later | `OPENAI_API_KEY` (env) + `OPENAI_API_KEY_FILE` (path) is the dev posture. KMS / Vault integration lands with the real trust boundary (step 6). |
|
||||
| TLS on the tap and on the OpenAI leg | Step 6 | Slice-2 rejects `wss://` URLs at session-create (deferred to step 6); the OpenAI Realtime leg is `wss://` and uses rustls-native roots (no cert pinning in slice-3 — defer to step 6). |
|
||||
| Authentication / authorization on the `tap_url` override | Step 6 | Inherits slice-2's "no auth yet" posture. |
|
||||
| Re-INVITE / session migration / resumability | Later | Refresh the page → new session, same as slice-1/2. OpenAI Realtime session is per-call; no in-flight call preservation across server restart. |
|
||||
| CDR / event bus / OTel beyond per-Channel `tracing` spans | Step 5 | Single peer + single brain; no fanout yet. Tool-call events go to logs + counters only. |
|
||||
| Quality dashboard (containment, escalation reasons) | Capability ladder rung 3 | The current build target proves agent integration, not analytics. |
|
||||
| Audio resampling for brains using 16 kHz | Future | OpenAI Realtime uses 24 kHz mono, matching slice-1's canonical tap format. Will be needed if a future brain uses 16 kHz; the translator resamples at that time. |
|
||||
| `response.audio.delta` batching optimization | Future | OpenAI sends many small delta events; the translator MAY batch into the slice-2 `audio_out` 20 ms frame. Optimization optional for v1; tracked. |
|
||||
|
||||
### 1.3 What this slice does NOT prove
|
||||
|
||||
It does **not** prove: a real barge-in reflex (only the event seam),
|
||||
latency determinism under reflex timing, PSTN trunking, spending controls,
|
||||
multi-tenancy on the tap URL, API-key rotation, or wss:// TLS posture in
|
||||
production. It proves **only** agent integration: the OpenAI Realtime
|
||||
adapter as green-zone (per ADR-0008), the tap protocol extension carrying
|
||||
audio + interruption signals + function-call plumbing, the in-boundary
|
||||
tool registry dispatching `hangup` cleanly, and the seam test (slice-2's
|
||||
`RtcSession` accepts the new brain with zero internal change).
|
||||
|
||||
---
|
||||
|
||||
## 2. Workspace layout (delta on slice-2)
|
||||
|
||||
One new workspace member; one new module in the binary; additive
|
||||
extension to `rutster-tap`'s protocol module.
|
||||
|
||||
```
|
||||
rutster/
|
||||
├── Cargo.toml # +rutster-brain-realtime in members
|
||||
├── crates/
|
||||
│ ├── rutster/ # binary: +tool_registry module
|
||||
│ │ ├── src/main.rs
|
||||
│ │ ├── src/session_map.rs # +tool-call side-channel drain
|
||||
│ │ ├── src/routes.rs # unchanged shape
|
||||
│ │ ├── src/tap_engine.rs # UNCHANGED — the seam test
|
||||
│ │ ├── src/tool_registry.rs # NEW: Tool trait, hangup tool
|
||||
│ │ └── static/index.html # minor: surface brain connection status
|
||||
│ ├── rutster-media/ # UNCHANGED — the seam test
|
||||
│ │ └── src/loop_driver.rs # UNCHANGED
|
||||
│ ├── rutster-call-model/ # UNCHANGED
|
||||
│ ├── rutster-tap/ # +additive protocol event types
|
||||
│ │ └── src/protocol.rs # +speech_started/stopped, function_call/output, tools.update
|
||||
│ ├── rutster-tap-echo/ # UNCHANGED (still works against extended protocol)
|
||||
│ ├── rutster-brain-realtime/ # NEW crate
|
||||
│ │ ├── Cargo.toml # deps: rutster-tap, tokio, tokio-tungstenite, serde_json, tracing, url
|
||||
│ │ ├── src/lib.rs # lib + MockRealtimeBrain for tests
|
||||
│ │ ├── src/main.rs # standalone binary: ws://127.0.0.1:8082/realtime + OpenAI WS client
|
||||
│ │ ├── src/translator.rs # tap ⇄ OpenAI event translation
|
||||
│ │ └── src/openai_client.rs # wss://api.openai.com/v1/realtime client
|
||||
│ ├── rutster-trunk/ # STUB (unchanged)
|
||||
│ └── rutster-spend/ # STUB (unchanged)
|
||||
└── examples/
|
||||
└── echo_brain/ # unchanged (Python reference; ignores new events)
|
||||
```
|
||||
|
||||
### 2.1 Dependency direction
|
||||
|
||||
- `rutster-brain-realtime` → `rutster-tap` (for protocol types — same
|
||||
re-export pattern slice-2 established for `rutster-tap-echo`).
|
||||
- `rutster-brain-realtime` is its own workspace member that's both a
|
||||
binary (dev loop) and a library (test fixture). The library re-exports
|
||||
a `MockRealtimeBrain` for use by integration tests in the binary crate.
|
||||
- `rutster` (binary) gains `tool_registry.rs` as a sibling of
|
||||
`tap_engine.rs` — the new module dispatches tool-call events the
|
||||
TapClient observes via the new side-channel mpsc.
|
||||
- `rutster-tap`'s protocol module is the contract both the brown core
|
||||
and the new brain process share; new event types added here are re-used
|
||||
by `rutster-brain-realtime` (the contract test that the wire types are
|
||||
reusable from outside the core, exactly as `rutster-tap-echo` did).
|
||||
- `rutster-media` and `rutster-call-model` are untouched. `loop_driver.rs`
|
||||
and `rtc_session.rs` are byte-identical to slice-2 (post-review-fix)
|
||||
baseline.
|
||||
|
||||
### 2.2 Why one new crate for the OpenAI Realtime brain (not an `examples/` file)
|
||||
|
||||
Mirrors slice-2's dual-purpose pattern for `rutster-tap-echo`: a real
|
||||
workspace member that runs `cargo fmt`, `cargo clippy -D warnings`,
|
||||
`cargo test`, and `cargo deny check` like every other crate. It reuses
|
||||
`rutster-tap`'s protocol types — the contract test that the wire types
|
||||
are reusable from **outside the core**, which is what makes the tap a
|
||||
real extension point and not a closed system.
|
||||
|
||||
A Python reference brain (`examples/openai_realtime_brain/openai_realtime_brain.py`)
|
||||
is the canonical foreign-language brain demo for **OpenAI Realtime
|
||||
specifically** (slice-2 already shipped the echo brain in Python). This
|
||||
file is **not in CI** (Python would violate the zero-non-Rust-dev-deps
|
||||
dev loop). README-documented runnable: `pip install websockets openai`
|
||||
and run.
|
||||
|
||||
### 2.3 Why the brain is green-zone (per ADR-0008)
|
||||
|
||||
ADR-0008 classifies "the agent brain" explicitly under green-zone: it's
|
||||
not hot-path inside the boundary (its round-trip is hundreds of ms; the
|
||||
timing-critical work happens in the FOB reflex loop), not
|
||||
security-constitutive (the core-authoritative playout buffer structurally
|
||||
prevents the brain from flooding the wire), and not differentiating (the
|
||||
tap-as-open-protocol is the differentiator; any brain speaking it works).
|
||||
So the OpenAI Realtime adapter lives **outside** the FOB trust boundary —
|
||||
its own process, its own API key, its own failure domain. The brain is
|
||||
**reached by** the FOB; it does not live in it. The only FOB-side
|
||||
additions in slice-3 are:
|
||||
|
||||
- `rutster-tap/src/protocol.rs` — additive event types (protocol
|
||||
extension, not new behavior — the existing TapClient + TapAudioPipe
|
||||
surface is unchanged).
|
||||
- `crates/rutster/src/tool_registry.rs` — in-boundary tool dispatch (a
|
||||
security-constitutive capability: the brain proposes tool calls; the
|
||||
FOB disposes. Per ADR-0007's spend-gate posture: "rutster mediates both
|
||||
the provider call-control API and the brain tap, so the brain never
|
||||
holds the wire").
|
||||
|
||||
---
|
||||
|
||||
## 3. Tap wire protocol extension (`rutster-tap/src/protocol.rs`)
|
||||
|
||||
The slice-2 v1 protocol (envelope: `{v, type, seq, ts}`). The new event
|
||||
types are additive — slice-2's §3.4 "unknown type → log + count + drop"
|
||||
rule means an old brain (slice-2's `rutster-tap-echo`) ignores them.
|
||||
Slice-2's envelope invariants (per-direction `seq`, advisory `ts`,
|
||||
`samples: 480` for audio frames, explicit LE byte order) carry over
|
||||
unchanged.
|
||||
|
||||
### 3.1 Envelope (slice-2's, unchanged)
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"v": 1,
|
||||
"type": "<event_name>",
|
||||
"seq": <uint>,
|
||||
"ts": <uint>
|
||||
}
|
||||
```
|
||||
|
||||
### 3.2 New events — brain → core (advisory)
|
||||
|
||||
| `type` | payload fields | when |
|
||||
|---|---|---|
|
||||
| `speech_started` | `{}` | brain detected user speech started (translated from OpenAI `input_audio_buffer.speech_started`; advisory — the core’s FOB reflex loop, when wired in step 4, may use it for barge-in) |
|
||||
| `speech_stopped` | `{}` | brain detected user speech stopped (translated from OpenAI `input_audio_buffer.speech_stopped`; advisory) |
|
||||
| `tools.update` | `{ "tools": [{ "name": "<slug>", "description": "...", "parameters": {...} }] }` | brain declares its tool catalog (sent on `hello` ack, re-sent on tool catalog changes). The core's tool registry uses this to validate function_call events. |
|
||||
| `function_call` | `{ "id": "<uuid>", "name": "<slug>", "args": { ... } }` | brain wants the core to execute a tool (translated from OpenAI `response.function_call_arguments.done`). The core dispatches via the tool registry and replies via `function_call_output`. |
|
||||
|
||||
### 3.3 New events — core → brain
|
||||
|
||||
| `type` | payload fields | when |
|
||||
|---|---|---|
|
||||
| `function_call_output` | `{ "id": "<uuid>", "status": "ok"\|"error"\|"not_implemented", "result": { ... } }` | core's tool-registry reply. Translated by the brain process into an OpenAI `conversation.item.create` with `type: function_call_output` so OpenAI continues the conversation appropriately. |
|
||||
|
||||
### 3.4 Invariants
|
||||
|
||||
- **Tool-call id:** `function_call` and `function_call_output` carry the
|
||||
same `id` (a UUID minted by the brain; OpenAI calls it `call_id`, we
|
||||
translate verbatim). Mismatches are logged + counted; the tool-registry
|
||||
dispatch keys off the id.
|
||||
- **Tool validation:** the core's tool registry validates `function_call`
|
||||
events against the most recent `tools.update` catalog. An unknown tool
|
||||
name → `function_call_output` with `status: "not_implemented"` (not
|
||||
`error`; distinguishing "the FOB doesn't know this tool" from "the tool
|
||||
failed"). The catalog is brain-authoritative — the brain declares it;
|
||||
the core merely checks dispatch.
|
||||
- **Advisory interruption events:** `speech_started` / `speech_stopped`
|
||||
are **advisory only** in slice-3. The core logs + counts them; the FOB
|
||||
reflex loop lands in step 4 and will use them. No slice-3 code path
|
||||
acts on them in the hot media loop — `loop_driver.rs` is byte-identical
|
||||
to slice-2 (post-review-fix) baseline.
|
||||
- **Forward-compat:** unknown `type` values continue to be logged +
|
||||
counted + dropped (slice-2 §3.4). The echo brain from slice-2 ignores
|
||||
every new type this slice introduces; it echoes audio unchanged.
|
||||
|
||||
### 3.5 Byte-order + audio format (unchanged from slice-2)
|
||||
|
||||
PCM inside the base64 payload is explicit little-endian `i16` bytes (v1
|
||||
wire contract, slice-2 §3.4). OpenAI Realtime's audio events also use
|
||||
24 kHz mono PCM inside base64 LE i16 — **no transcoding, no resample,
|
||||
no endianness swap.** The translator passes the audio payload through
|
||||
to the WS frame verbatim. This is a happy convergence that simplifies
|
||||
the translator; it is not a coincidence (slice-1's choice of 24 kHz mono
|
||||
was made with this ecosystem in mind).
|
||||
|
||||
---
|
||||
|
||||
## 4. The OpenAI Realtime translation layer (`rutster-brain-realtime/src/translator.rs`)
|
||||
|
||||
Pure (no schema of its own beyond the new tap event types); maps event
|
||||
names + payload shapes between the two wire protocols. Owns no call
|
||||
state beyond the OpenAI Realtime session/connection state.
|
||||
|
||||
### 4.1 The two-leg brain process
|
||||
|
||||
```
|
||||
WS server side WS client side
|
||||
(core-as-client dials here) (brain-as-client dials OpenAI)
|
||||
┌──────────────────┐ ┌──────────┐
|
||||
tap protocol │ rutster-brain- │ translation layer │ OpenAI │
|
||||
core ◄────────┤ realtime ├────────────────────────────► │ Realtime │
|
||||
(audio_in/out, │ │ tap event ⇄ OpenAI event │ API WS │
|
||||
speech_started, │ │ │ (wss://api.openai.com)│
|
||||
func_call, ...) └──────────────────┘ └──────────┘
|
||||
│
|
||||
│ brain process also owns:
|
||||
│ • OpenAI session.update (turn_detection: null — S4)
|
||||
│ • interruption-signal forwarding (OpenAI → tap)
|
||||
│ • function_call forwarding (bidirectional)
|
||||
│ • tools.update on startup (catalog to core)
|
||||
```
|
||||
|
||||
The translation layer is pure (no schema of its own beyond the new tap
|
||||
event types); it maps event names + payload shapes between the two wire
|
||||
protocols. It owns no call state beyond OpenAI's session/connection state.
|
||||
|
||||
### 4.2 Event mapping table
|
||||
|
||||
| Tap protocol | OpenAI Realtime | Notes |
|
||||
|---|---|---|
|
||||
| `hello` | `session.update` (modalities: ["audio", "text"], audio config, **turn_detection: null**) | Sent on handshake + reconnect; the S4 decision is encoded here |
|
||||
| `audio_in` (base64 PCM LE i16) | `input_audio_buffer.append` (base64, same wire shape) | Pass-through; no transcoding |
|
||||
| `audio_out` (from brain → core, advisory) | `response.audio.delta` events | The brain receives, formats, sends to core via tap_audio_out mpsc |
|
||||
| `speech_started` (brain → core advisory) | `input_audio_buffer.speech_started` | Caught and forwarded as advisory |
|
||||
| `speech_stopped` (brain → core advisory) | `input_audio_buffer.speech_stopped` | Caught and forwarded as advisory |
|
||||
| `function_call` (brain → core) | `response.function_call_arguments.done` (OpenAI emits) → translator formats as tap function_call | Brain's translator extracts `call_id`, `name`, `arguments`; the core dispatches via tool registry |
|
||||
| `function_call_output` (core → brain) | `conversation.item.create` with `item.type: function_call_output`, `call_id: <id>`, `output: <result-json>` | Closes the tool-call loop |
|
||||
| `bye` / `session_end` | `session.delete` + WS close | Graceful teardown |
|
||||
|
||||
### 4.3 The S4 turn-ownership decision (load-bearing per ADR-0008)
|
||||
|
||||
> ARCHITECTURE.md is right that the core should be authoritative on
|
||||
> VAD/barge-in ("the tap carries the *results* of reflexes, not the
|
||||
> responsibility"; "`AudioOut` advisory / core-authoritative"). But the
|
||||
> most likely first brain — OpenAI Realtime — does its own server-side
|
||||
> VAD and turn detection by default. Integrating step 3–4 means either:
|
||||
> disable the brain's turn detection and feed it clean, locally-detected
|
||||
> turns (preserves the reflex-authoritative principle, but is more
|
||||
> integration work and fights the API's grain), or
|
||||
> accept split-brain turn-taking where local VAD and the brain's VAD can
|
||||
> disagree (double-triggers, dropped barge-ins).
|
||||
>
|
||||
> **Decision implied:** make "who owns turn detection" an explicit decision
|
||||
> in the step-3 (brain) design doc, defaulting to core-authoritative,
|
||||
> brain VAD off, with the integration cost budgeted. Don't let it be
|
||||
> discovered at wiring time.
|
||||
> — Claude, vision-sanity-check S4
|
||||
|
||||
ADR-0008's FOB/green-zone split makes this decidable without ambiguity:
|
||||
|
||||
- **The reflex loop is FOB** (hot-path + differentiating — turn-taking,
|
||||
VAD-driven barge-in, jitter, pacing).
|
||||
- **The brain is green-zone** (per ADR-0008's explicit classification).
|
||||
- **The tap is core-authoritative** (slice-2 §4.1 — the playout buffer
|
||||
structurally prevents the brain from gating playout).
|
||||
|
||||
The only doctrine-consistent posture is: **OpenAI Realtime's server-side
|
||||
turn detection is disabled (`session.update` with `turn_detection: null`),
|
||||
and the FOB owns turn-taking.** The `speech_started` /
|
||||
`speech_stopped` events are forwarded as advisory signals so step 4 can
|
||||
use them as one input to the FOB reflex loop — but OpenAI does not gate
|
||||
playout, ever. The core-authoritative playout buffer (slice-2 §4.1) is
|
||||
the only thing that gates playout in slice-3 (and continues to be in
|
||||
step 4).
|
||||
|
||||
**Why this is the right call now (rather than deferring to step 4):**
|
||||
leaving OpenAI's turn detection on would mean slice-3's demo
|
||||
"works by accident" — the slice-4 barge-in integration would then have to
|
||||
disable it mid-flight, with unpredictable behavioral changes. By nailing
|
||||
it down at slice-3's `session.update`, the brain process never wires up
|
||||
the wrong posture and step 4's barge-in work composes cleanly on top.
|
||||
|
||||
### 4.4 Failure mode + reconnect
|
||||
|
||||
- **Tap-side WS reconnect (brain unreachable):** slice-2's
|
||||
`TapEngine::spawn_tap_engine` already does bounded-backoff reconnect
|
||||
(`250 ms → 500 ms → 1 s → 2 s → cap 5 s`, infinite retries,
|
||||
re-`hello`s with the same `session_id`). The brain process being
|
||||
unreachable looks identical to slice-2's echo brain being unreachable
|
||||
— the slice-2 reconnect path is unchanged. The seam test (slice-2
|
||||
§8.5 #6) holds: `tap_engine.rs` is byte-identical.
|
||||
- **OpenAI-side WS failure:** the brain process's own concern — it enters
|
||||
its own bounded-backoff reconnect to OpenAI, and forwards a tap `error`
|
||||
event (`{ "code": "brain_upstream_disconnect", "message": "..." }`)
|
||||
to the core as an FYI. The core logs + counts; the call stays up; the
|
||||
tap-side playout goes silent (slice-2's underflow path) until OpenAI
|
||||
reconnects. The slice-3 brain's OpenAI-side reconnect is independent
|
||||
of the slice-2 tap-side reconnect — two distinct failure surfaces.
|
||||
- **API-key invalid (OpenAI returns 401):** the brain process emits a tap
|
||||
`error` event (`{ "code": "brain_auth_failed", "message": "..." }`)
|
||||
and exits the OpenAI leg. The core's response is unchanged from the
|
||||
upstream-disconnect case (logs + silence + reconnect path). Operator
|
||||
intervention; the call stays up.
|
||||
- **Tool-registry dispatch failure:** a tool-registry error (e.g.,
|
||||
`hangup` fires but the channel state machine rejects the transition
|
||||
because the call is already `Closing`) returns a `function_call_output`
|
||||
with `status: "error"` and a result body explaining. The brain forwards
|
||||
the error to OpenAI; OpenAI may or may not retry depending on its own
|
||||
logic. No core-side retry; the model gets one chance.
|
||||
|
||||
---
|
||||
|
||||
## 5. Lifecycle & integration
|
||||
|
||||
### 5.1 Session lifecycle (slice-3 delta on slice-2)
|
||||
|
||||
1. `POST /v1/sessions` — body still *optionally* carries `tap_url`. If
|
||||
absent, falls back to `RUTSTER_TAP_URL` env. Default now documented
|
||||
as either `ws://127.0.0.1:8081/echo` (slice-2 Rust echo brain) or
|
||||
`ws://127.0.0.1:8082/realtime` (slice-3 OpenAI Realtime brain); the
|
||||
operator picks which brain to run.
|
||||
2. `POST /v1/sessions/:id/offer` — unchanged from slice-1/2; SDP answer.
|
||||
3. On `Connected`: the binary's poll task observes the state transition
|
||||
+ spawns the `TapEngine` task (unchanged from slice-2). The
|
||||
`TapEngine` connects to the brain process's WS server.
|
||||
4. Brain process startup: connects to `wss://api.openai.com/v1/realtime`,
|
||||
sends `session.update` with `turn_detection: null`, waits for OpenAI
|
||||
`session.created` ack.
|
||||
5. Brain process WS server accepts the core's tap WS; `hello` exchange;
|
||||
brain process sends `tools.update` with the catalog (currently:
|
||||
`hangup` only, plus any tool schemas the brain process's startup
|
||||
config declares — see §6).
|
||||
6. Audio flows: core decodes Opus → PCM → `audio_in` to brain → translator
|
||||
formats as `input_audio_buffer.append` → OpenAI processes →
|
||||
`response.audio.delta` → translator formats as `audio_out` → core's
|
||||
playout ring → str0m encode.
|
||||
7. Interruption signals: OpenAI `input_audio_buffer.speech_started` /
|
||||
`.speech_stopped` → translator → tap `speech_started` / `speech_stopped`
|
||||
→ core logs + counts (advisory). Step 4 will wire these into the FOB
|
||||
reflex loop.
|
||||
8. Tool calls: OpenAI `response.function_call_arguments.done` → translator
|
||||
→ tap `function_call` → core's `tool_registry` dispatches (via the
|
||||
TapClient's new side-channel mpsc) → `function_call_output` reply →
|
||||
translator → OpenAI `conversation.item.create`.
|
||||
9. `DELETE /v1/sessions/:id` or peer-close → `Closing`: slice-2's
|
||||
unmodified teardown sequence fires — the TapEngine's `session_end` →
|
||||
brain process's `session.delete` to OpenAI → WS close on both legs.
|
||||
|
||||
### 5.2 The tool-call side-channel (the only brown-binary wiring)
|
||||
|
||||
Slice-2's `TapClient` runs the WSS pump loop with two mpsc channels
|
||||
(`tx_pcm_in` for inbound audio to brain, `rx_audio_out` for outbound
|
||||
audio from brain). Slice-3 adds a third:
|
||||
|
||||
- `tx_function_call: mpsc::Sender<FunctionCallEvent>` — the TapClient
|
||||
emits a `FunctionCallEvent` whenever it observes a tap `function_call`
|
||||
message on its inbound stream. The binary's poll task drains this in
|
||||
the same cycle it drains the existing `flush_tx` side-channel
|
||||
(slice-2 §5.3 step 4) — same pattern, one extra channel.
|
||||
- `rx_function_call_output: mpsc::Receiver<FunctionCallOutputEvent>` —
|
||||
the binary writes `function_call_output` events through this channel;
|
||||
the TapClient picks them up and sends them as tap WS frames.
|
||||
|
||||
The shape keeps the seam test (§7 #5 of done criteria below): all this
|
||||
lives in the binary's `tap_engine.rs` callers + the new `tool_registry.rs`
|
||||
module. `loop_driver.rs` and `rtc_session.rs` are untouched.
|
||||
|
||||
### 5.3 Brain process startup configuration
|
||||
|
||||
| Env var | Purpose | Default |
|
||||
|---|---|---|
|
||||
| `RUTSTER_TAP_BIND` | WS server bind addr for the brain process | `127.0.0.1:8082` |
|
||||
| `OPENAI_API_KEY` | OpenAI Realtime API key (mutually exclusive with `_FILE`) | required unless `--features=mock` |
|
||||
| `OPENAI_API_KEY_FILE` | Path to a file containing the API key | optional; overrides `_KEY` |
|
||||
| `OPENAI_REALTIME_MODEL` | OpenAI Realtime model id | `gpt-4o-realtime` (or current equivalent) |
|
||||
| `OPENAI_REALTIME_VOICE` | Voice for TTS output | `alloy` (OpenAI's default; documented) |
|
||||
| `RUTSTER_BRAIN_TOOLS` | Comma-separated tool names the brain should advertise in `tools.update` | `hangup` (only tool the core wires; others cause `not_implemented` replies) |
|
||||
|
||||
The dev mode (`--features=mock`) doesn't read `OPENAI_API_KEY` and
|
||||
instruments an in-process mock OpenAI Realtime WS server that generates
|
||||
canned `response.audio.delta` events on `input_audio_buffer.append` and
|
||||
asserts on the `session.update` having `turn_detection: null`. Used by:
|
||||
the slice-3 integration test (no real OpenAI credentials, no network
|
||||
calls), and the offline dev loop.
|
||||
|
||||
---
|
||||
|
||||
## 6. The tool registry (`crates/rutster/src/tool_registry.rs`)
|
||||
|
||||
A FOB-internal dispatch over the brain's proposed tool calls. The brain
|
||||
proposes (via `function_call`); the FOB disposes (via `function_call_output`).
|
||||
Per ADR-0007's spend-gate posture: the brain never holds the wire,
|
||||
structurally.
|
||||
|
||||
### 6.1 Trait + registry shape
|
||||
|
||||
```rust
|
||||
#[async_trait]
|
||||
pub trait Tool: Send + Sync {
|
||||
/// Tool name as the brain will reference it in function_call events.
|
||||
fn name(&self) -> &str;
|
||||
/// JSON-schema description the registry sends to the brain on tools.update.
|
||||
fn schema(&self) -> serde_json::Value;
|
||||
/// Execute the tool; return a result that the registry will serialize
|
||||
/// into the function_call_output payload.
|
||||
async fn call(&self, args: serde_json::Value) -> ToolResult;
|
||||
}
|
||||
|
||||
pub enum ToolResult {
|
||||
Ok(serde_json::Value),
|
||||
Error(String),
|
||||
NotImplemented,
|
||||
}
|
||||
|
||||
pub struct ToolRegistry {
|
||||
tools: Vec<Box<dyn Tool>>,
|
||||
}
|
||||
|
||||
impl ToolRegistry {
|
||||
pub fn new() -> Self;
|
||||
pub fn register(&mut self, tool: Box<dyn Tool>);
|
||||
/// Dispatch by tool name; returns ToolResult::NotImplemented if unknown.
|
||||
pub async fn dispatch(&self, name: &str, args: serde_json::Value) -> ToolResult;
|
||||
/// Used on startup + tools.update: enumerate the registered catalog.
|
||||
pub fn catalog(&self) -> Vec<serde_json::Value>;
|
||||
}
|
||||
```
|
||||
|
||||
### 6.2 The `hangup` tool (the only wired impl in slice-3)
|
||||
|
||||
A `Tool` impl that on `call`:
|
||||
1. Looks up the `ChannelId` (passed via the dispatch context — the
|
||||
`tool_registry` is keyed by `ChannelId`, one registry per active
|
||||
channel).
|
||||
2. Fires `AppState::close(channel_id).await` — the existing slice-2
|
||||
teardown path (sets `Closing`, sends `session_end`, aborts the engine).
|
||||
3. Returns `ToolResult::Ok({"channel_state": "Closing"})` so the brain
|
||||
gets confirmation.
|
||||
|
||||
Other tool names (anything in `RUTSTER_BRAIN_TOOLS` the brain advertises
|
||||
that isn't `hangup`) → the registry returns `ToolResult::NotImplemented`:
|
||||
the brain process forwards the reply as an OpenAI
|
||||
`conversation.item.create` with the not_implemented status, and the model
|
||||
is free to continue.
|
||||
|
||||
### 6.3 What slice-3 does NOT wire
|
||||
|
||||
- No `transfer` tool (escalation rung 2 — separate slice).
|
||||
- No `lookup` / CRM-integration tools (business-logic integrations are
|
||||
green-zone + future-rung).
|
||||
- No spend/abuse-triggered tools (step 6).
|
||||
- No dynamic tool registration at runtime (the catalog is fixed at
|
||||
startup from `RUTSTER_BRAIN_TOOLS`).
|
||||
|
||||
If a future brain proposes a tool not in the registry: `not_implemented`
|
||||
reply; the model is free to retry with different arguments or give up.
|
||||
The registry's job is to be the auditable dispatch boundary, not to be
|
||||
extensible mid-call.
|
||||
|
||||
---
|
||||
|
||||
## 7. CI, dev loop, testing
|
||||
|
||||
### 7.1 New `[workspace.dependencies]` (Cargo.toml)
|
||||
|
||||
- `async-trait = "0.1"` (for the `Tool` trait's async fn).
|
||||
- `tokio-tungstenite`, `serde_json`, `tracing`, `url` — already pulled
|
||||
by slice-2; reused.
|
||||
|
||||
Member crate `rutster-brain-realtime` references these with
|
||||
`dep.workspace = true`.
|
||||
|
||||
### 7.2 CI (`.github/workflows/ci.yml`)
|
||||
|
||||
Unchanged structure: `cargo fmt --check`, `cargo clippy -D warnings`,
|
||||
`cargo test --all`, `cargo deny check`. The new `rutster-brain-realtime`
|
||||
crate joins `--all`. **No Python in CI** — the OpenAI Realtime Python
|
||||
reference brain is README-documented only. The Rust
|
||||
`rutster-brain-realtime`'s in-process `MockRealtimeBrain` powers the
|
||||
integration test; no real OpenAI credentials needed.
|
||||
|
||||
### 7.3 Dev loop
|
||||
|
||||
- `cargo run -p rutster-brain-realtime --features=mock` → starts the
|
||||
brain process with an in-process mock OpenAI Realtime (on `:8082`):
|
||||
no API key required, no network calls to OpenAI.
|
||||
- `cargo run -p rutster-brain-realtime` (with `OPENAI_API_KEY` set) →
|
||||
starts the brain process with the real OpenAI Realtime.
|
||||
- `cargo run` (or `cargo run -p rutster`) → starts the axum signaling
|
||||
server on `0.0.0.0:8080`, dials out to `$RUTSTER_TAP_URL` on each
|
||||
session.
|
||||
- Browser → `http://localhost:8080/` → click "Start call" → grant mic →
|
||||
speak → hear AI reply through the brain process.
|
||||
- `RUST_LOG=rutster=debug cargo run` for verbose tracing including
|
||||
tap + tool-call events.
|
||||
- `--features=echo` on the binary (slice-2 inherited) → bypasses the
|
||||
brain entirely, routes audio through `EchoAudioPipe`.
|
||||
|
||||
### 7.4 Testing strategy
|
||||
|
||||
- **Unit tests in `rutster-brain-realtime`:**
|
||||
- Protocol-event translation round-trips (tap `audio_in` ↔ OpenAI
|
||||
`input_audio_buffer.append`; tap `function_call` ↔ OpenAI
|
||||
`response.function_call_arguments.done`; tap `function_call_output`
|
||||
↔ OpenAI `conversation.item.create`).
|
||||
- `session.update` body asserts `turn_detection: null` (S4 decision
|
||||
encoded as a test).
|
||||
- `tools.update` serialization round-trips.
|
||||
- Error path: OpenAI-side WS error → tap `error` event forwarding.
|
||||
- **Unit tests in `rutster-tap`:**
|
||||
- New event types (de)serialization round-trips with golden JSON
|
||||
fixtures in `tests/fixtures/`.
|
||||
- Slice-2's existing echo-brain tests stay green (the new event types
|
||||
are ignored by the echo brain — forwards-compat).
|
||||
- **Unit tests in `rutster` `tool_registry.rs`:**
|
||||
- `hangup` tool fires `AppState::close` correctly.
|
||||
- Unknown tool name → `ToolResult::NotImplemented`.
|
||||
- Registry catalog serialization.
|
||||
- **Integration test in `rutster` binary crate:** spin up the axum server
|
||||
(ephemeral port) + the in-process `MockRealtimeBrain` (ephemeral port) +
|
||||
set `RUTSTER_TAP_URL`. Drive a synthetic WebRTC peer (extending
|
||||
slice-2's `tap_integration.rs` harness): push PCM into the core via
|
||||
the WebRTC peer → assert `audio_out` flows back through the tap (the
|
||||
mock brain generates canned `response.audio.delta` on
|
||||
`input_audio_buffer.append`) → assert they're re-encoded + pushed to
|
||||
str0m. Plus: function-call round-trip (mock brain emits a
|
||||
`function_call` for `hangup` → core's `tool_registry.dispatch("hangup")`
|
||||
fires → channel state `Closing` → `session_end` over the tap →
|
||||
mock brain's `session.delete` recorded).
|
||||
- **Manual e2e test plan (README):**
|
||||
1. `cargo run -p rutster-brain-realtime --features=mock` (mock brain on `:8082`).
|
||||
2. `cargo run` (core on `:8080`).
|
||||
3. Browser → speak → hear mock-brain reply within ~250 ms (slice-2's
|
||||
round-trip; no real OpenAI).
|
||||
4. Repeat with the real brain (`cargo run -p rutster-brain-realtime`
|
||||
with `OPENAI_API_KEY` set) → end-to-end speech-to-speech with real
|
||||
OpenAI Realtime within ~700 ms.
|
||||
5. Repeat with the Python brain (`python
|
||||
examples/openai_realtime_brain/openai_realtime_brain.py`) → should
|
||||
also work (proves the protocol is language-agnostic, same as
|
||||
slice-2's Python echo brain).
|
||||
6. Function-call: trigger a model behavior that emits a tool call
|
||||
(e.g. a test prompt that says "please hang up") → assert the call
|
||||
closes + `session_end` over the tap + `session.delete` to OpenAI.
|
||||
7. `cargo test --all` green; `cargo fmt --check` /
|
||||
`cargo clippy -D warnings` / `cargo deny check` green.
|
||||
|
||||
### 7.5 Slice 3 "done" criteria
|
||||
|
||||
The slice is complete when, on a clean checkout (+ open pivot / slice-1
|
||||
review fixes / slice-2 plan-rebaseline PRs merged):
|
||||
|
||||
1. `cargo test --all` passes (unit + integration). The new
|
||||
`rutster-brain-realtime` crate tests green alongside slice-1/2's suite.
|
||||
2. `cargo fmt --check`, `cargo clippy -D warnings`, `cargo deny check`
|
||||
all pass.
|
||||
3. `cargo run -p rutster-brain-realtime --features=mock` + `cargo run` →
|
||||
browser, speak, hear mock-brain reply within ~250 ms.
|
||||
4. With `OPENAI_API_KEY` set: real OpenAI Realtime end-to-end within
|
||||
~700 ms.
|
||||
5. **Both** `rutster-brain-realtime` (Rust, with `--features=mock`) **and**
|
||||
`examples/openai_realtime_brain/openai_realtime_brain.py` (Python)
|
||||
successfully interop against the core — proves the extended protocol
|
||||
is language-agnostic.
|
||||
6. **The seam test (load-bearing):** `rutster-media`'s `loop_driver.rs`
|
||||
and `rtc_session.rs` keep their media-loop trait-method call sites
|
||||
**byte-identical** to slice-2 (post-review-fix) baseline. The only
|
||||
brown-binary additions are: `crates/rutster/src/tool_registry.rs`
|
||||
(new module) + a new side-channel drain in `session_map.rs`. The
|
||||
`tap_engine.rs` and `TapClient` get only additive new mpsc channels
|
||||
(one new sender + one new receiver); the WSS pump loop's structure
|
||||
is unchanged. A `git diff v<slice-2-post-review-fix-tag> --
|
||||
crates/rutster-media/src/loop_driver.rs
|
||||
crates/rutster-media/src/rtc_session.rs` shows no behavior-changing
|
||||
hunks (doc-comment or import changes permitted).
|
||||
7. **S4 turn-ownership test:** the integration test asserts that the
|
||||
`session.update` sent to OpenAI Realtime (or its mock equivalent)
|
||||
contains `turn_detection: null`. The brain never auto-barges; the
|
||||
core-authoritative playout buffer is the only playout gate.
|
||||
8. `LEARNING.md` grows ≥3 new pointers: `async-trait` patterns →
|
||||
`crates/rutster/src/tool_registry.rs`; OpenAI Realtime adapter →
|
||||
`crates/rutster-brain-realtime/src/translator.rs`; tap protocol
|
||||
extension + forwards-compat → `crates/rutster-tap/src/protocol.rs`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Open decisions (tracked)
|
||||
|
||||
- **`response.audio.delta` batching.** OpenAI sends many small delta
|
||||
events per response; the translator MAY batch into the slice-2
|
||||
`audio_out` 20 ms frame, or pass each delta through immediately as
|
||||
its own `audio_out`. Batching reduces WS round-trips but adds latency.
|
||||
Track + revisit after latency measurement with the real brain.
|
||||
- **Tool-registry extensibility.** Slice-3 fixes the catalog at startup
|
||||
via `RUTSTER_BRAIN_TOOLS`. Runtime-extensible registration (new tools
|
||||
added mid-call) is a future-rung concern — would need a tool-register
|
||||
API event. Track.
|
||||
- **API-key rotation.** Slice-3 reads the key once at startup; a key
|
||||
rotation requires a brain-process restart. KMS / Vault integration
|
||||
lands with the real trust boundary (step 6). Track.
|
||||
- **Voice + persona selection.** `OPENAI_REALTIME_VOICE` is the only
|
||||
persona knob now; future brain processes may carry full prompt /
|
||||
system-message customization (rung 2+). Track.
|
||||
- **Multi-brain routing.** Slice-3 ships one brain process: OpenAI
|
||||
Realtime. A Deepgram+LLM+TTS composite adapter or a self-hosted
|
||||
open-weights brain would be its own crate
|
||||
(`rutster-brain-deepgram`, `rutster-brain-local`) — the tap protocol
|
||||
is brain-agnostic by design. Track + revisit when a second brain
|
||||
lands.
|
||||
|
||||
---
|
||||
|
||||
## 9. Out-of-scope re-check (against AGENTS.md + ADR-0007/0008)
|
||||
|
||||
| Item | Status |
|
||||
|---|---|
|
||||
| Dedicated timing thread for media loop | Still step 4. |
|
||||
| TLS on the HTTP signaling surface | Still step 5. |
|
||||
| Authn / authz / multi-tenancy on `/v1/sessions` | Still step 6. |
|
||||
| Trickle ICE | Unchanged. |
|
||||
| Barge-in / VAD-driven playout kill | **Step 4.** Slice-3 pre-paves the `speech_started` / `speech_stopped` advisory event seam so step 4 lands cleanly. |
|
||||
| PSTN trunk / rented transport | Still step 5. |
|
||||
| Spend cap / abuse gate | Still step 6. |
|
||||
| CDR / event bus / OTel beyond per-Channel tracing | Still step 5. |
|
||||
| Browser automation / Playwright | Still post-slice-1. |
|
||||
| Docker / compose | Still later-rung. |
|
||||
| Transfer / park / pickup / barge (call features) | Still escalation rung 2. |
|
||||
|
||||
If an agent proposes adding any of these in slice 3, the right answer is
|
||||
"no, see the slice-3 spec §1.2."
|
||||
|
||||
---
|
||||
|
||||
## 10. Key design decisions (summary of the brainstorming session)
|
||||
|
||||
| Decision | Choice | Rejected alternatives | Why |
|
||||
|---|---|---|---|
|
||||
| Brain target | **OpenAI Realtime NOW; design for multi-brain later.** Adapter trait shape is the tap protocol itself, not a Rust trait. | Trait crate `rutster-brain` + impl crate; Deepgram+LLM+TTS now | Slice-2's protocol-as-contract framing already delivers "multi-brain later" without a speculative trait; YAGNI on abstraction. A Deepgram+LLM+TTS composite brain's friction will be in its own internal topology (composing three async WS/HTTP streams), not in `trait BrainAdapter` ergonomics — a trait wouldn't ease that anyway. |
|
||||
| Cool scope | **Round-trip speech + interruption signals (advisory) + function-calling plumbing** | Speech only; speech + interruption; speech + interruption + function calling | Function calling is the gateway to escalation (rung 2 — `hangup` is the only wired tool; `transfer` lands later). Plumbing the FOB dispatch contract in slice-3 means step-4 barge-in composes on a clean foundation; deferring it would force step-4 to retrofit the seam. |
|
||||
| Function-call handler | **Built-in tool registry in the core (FOB)** | Outbound tool-server URL (second egress); adapter handles function calls itself; plumbing only | Per ADR-0007: "rutster mediates both the provider call-control API and the brain tap, so the brain never holds the wire." The FOB disposes; the brain proposes. `hangup` is the only wired tool; others reply `not_implemented` honestly. The registry's job is the auditable dispatch boundary, not extensibility. |
|
||||
| Turn-detection ownership (S4) | **Core-authoritative, brain VAD off** (`session.update` with `turn_detection: null`) | Accept split-brain (OpenAI VAD on; core-authoritative too → disagreement); defer the decision to step 4 | ADR-0008 makes the FOB/green-zone split mechanical: reflex loop is FOB, brain is green-zone, playout is core-authoritative. The only doctrine-consistent choice is to disable OpenAI's server-side VAD and let the FOB own turn-taking. Nailing it down at slice-3's `session.update` means step-4 barge-in composes cleanly. |
|
||||
| API-key posture | **Env var + file-path override.** `OPENAI_API_KEY` env default; `OPENAI_API_KEY_FILE` overrides. | Env-only; KMS-required-now | Matches slice-2's "no auth yet" stance cleanly. KMS / Vault lands with the real trust boundary (step 6). The file-path override makes secret-manager injection (k8s secrets, Vault agent) trivial when that layer exists. |
|
||||
| Dev mode | **`--features=mock` in-process mocked OpenAI Realtime** | No dev mock (always require real OpenAI); external mock server | The slice-3 dev loop must be zero-OpenAI-credentials + zero-network-calls-to-OpenAI. The integration test uses the same mock. Same pattern as slice-2's `EchoServer`. |
|
||||
| Workspace shape | **One new crate `rutster-brain-realtime` (library + binary), mirroring `rutster-tap-echo`** | Put the adapter in `examples/`; trait crate + impl crate | Slices-2's precedent. Real workspace member runs fmt/clippy/test/deny like everyone else. Reuses `rutster-tap`'s protocol types — the contract test that the wire types are reusable from outside the core. A future second brain is its own crate, same shape. |
|
||||
| Brain process port + protocol | **`ws://127.0.0.1:8082/realtime` (slice-2's echo brain defaults to `:8081/echo`)** | Single brain port with feature toggle | Two brain processes coexist (operator picks which to run via `RUTSTER_TAP_URL`). The default URL stays `ws://127.0.0.1:8081/echo` (slice-2 documented); slice-3 adds `ws://127.0.0.1:8082/realtime` as the documented OpenAI-brain default. |
|
||||
| Tap protocol extension posture | **Additive v1 events** (not v2) | v2 protocol bump; OpenAI Realtime event-schema adoption | Slice-2 §3.4's "unknown type → log + count + drop" rule makes additive events free — old echo brains ignore them. No wire-format break, no version negotiation needed. A v2 binary mode (raw LE i16 over WS binary frames) remains a future-rung optimization per slice-2 §9. |
|
||||
|
||||
---
|
||||
|
||||
## 11. References
|
||||
|
||||
- [README.md](../../../README.md) — north star, capability ladder
|
||||
- [ARCHITECTURE.md §"Agent tap"](../../ARCHITECTURE.md) — the presumptive
|
||||
tap shape this slice hardens against a real brain
|
||||
- [PORT_PLAN.md](../../PORT_PLAN.md) — capability checklist + thin-slice
|
||||
phasing; §10 "WASM demoted, agent tap is the extension point"
|
||||
- [ADR-0002](../../adr/0002-north-star-and-fused-core.md) — fused vertical
|
||||
- [ADR-0007](../../adr/0007-trunk-rented-transport.md) — rent the trunk;
|
||||
the brain is unaffected (PSTN reaches the reflex loop via media-leg
|
||||
ingress, not via the tap)
|
||||
- [ADR-0008](../../adr/0008-fob-and-green-zone.md) — FOB / green-zone
|
||||
doctrine; the brain is green-zone, the reflex loop is FOB, the tap is
|
||||
core-authoritative (the S4 turn-ownership decision follows)
|
||||
- [Slice 1 — WebRTC media loopback](2026-06-28-slice-1-webrtc-loopback-design.md)
|
||||
— this slice's foundation
|
||||
- [Slice 2 — the agent tap](2026-06-28-slice-2-agent-tap-design.md) — the
|
||||
tap interface + the `TapAudioPipe` seam this slice composes on
|
||||
- [Vision-revision spec](2026-06-26-vision-revision-design.md) — the
|
||||
pressure-test that produced the architecture
|
||||
- [Default UI design](2026-06-29-default-ui-design.md) — the operator
|
||||
console design record (later-rung, not slice-3's concern)
|
||||
- [Vision sanity-check review S4](../reviews/2026-06-28-vision-sanity-check.md)
|
||||
— the original finding that named the S4 turn-ownership decision;
|
||||
slice-3 §4.3 resolves it under ADR-0008
|
||||
52
examples/openai_realtime_brain/README.md
Normal file
52
examples/openai_realtime_brain/README.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# OpenAI Realtime reference brain — Python (slice-3 spec §7.5)
|
||||
|
||||
A Python implementation of the slice-3 OpenAI Realtime brain — the canonical
|
||||
foreign-language brain demo, hand-rolled from the documented tap protocol
|
||||
([`docs/superpowers/specs/2026-06-30-slice-3-realtime-brain-design.md`](../../docs/superpowers/specs/2026-06-30-slice-3-realtime-brain-design.md)).
|
||||
|
||||
## Why
|
||||
|
||||
Proves the slice-3 tap protocol extension is language-agnostic. A Python
|
||||
script speaking JSON-via-WSS matches the OpenAI-Realtime-related portions of
|
||||
the spec without depending on Rust code paths. Same rationale as slice-2's
|
||||
Python echo brain — the project's `examples/` dir is the home for canonical
|
||||
foreign-language brain demos.
|
||||
|
||||
The structure mirrors the Rust `rutster-brain-realtime` crate: a WS server
|
||||
(the tap side the core dials into) bridged bidirectionally to a WS client
|
||||
(the OpenAI Realtime side). The event mapping is the spec §4.2 table verbatim.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py
|
||||
```
|
||||
|
||||
The Python brain binds the tap WS server on `RUTSTER_BRAIN_BIND` (default
|
||||
`127.0.0.1:8082`). The rutster binary, started normally (`cargo run`), dials
|
||||
out to `RUTSTER_TAP_URL` (default `ws://127.0.0.1:8082/realtime` — set
|
||||
`RUTSTER_TAP_URL=ws://127.0.0.1:8082` to match the Python brain's bind).
|
||||
|
||||
Other env vars:
|
||||
|
||||
| Var | Default | Purpose |
|
||||
|---|---|---|
|
||||
| `OPENAI_API_KEY` | (required) | OpenAI auth |
|
||||
| `RUTSTER_BRAIN_BIND` | `127.0.0.1:8082` | host:port for the tap WS server |
|
||||
| `OPENAI_REALTIME_MODEL` | `gpt-4o-realtime` | OpenAI model query-param |
|
||||
| `OPENAI_REALTIME_VOICE` | `alloy` | Voice passed in `session.update` |
|
||||
|
||||
## Dependencies
|
||||
|
||||
Only `websockets>=12.0` — both the tap-side server and the OpenAI-side client
|
||||
speak plain WSS, so the `openai` Python SDK is not needed (the spec §4.2
|
||||
mapping is explicit enough to hand-roll against the wire). This is a
|
||||
deliberate simplification over the slice-3 plan skeleton, which imported the
|
||||
SDK but never used it.
|
||||
|
||||
## Not in CI
|
||||
|
||||
Per AGENTS.md's "no Python in the dev loop" rule. The slice-3 integration
|
||||
test uses `rutster-brain-realtime --features=mock` — the in-process Rust
|
||||
mock — not this Python file.
|
||||
195
examples/openai_realtime_brain/openai_realtime_brain.py
Normal file
195
examples/openai_realtime_brain/openai_realtime_brain.py
Normal file
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
"""OpenAI Realtime reference brain -- Python implementation (slice-3 spec §7.5).
|
||||
|
||||
Mirrors rutster-brain-realtime's WS-server + OpenAI-WS-client glue in
|
||||
Python, proving the slice-3 tap protocol is language-agnostic. The mapping
|
||||
is the §4.2 event table verbatim.
|
||||
|
||||
Run:
|
||||
pip install -r requirements.txt
|
||||
OPENAI_API_KEY=sk-... python examples/openai_realtime_brain/openai_realtime_brain.py
|
||||
|
||||
Not in CI (violates the zero-non-Rust-dev-deps dev loop per AGENTS.md).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import websockets
|
||||
|
||||
PROTOCOL_VERSION = 1
|
||||
SAMPLES = 480
|
||||
|
||||
# Configuration via env (matching the Rust binary's naming convention so the
|
||||
# docs read the same on both sides).
|
||||
RUTSTER_TAP_BIND = os.environ.get("RUTSTER_BRAIN_BIND", "127.0.0.1:8082")
|
||||
OPENAI_MODEL = os.environ.get("OPENAI_REALTIME_MODEL", "gpt-4o-realtime")
|
||||
OPENAI_VOICE = os.environ.get("OPENAI_REALTIME_VOICE", "alloy")
|
||||
|
||||
|
||||
def openai_realtime_url(model: str) -> str:
|
||||
"""wss://api.openai.com/v1/realtime?model=<model> (spec §4.2 + §5.3)."""
|
||||
return f"wss://api.openai.com/v1/realtime?model={model}"
|
||||
|
||||
|
||||
def openai_headers(api_key: str):
|
||||
"""The two headers the OpenAI WS handshake requires (spec §5.3)."""
|
||||
return [
|
||||
("Authorization", f"Bearer {api_key}"),
|
||||
("OpenAI-Beta", "realtime=v1"),
|
||||
]
|
||||
|
||||
|
||||
async def handle_tap_connection(tap_ws):
|
||||
"""Bridge one rutster tap WS connection to one OpenAI Realtime session.
|
||||
|
||||
Spec §4.2 mapping table, in both directions. One tap connection == one
|
||||
OpenAI Realtime session; stateless across reconnects (spec §5.3), exactly
|
||||
like the Rust brain and the slice-2 echo brain pattern.
|
||||
"""
|
||||
api_key = os.environ.get("OPENAI_API_KEY")
|
||||
if not api_key:
|
||||
await tap_ws.close(code=1011, reason="OPENAI_API_KEY required")
|
||||
return
|
||||
|
||||
# Open the OpenAI WS with the two required headers. additional_headers is
|
||||
# the websockets>=12 spelling (was extra_headers in v11).
|
||||
openai_ws = await websockets.connect(
|
||||
openai_realtime_url(OPENAI_MODEL),
|
||||
additional_headers=openai_headers(api_key),
|
||||
)
|
||||
|
||||
# S4: send session.update with turn_detection: null. The FOB (the core)
|
||||
# owns turn-taking; OpenAI's server-side VAD is explicitly disabled so it
|
||||
# does not contradict core-authoritative playout decisions. See
|
||||
# openai_client.rs for the Rust-side commentary on why this matters.
|
||||
await openai_ws.send(json.dumps({
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"modalities": ["text", "audio"],
|
||||
"voice": OPENAI_VOICE,
|
||||
"input_audio_format": "pcm16",
|
||||
"output_audio_format": "pcm16",
|
||||
"sample_rate": 24000,
|
||||
"turn_detection": None,
|
||||
},
|
||||
}))
|
||||
print(f"[openai_brain] session.update sent (turn_detection=null); "
|
||||
f"model={OPENAI_MODEL} voice={OPENAI_VOICE}")
|
||||
|
||||
seq_egress = 0
|
||||
|
||||
def next_seq():
|
||||
nonlocal seq_egress
|
||||
s = seq_egress
|
||||
seq_egress += 1
|
||||
return s
|
||||
|
||||
async def tap_to_openai():
|
||||
"""rutster tap events -> OpenAI Realtime events (spec §4.2 left column)."""
|
||||
async for raw in tap_ws:
|
||||
try:
|
||||
env = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
await tap_ws.send(json.dumps({
|
||||
"v": PROTOCOL_VERSION, "type": "error", "seq": next_seq(),
|
||||
"ts": 0, "code": "decode_failed", "message": str(e),
|
||||
}))
|
||||
continue
|
||||
|
||||
t = env.get("type")
|
||||
if t == "hello":
|
||||
# Stateless across reconnects: ack hello, do NOT re-send
|
||||
# session.update (already sent on this OpenAI WS session).
|
||||
await tap_ws.send(json.dumps({
|
||||
"v": PROTOCOL_VERSION, "type": "hello", "seq": next_seq(),
|
||||
"ts": 0, "session_id": env.get("session_id", ""),
|
||||
}))
|
||||
elif t == "audio_in":
|
||||
# Pass-through: same base64 PCM wire shape. No transcoding.
|
||||
await openai_ws.send(json.dumps({
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": env["pcm"],
|
||||
}))
|
||||
elif t == "function_call_output":
|
||||
# Core's tool-registry reply -> OpenAI conversation.item.create
|
||||
# so the model can continue. Spec §4.2 row 5.
|
||||
await openai_ws.send(json.dumps({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "function_call_output",
|
||||
"call_id": env["id"],
|
||||
"output": json.dumps(env.get("result", {})),
|
||||
},
|
||||
}))
|
||||
elif t in ("bye", "session_end"):
|
||||
await openai_ws.close()
|
||||
return
|
||||
# Unknown frame types: ignore (forward-compat via additive protocol).
|
||||
|
||||
async def openai_to_tap():
|
||||
"""OpenAI Realtime events -> rutster tap events (spec §4.2 right column)."""
|
||||
async for raw in openai_ws:
|
||||
try:
|
||||
event = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
t = event.get("type")
|
||||
if t == "response.audio.delta":
|
||||
# Forward as tap audio_out. The delta is base64 PCM16; pass
|
||||
# through. (Batching into 20 ms frames is a future
|
||||
# optimization, spec §1.2; v1 passes each delta through.)
|
||||
await tap_ws.send(json.dumps({
|
||||
"v": PROTOCOL_VERSION, "type": "audio_out",
|
||||
"seq": next_seq(), "ts": 0,
|
||||
"pcm": event["delta"], "samples": SAMPLES,
|
||||
}))
|
||||
elif t == "input_audio_buffer.speech_started":
|
||||
await tap_ws.send(json.dumps({
|
||||
"v": PROTOCOL_VERSION, "type": "speech_started",
|
||||
"seq": next_seq(), "ts": 0,
|
||||
}))
|
||||
elif t == "input_audio_buffer.speech_stopped":
|
||||
await tap_ws.send(json.dumps({
|
||||
"v": PROTOCOL_VERSION, "type": "speech_stopped",
|
||||
"seq": next_seq(), "ts": 0,
|
||||
}))
|
||||
elif t == "response.function_call_arguments.done":
|
||||
# Tool invocation from the model -> core's tool registry.
|
||||
await tap_ws.send(json.dumps({
|
||||
"v": PROTOCOL_VERSION, "type": "function_call",
|
||||
"id": event["call_id"], "name": event["name"],
|
||||
"args": json.loads(event.get("arguments", "{}")),
|
||||
"seq": next_seq(), "ts": 0,
|
||||
}))
|
||||
elif t == "error":
|
||||
# Surface OpenAI-side errors to the core for observation.
|
||||
await tap_ws.send(json.dumps({
|
||||
"v": PROTOCOL_VERSION, "type": "error",
|
||||
"seq": next_seq(), "ts": 0,
|
||||
"code": "openai_error",
|
||||
"message": event.get("error", {}).get("message", "unknown"),
|
||||
}))
|
||||
|
||||
try:
|
||||
await asyncio.gather(tap_to_openai(), openai_to_tap())
|
||||
finally:
|
||||
await openai_ws.close()
|
||||
|
||||
|
||||
async def main():
|
||||
api_key = os.environ.get("OPENAI_API_KEY")
|
||||
if not api_key:
|
||||
sys.exit("OPENAI_API_KEY required (see README.md)")
|
||||
|
||||
host, port = RUTSTER_TAP_BIND.split(":")
|
||||
print(f"[openai_brain] listening on ws://{RUTSTER_TAP_BIND} "
|
||||
f"(OpenAI model={OPENAI_MODEL} voice={OPENAI_VOICE})")
|
||||
async with websockets.serve(handle_tap_connection, host, int(port)):
|
||||
await asyncio.Future() # run forever
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
1
examples/openai_realtime_brain/requirements.txt
Normal file
1
examples/openai_realtime_brain/requirements.txt
Normal file
@@ -0,0 +1 @@
|
||||
websockets>=12.0
|
||||
Reference in New Issue
Block a user