From 754614bd56fb1c5e6c9eef2eae8f2359f21fba4a Mon Sep 17 00:00:00 2001 From: "Aaron D. Lee" Date: Thu, 9 Jul 2026 23:54:57 +0000 Subject: [PATCH] fix(deploy-epoch): un-spoofable smoke gates + honest docs (deploy-epoch-fix) (#29) Co-authored-by: Aaron D. Lee Co-committed-by: Aaron D. Lee --- .github/workflows/ci.yml | 8 +- .github/workflows/publish-images.yml | 18 ++ Cargo.toml | 21 +- crates/rutster/src/config.rs | 104 ++++++-- crates/rutster/src/main.rs | 7 +- crates/rutster/src/routes.rs | 15 +- crates/rutster/tests/tls_boot.rs | 224 ++++++++++++++++++ deny.toml | 34 +-- deploy/.env.example | 28 ++- deploy/Caddyfile | 20 +- deploy/Dockerfile | 24 +- deploy/compose.yaml | 11 + deploy/s6-rc.d/brain/type | 1 + deploy/s6-rc.d/caddy/type | 1 + deploy/s6-rc.d/engine/type | 1 + .../valkey-database/dependencies.d/engine | 1 - deploy/s6-rc.d/valkey-database/type | 1 + deploy/smoke/allinone_smoke.py | 46 ++-- deploy/smoke/compose_smoke.sh | 67 ++++-- deploy/smoke/ws_handshake.py | 162 +++++++++++++ docs/adr/0011-deployment-topology.md | 6 +- docs/deploy/aws.md | 5 +- docs/deploy/certificates.md | 2 +- docs/deploy/quickstart-docker.md | 3 +- docs/deploy/reverse-proxies.md | 4 +- docs/deploy/topologies.md | 2 +- 26 files changed, 704 insertions(+), 112 deletions(-) create mode 100644 crates/rutster/tests/tls_boot.rs create mode 100644 deploy/s6-rc.d/brain/type create mode 100644 deploy/s6-rc.d/caddy/type create mode 100644 deploy/s6-rc.d/engine/type delete mode 100644 deploy/s6-rc.d/valkey-database/dependencies.d/engine create mode 100644 deploy/s6-rc.d/valkey-database/type create mode 100755 deploy/smoke/ws_handshake.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7673d52..3381933 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,9 @@ jobs: --health-timeout 5s --health-retries 3 env: - VALKEY_URL: redis://127.0.0.1:6379/ + # Service-name form: act_runner hosts jobs in a separate netns — + # loopback doesn't reach the valkey service container. + VALKEY_URL: redis://valkey:6379/ steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -67,7 +69,9 @@ jobs: matrix: toolchain: [stable, "1.85"] env: - VALKEY_URL: redis://127.0.0.1:6379/ + # Service-name form: act_runner hosts jobs in a separate netns — + # loopback doesn't reach the valkey service container. + VALKEY_URL: redis://valkey:6379/ steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index dc687b3..a670930 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -62,6 +62,24 @@ jobs: strategy: matrix: toolchain: [stable, "1.85"] + # T9: service container block mirrors ci.yml's test job (post-T1 form). + # Without this, the valkey_sink_lifecycle test self-skips on every v* tag + # (the tag-gate promise: "byte-identical to the image CI tested" — silent + # skip on a non-running test is the silent-skip failure mode the T1 fix + # was specifically about). Service-name VALKEY_URL matches T1: act_runner + # hosts jobs in a separate netns, loopback doesn't reach the service. + services: + valkey: + image: valkey/valkey:latest + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 + env: + VALKEY_URL: redis://valkey:6379/ steps: - uses: actions/checkout@v4 diff --git a/Cargo.toml b/Cargo.toml index 31688a5..5edd843 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,11 +84,26 @@ axum-server = { version = "0.8", features = ["tls-rustls"] } # feature. Already in Cargo.lock at 0.23.41 via str0m → aws-lc-rs; this surfaces the # direct edge for the integration test's ClientConfig. Apache-2.0 / MIT / ISC, # cargo-deny clean. -rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs", "ring"] } +# +# Single-provider feature gate (deploy-epoch-fix T3): rustls 0.23's +# CryptoProvider::from_crate_features() picks the process default via +# `#[cfg(all(feature = "aws_lc_rs", not(feature = "ring"), ...))]` blocks — +# if BOTH features are on, neither cfg block matches and `from_pem_file` +# panics with "Could not automatically determine the process-level +# CryptoProvider". The str0m tree still pulls the `ring` *crate* via +# `dimpl → rcgen → ring`, but that's rcgen's own direct dep, NOT an +# activation of rustls's `ring` *feature*. Verified via +# `cargo tree -e features -i rustls` — the only edge activating +# `rustls/ring` was this manifest line. Dropping `ring` here leaves +# `aws-lc-rs` as the sole rustls provider; `axum_server::RustlsConfig::from_pem_file` +# installs `aws_lc_rs::default_provider()` automatically. +rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs"] } # tokio-rustls 0.26: tokio integration for rustls. Already in Cargo.lock at 0.26.4 # (transitive once Task 1's Cargo.lock resolves). Used by the serve_tls integration -# test's TLS client handshake. -tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs", "ring"] } +# test's TLS client handshake. Single-provider rationale matches rustls above +# (deploy-epoch-fix T3): dropping `ring` so tokio-rustls doesn't re-activate +# `rustls/ring` via feature forwarding. +tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc-rs"] } # redis 0.27: Valkey/Redis client for ValkeyEventSink (deploy spec §5.6). # BSD-3-Clause (verified via docs.rs/redis + crates.io API), cargo-deny clean against # the deny.toml allow list (line 88). Pinned to 0.27 (NOT 1.3.0) because redis 1.3.0 diff --git a/crates/rutster/src/config.rs b/crates/rutster/src/config.rs index acd8e5d..c542b9c 100644 --- a/crates/rutster/src/config.rs +++ b/crates/rutster/src/config.rs @@ -214,6 +214,31 @@ pub fn twilio_credentials( let webhook_base = webhook_base_raw .parse::() .map_err(|e| format!("RUTSTER_TWILIO_WEBHOOK_BASE is not a URL: {e}"))?; + // Fail-fast: the webhook base's scheme drives the + // derivation in routes::twilio_inbound_webhook. A schemeless input + // like "pbx.example.com:8080" parses successfully as a URL (scheme + // becomes "pbx.example.com") but would 500 every webhook at runtime + // — fail at boot instead, naming the var. + let scheme = webhook_base.scheme(); + if scheme != "http" && scheme != "https" { + return Err(format!( + "RUTSTER_TWILIO_WEBHOOK_BASE scheme must be http or https, got {scheme:?}" + )); + } + // XML-specials guard: the webhook base is interpolated into the + // TwiML response body (routes::twilio_inbound_webhook) without an + // XML-escape pass. The `url` crate percent-encodes `"` `<` `>` in + // authorities/queries/fragments, but `&` survives literally as a + // path char (e.g. `&` parses to `/&`); reject any literal + // `&` in the URL's `as_str()` to keep the TwiML body well-formed. + // The `'` char is NOT rejected: the TwiML body uses `"` as its + // attribute delimiter, so `'` is a valid path character. + let as_str = webhook_base.as_str(); + if as_str.contains('&') { + return Err("RUTSTER_TWILIO_WEBHOOK_BASE contains XML-special char '&' \ + (literal ampersand in path/query); rejected to keep TwiML body well-formed" + .to_string()); + } Ok(Some(TwilioCredentials { account_sid, @@ -295,8 +320,10 @@ pub fn metrics_bind(raw: Option) -> Result { /// `RUTSTER_VALKEY_URL` — optional Valkey event-bus endpoint. /// /// Returns `Ok(None)` when unset (the binary keeps the default -/// [`TracingEventSink`]). When set, the scheme MUST be `redis://` or -/// `rediss://`; anything else is a fail-fast operator error. The `Url` +/// [`TracingEventSink`]). When set, the scheme MUST be `redis://` — +/// `rediss://` (TLS) is rejected at boot because the redis crate's TLS +/// feature is not compiled in (spec §3.4 staging; rustls Phase 2 is +/// trigger-gated). Anything else is a fail-fast operator error. The `Url` /// parser is reused (already a workspace dep) so malformed input names /// the variable in the error. pub fn valkey_url(raw: Option) -> Result, String> { @@ -306,13 +333,16 @@ pub fn valkey_url(raw: Option) -> Result, String> { let u = s .parse::() .map_err(|e| format!("RUTSTER_VALKEY_URL {s:?}: {e}"))?; - if u.scheme() != "redis" && u.scheme() != "rediss" { - return Err(format!( - "RUTSTER_VALKEY_URL {s:?}: scheme must be redis:// or rediss://, got {:?}", - u.scheme() - )); + match u.scheme() { + "redis" => Ok(Some(u)), + "rediss" => Err(format!( + "RUTSTER_VALKEY_URL {s:?}: rediss:// requires a redis TLS feature that is \ + not compiled (see spec §3.4 staging); use redis://" + )), + other => Err(format!( + "RUTSTER_VALKEY_URL {s:?}: scheme must be redis:// (got {other:?})" + )), } - Ok(Some(u)) } } } @@ -590,16 +620,62 @@ mod tests { assert_eq!(url.unwrap().as_str(), "redis://127.0.0.1:6379/"); } - #[test] - fn valkey_url_parses_rediss_scheme() { - let url = valkey_url(Some("rediss://127.0.0.1:6379/".into())).unwrap(); - assert_eq!(url.unwrap().scheme(), "rediss"); - } - #[test] fn valkey_url_rejects_non_redis_scheme_with_var_name_in_error() { let err = valkey_url(Some("https://example.com".into())).unwrap_err(); assert!(err.contains("RUTSTER_VALKEY_URL")); assert!(err.contains("redis://")); } + + #[test] + fn valkey_url_rejects_rediss_with_uncompiled_tls_pointer() { + // T4: rediss:// is rejected at boot because the redis crate's TLS + // feature is not compiled in (spec §3.4 staging; rustls Phase 2). + // The error must point the operator at the uncompiled feature + + // the spec section, not just "scheme must be redis://". + let err = valkey_url(Some("rediss://127.0.0.1:6379/".into())).unwrap_err(); + assert!(err.contains("RUTSTER_VALKEY_URL")); + assert!(err.contains("rediss://")); + assert!(err.contains("not compiled")); + assert!(err.contains("§3.4")); + } + + #[test] + fn webhook_base_rejects_schemeless_url() { + // T4: `Url::parse("pbx.example.com:8080")` succeeds with scheme + // "pbx.example.com" — a schemeless operator input that would 500 + // every webhook at runtime. Boot-time scheme check fails fast. + let err = twilio_credentials( + Some("AC_x".into()), + Some("tok".into()), + Some("0.0.0.0:8081".into()), + Some("pbx.example.com:8080".into()), + ) + .unwrap_err(); + assert!(err.contains("RUTSTER_TWILIO_WEBHOOK_BASE")); + assert!(err.contains("http")); + assert!(err.contains("pbx.example.com")); + } + + #[test] + fn webhook_base_rejects_xml_special_chars() { + // T4: the webhook base is interpolated into the TwiML body without + // an XML-escape pass. The `url` crate percent-encodes `"`, `<`, `>` + // before they reach `as_str()`, but `&` survives literally as a + // path char (e.g. `&` parses to `/&`); the boot-time guard + // must reject that literal `&` to keep the TwiML body well-formed. + let err = twilio_credentials( + Some("AC_x".into()), + Some("tok".into()), + Some("0.0.0.0:8081".into()), + Some("https://example.com/&".into()), + ) + .expect_err("expected error for 'https://example.com/&'"); + assert!(err.contains("RUTSTER_TWILIO_WEBHOOK_BASE"), "msg: {err}"); + assert!(err.contains('&'), "msg: {err}"); + assert!( + err.contains("XML-special") || err.contains("ampersand"), + "msg: {err}" + ); + } } diff --git a/crates/rutster/src/main.rs b/crates/rutster/src/main.rs index 1d9e6bd..6c58dc9 100644 --- a/crates/rutster/src/main.rs +++ b/crates/rutster/src/main.rs @@ -100,7 +100,6 @@ async fn main() { rutster::config::drain_deadline(std::env::var("RUTSTER_DRAIN_DEADLINE_SECS").ok()) .expect("RUTSTER_DRAIN_DEADLINE_SECS must be seconds"); info!(%addr, "listening"); - let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); // Two-phase shutdown (slice-5, review M1): // SIGTERM → Drain (stop admitting; keep serving HTTP for in-flight @@ -252,7 +251,11 @@ async fn main() { } (None, None) => { // Plaintext path (plan A). Behind Caddy (default) or any - // TLS-terminating edge. + // TLS-terminating edge. Listener binds here (only the + // plaintext arm owns the bind; the TLS arm at + // serve_tls_with_nodelay above binds addr itself — two + // binders would EADDRINUSE). + let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); rutster::serve::serve_with_nodelay(listener, app, async { let _ = http_stop_rx.await; }) diff --git a/crates/rutster/src/routes.rs b/crates/rutster/src/routes.rs index 4e0a55f..924f5b8 100644 --- a/crates/rutster/src/routes.rs +++ b/crates/rutster/src/routes.rs @@ -281,9 +281,10 @@ fn stream_url_from_webhook_base(base: &url::Url) -> Result { /// `RUTSTER_TWILIO_*` set is an operator misconfiguration (Twilio was /// pointed here, the binary wasn't told its public name) — failing the /// call loudly beats returning a loopback Stream URL that dies silently -/// 10 s later. No XML escaping needed: the derived URL is -/// scheme+authority+fixed-path, no query, and `url` forbids `"` in -/// authorities. +/// 10 s later. No XML escaping needed: `config::twilio_credentials` +/// rejects non-http(s) schemes + any XML-special char (`"`, `'`, `<`, +/// `>`, `&`) at boot, so the URL's `as_str()` is interpolatable into +/// the TwiML body without an escape pass. pub async fn twilio_inbound_webhook(State(state): State) -> Response { let Some(base) = state.trunk_webhook_base.as_ref() else { return ( @@ -297,9 +298,11 @@ pub async fn twilio_inbound_webhook(State(state): State) -> Response { let stream_url = match stream_url_from_webhook_base(base) { Ok(u) => u, Err(e) => { - // Defensive: config::twilio_credentials validated the URL at - // startup, so this arm is unreachable in a correctly-booted - // binary. 500, never a panic. + // config::twilio_credentials rejects non-http(s) schemes at + // boot, so this arm is unreachable in a correctly-booted + // binary (the scheme check in stream_url_from_webhook_base + // would only fail on a non-http(s) scheme, which boot + // validation already rejected). 500, never a panic. tracing::error!(error = %e, "webhook base failed Stream-URL derivation"); return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } diff --git a/crates/rutster/tests/tls_boot.rs b/crates/rutster/tests/tls_boot.rs new file mode 100644 index 0000000..15e0443 --- /dev/null +++ b/crates/rutster/tests/tls_boot.rs @@ -0,0 +1,224 @@ +//! Integration test for the deploy-epoch-fix T3 fixes: +//! +//! 1. **rustls single-provider gate** — dropping `"ring"` from the direct +//! `rustls` + `tokio-rustls` feature lists (root Cargo.toml) leaves +//! `aws-lc-rs` as rustls's sole compiled CryptoProvider. rustls 0.23's +//! `CryptoProvider::from_crate_features()` picks the process default +//! via `#[cfg(all(feature = "aws_lc_rs", not(feature = "ring"), ...))]` +//! — when both features were on, NEITHER cfg block matched and +//! `RustlsConfig::from_pem_file` panicked with "Could not +//! automatically determine the process-level CryptoProvider". +//! +//! This test reproduces the production code path: it calls +//! `RustlsConfig::from_pem_file` WITHOUT an explicit +//! `CryptoProvider::install_default(...)` first (the existing +//! `serve_tls_nodelay.rs` test installs one up front — that's the +//! workaround, not the fix). After T3, the single-feature gate makes +//! the explicit install unnecessary. +//! +//! 2. **TLS bind moved into the plaintext match arm** — `main.rs` used +//! to bind `addr` unconditionally before branching to TLS or +//! plaintext. The TLS arm's `serve_tls_with_nodelay(addr, ...)` +//! binds `addr` itself, so the unconditional bind stole the port +//! first → EADDRINUSE the moment TLS env was set. After T3, the +//! bind lives inside the `(None, None)` plaintext arm only. This +//! test exercises the TLS arm directly to confirm it binds cleanly +//! (no competing listener). `main.rs` itself isn't invoked — the +//! test fires `serve_tls_with_nodelay` against an ephemeral port and +//! proves the same code path the TLS arm uses binds without +//! EADDRINUSE. +//! +//! Test must FAIL before the T3 fix (provider panic + EADDRINUSE) and +//! PASS after. See `.omo/plans/deploy-epoch-fix.md` T3. + +use std::sync::Arc; +use std::time::Duration; + +use axum_server::tls_rustls::RustlsConfig; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{ClientConfig, DigitallySignedStruct, SignatureScheme}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +/// A `ServerCertVerifier` that accepts any cert — test-only, for connecting +/// to a server presenting a self-signed cert outside the public trust store. +/// Do NOT use in production code paths (it would defeat TLS authentication). +#[derive(Debug)] +struct InsecureVerifier; + +impl ServerCertVerifier for InsecureVerifier { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + vec![ + SignatureScheme::RSA_PKCS1_SHA256, + SignatureScheme::ECDSA_NISTP256_SHA256, + SignatureScheme::ED25519, + SignatureScheme::RSA_PSS_SHA256, + SignatureScheme::RSA_PKCS1_SHA384, + SignatureScheme::ECDSA_NISTP384_SHA384, + SignatureScheme::RSA_PSS_SHA384, + SignatureScheme::RSA_PKCS1_SHA512, + SignatureScheme::RSA_PSS_SHA512, + SignatureScheme::ECDSA_NISTP521_SHA512, + ] + } +} + +fn client_config() -> Arc { + Arc::new( + ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(InsecureVerifier)) + .with_no_client_auth(), + ) +} + +fn write_self_signed_cert( + cert_path: &std::path::Path, + key_path: &std::path::Path, + common_name: &str, +) { + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec![common_name.to_string()]).unwrap(); + std::fs::write(cert_path, cert.pem()).unwrap(); + std::fs::write(key_path, signing_key.serialize_pem()).unwrap(); +} + +/// Pick a currently-free IPv4 loopback address. +/// +/// We bind a throw-away listener, read the ephemeral port, and drop it so the +/// server can bind the same address. The race window is tiny and acceptable +/// for a single-threaded test; a failure would surface as a bind error from +/// the server task. +async fn free_ipv4_addr() -> std::io::Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + drop(listener); + Ok(addr) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tls_boot_without_explicit_crypto_provider_install() { + // THE T3 ASSERTION: do NOT call rustls::crypto::CryptoProvider::install_default(...) + // before RustlsConfig::from_pem_file. The serve_tls_nodelay.rs test does + // install_default as a workaround; this test exercises the production code + // path, which has no install_default call in main.rs. After T3's fix + // (drop "ring" feature → single-provider cfg gate → aws-lc-rs auto-installed), + // from_pem_file succeeds without the explicit install. Before T3 it would + // panic here with "Could not automatically determine the process-level + // CryptoProvider". + + let dir = std::env::temp_dir().join(format!( + "rutster-tls-boot-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let cert_path = dir.join("cert.pem"); + let key_path = dir.join("key.pem"); + write_self_signed_cert(&cert_path, &key_path, "tls-boot-test"); + + // THE T3 ASSERTION (crypto provider): must not panic. Before the fix + // (both `ring` + `aws-lc-rs` features on rustls), this call panics + // inside axum-server's RustlsConfig::from_pem_file when it tries to + // resolve the process-level CryptoProvider. + let config = RustlsConfig::from_pem_file(&cert_path, &key_path) + .await + .expect("RustlsConfig::from_pem_file must not panic with single rustls provider"); + + // THE T3 ASSERTION (listener bind): the TLS arm is the sole binder of + // `addr` now — the unconditional `TcpListener::bind(addr)` was moved + // into the plaintext match arm in main.rs. This test exercises the + // same `serve_tls_with_nodelay(addr, ...)` path the TLS arm uses. + // Before the fix, if anything (the test infra, or — in production — + // the moved-out unconditional bind) had already bound addr, the + // server task would exit with EADDRINUSE. After the fix, the bind + // succeeds because nothing competes for the port. + let app = rutster::routes::router(rutster::session_map::AppState::default()); + let addr = free_ipv4_addr().await.unwrap(); + + let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); + let mut server = tokio::spawn(rutster::serve_tls::serve_tls_with_nodelay( + addr, + config, + app, + async { + let _ = stop_rx.await; + }, + )); + + // Give the server a moment to bind + enter the accept loop. + tokio::time::sleep(Duration::from_millis(200)).await; + + // Poll the server task: it must NOT have exited (a bind failure would + // surface as Ok(()) or Err(...)) — Pending means it's running. + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut server) + .await + .is_err(), + "server task unexpectedly exited (bind failure? EADDRINUSE?)" + ); + + // Dial with a tokio-rustls client using the test-only verifier + assert + // an https GET /healthz returns 200. + let stream = TcpStream::connect(addr).await.unwrap(); + let connector = tokio_rustls::TlsConnector::from(client_config()); + let server_name = ServerName::try_from("tls-boot-test").unwrap(); + let mut tls = connector.connect(server_name, stream).await.unwrap(); + + tls.write_all(b"GET /healthz HTTP/1.1\r\nHost: tls-boot-test\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + + let mut buf = [0u8; 4096]; + let n = tls.read(&mut buf).await.unwrap(); + let response = String::from_utf8_lossy(&buf[..n]); + assert!( + response.contains("200 OK"), + "expected 200 OK from https /healthz, got: {response}" + ); + + // The server should still be running after the request. + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut server) + .await + .is_err(), + "server task exited after serving the request" + ); + + let _ = stop_tx.send(()); + server.abort(); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/deny.toml b/deny.toml index c6695bf..58de7bb 100644 --- a/deny.toml +++ b/deny.toml @@ -41,27 +41,6 @@ ignore = [ # str0m refreshes its `time` dep to a release where the fix landed # without MSRV bump. "RUSTSEC-2026-0009", - - # RUSTSEC-2024-0421 — `idna` ≤ 0.5.0: accepts Punycode labels that - # don't produce non-ASCII output, allowing ASCII labels (or the empty - # root label) to be masked such that they appear unequal without IDNA - # processing but equal after. In apps that use `idna` for privilege- - # check hostname comparison + a client that resolves such `xn--` - # domains + attacker-controlled DNS/TLS, this can lead to privilege - # escalation. - # https://rustsec.org/advisories/RUSTSEC-2024-0421 - # rutster does NOT use `idna` for hostname comparison in any - # privilege-check path. `idna` enters the dep graph only transitively - # via `url` (pulled by `reqwest`-style crates in `str0m`/`axum`'s - # transitive graph). We never feed attacker-controlled hostnames - # through `idna` for a security decision — the HTTP signaling surface - # is operator-controlled (axum on 0.0.0.0) and the media plane speaks - # IP:port directly. Bumping `idna` past 0.5.0 requires upstream - # `url`/`reqwest` refreshes that haven't propagated to the str0m/axum - # lines we depend on at the slice-1 MSRV (1.85). Revisit when the - # workspace MSRV lifts past 1.88 (cargodeny 0.19.x installs) or when - # `url` 2.5.x refreshes its `idna` dep. - "RUSTSEC-2024-0421", ] [licenses] @@ -90,6 +69,10 @@ allow = [ "Zlib", "Unicode-DFS-2016", "Unicode-3.0", + # webpki-roots CA-bundle data license (via reqwest → hyper-rustls); + # data-only, aggregation-clean vs GPL-3.0 (CDLA-Permissive-2.0 §B.3 + # permits redistribution + combination with copyleft). + "CDLA-Permissive-2.0", ] confidence-threshold = 0.93 @@ -155,6 +138,15 @@ skip = [ "rand_core@0.6.4", "rand_chacha@0.3.1", "getrandom@0.2.17", + # `hashbrown` 0.14.5 vs 0.17.1 — transitive legacy split: 0.14.5 is + # dragged in by the older `rustls`/`tokio` chain; 0.17.1 is the new + # half coming through the `redis`/`axum-server` stack. Skip the old + # half (matching the existing skip-entry style); revisit when the + # workspace MSRV lifts past 1.88 and the older chain refreshes. + { crate = "hashbrown@0.14.5", reason = "transitive legacy; new 0.17 via redis/axum-server stack" }, + # `windows-sys` 0.52.0 vs 0.61.2 — non-target platform; both halves + # transitive. Skip the old half; revisit alongside hashbrown. + { crate = "windows-sys@0.52.0", reason = "transitive legacy pair with 0.61; non-target platform" }, ] skip-tree = [] diff --git a/deploy/.env.example b/deploy/.env.example index a0c6b31..88e9be1 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -128,21 +128,27 @@ RUTSTER_TWILIO_WEBHOOK_BASE=https://pbx.example.com RUTSTER_TAP_URL=ws://127.0.0.1:8082 ################################################################################ -# Future vars — land with sibling slices, NOT this slice +# FOB engine — TLS + metrics + Valkey event sink (slices C/D/E) ################################################################################ -# RUTSTER_TLS_CERT / RUTSTER_TLS_KEY — land with slice C (rustls Phase 1, -# BYO-cert in-process TLS). The engine binary stays compiled-with-rustls -# from slice C onward; runtime-gated by these vars. See the slice-C plan -# at docs/superpowers/plans/2026-07-05-deploy-c-rustls-tls.md (path may -# vary — check the plans dir). +# RUTSTER_TLS_CERT / RUTSTER_TLS_KEY — paths to the TLS cert + key (PEM) for +# the engine's in-process rustls HTTPS surface (deploy slice C — Phase 1, +# BYO-cert). The engine binary stays compiled-with-rustls from slice C +# onward; runtime-gated by these vars (set BOTH to enable, leave both +# unset for plaintext-behind-Caddy). See +# docs/superpowers/plans/2026-07-05-deploy-c-binary-features.md. # RUTSTER_TLS_CERT=/etc/rutster/tls/fullchain.pem # RUTSTER_TLS_KEY=/etc/rutster/tls/privkey.pem -# RUTSTER_METRICS_BIND — lands with slice D (/metrics endpoint). Default -# 127.0.0.1:9090 (internal — never routed through Caddy). +# RUTSTER_METRICS_BIND — the /metrics Prometheus scrape endpoint bind +# (deploy slice D). Default 127.0.0.1:9090 (internal — never routed +# through Caddy). # RUTSTER_METRICS_BIND=127.0.0.1:9090 -# RUTSTER_VALKEY_URL — lands with slice E (ValkeyEventSink). Unset = today's -# TracingEventSink (logs to stdout). Set to redis://valkey:6379/0 in T2. -# RUTSTER_VALKEY_URL=redis://valkey:6379/0 +# RUTSTER_VALKEY_URL — Valkey/Redis endpoint for the ValkeyEventSink +# (deploy slice E). Unset = TracingEventSink (logs to stdout, no +# durability). Set to redis://127.0.0.1:6379/ for T1 (bundled valkey) +# or redis://valkey:6379/ for T2 (compose network — service-name form). +# rediss:// (TLS) is rejected at boot: the redis crate's TLS feature is +# not compiled (spec §3.4 staging; rustls Phase 2 is trigger-gated). +# RUTSTER_VALKEY_URL=redis://valkey:6379/ diff --git a/deploy/Caddyfile b/deploy/Caddyfile index 6bd349f..76e06a5 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -47,16 +47,14 @@ {$RUTSTER_DOMAIN:localhost} { encode zstd gzip - # The reverse_proxy to the FOB. 127.0.0.1:8080 is correct in T1 - # (single container, loopback) AND T2 (compose: the engine service's - # `rutster-engine` DNS name resolves on the compose network; Caddy - # reaches it via that name — replace 127.0.0.1:8080 with - # http://engine:8080 in advance; here we use the loopback literal so - # the same Caddyfile works in both T1 and T2 spelled identically — - # for T2 deployments, the operator overrides the upstream via a env- - # substituted site-address block; the shipped default targets loopback - # for the bundled all-in-one single-container case which is the harder - # constraint). + # The reverse_proxy upstream. Caddy env-substituted form + # {$RUTSTER_UPSTREAM:127.0.0.1:8080} — preserves the T1 loopback + # default (single container) AND lets T2 compose override via the + # engine service's `RUTSTER_UPSTREAM: engine:8080` env entry. (Pre-T6 + # the upstream was a hardcoded 127.0.0.1:8080 literal — fine in T1, + # wrong in T2 where Caddy reaches the engine on the compose network + # via the `engine` DNS name, not loopback.) Same Caddyfile, two netns + # realities. # # `header_up Host {host}` + X-Forwarded-* — sets the honest hop values # (the engine's reconstruct_public_url reads these; spec §3.1 invariant 5). @@ -64,7 +62,7 @@ # the engine does NOT use X-Forwarded-For for signature validation, only # X-Forwarded-Proto/Host. Forwarding the For header is standard proxy # hygiene — engine-side RUTSTER_TRUSTED_PROXIES is the gate. - reverse_proxy 127.0.0.1:8080 { + reverse_proxy {$RUTSTER_UPSTREAM:127.0.0.1:8080} { header_up Host {host} header_up X-Real-IP {remote_host} header_up X-Forwarded-Proto {scheme} diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 72218a7..f32ce50 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -196,6 +196,19 @@ COPY --from=valkey-src /usr/local/bin/valkey-server /usr/local/bin/valkey-server COPY deploy/Caddyfile /etc/caddy/Caddyfile COPY deploy/s6-rc.d/ /etc/s6-overlay/s6-rc.d/ +# Baked-in env for the T1 all-in-one shape (deploy spec §5.6 + plan T5): +# the brain listens on 127.0.0.1:8082 (loopback; the resolve_tap_url +# gate rejects non-loopback until step 6), so the engine's tap URL MUST +# point at it. The default RUTSTER_TAP_URL baked into main.rs +# (ws://127.0.0.1:8081/echo) targets the slice-2 echo port, which +# nothing listens on in the T1 image — without this override the engine +# boots, the brain boots, but the tap handshake fails on every call. +# RUTSTER_VALKEY_URL points at the bundled valkey (loopback, +# same netns); makes the ValkeyEventSink LIVE per spec §5.6 (was a +# no-op TracingEventSink in the pre-T5 image). +ENV RUTSTER_TAP_URL=ws://127.0.0.1:8082/ +ENV RUTSTER_VALKEY_URL=redis://127.0.0.1:6379/ + # Volumes: Caddy /data (ACME state — loss risks LE lockout per TLS brief # §5 risk 5) + Valkey /var/lib/valkey (stream/state persistence per # ADR-0005). @@ -205,7 +218,10 @@ VOLUME /data /var/lib/valkey # valkey :6379 loopback; media UDP direct. EXPOSE 80 443 8080 8082 6379 49152-49407/udp -# s6-overlay v3 entrypoint — the binary `s6-overlay-init` (symlinked from -# /s6-init in the noarch tarball) runs the service bundle in -# /etc/s6-overlay/s6-rc.d/ via s6-rc. -ENTRYPOINT ["/s6-init"] +# s6-overlay v3 entrypoint — `/init` is the canonical binary installed by +# the s6-overlay noarch tarball. (Pre-T5 the entrypoint was `/s6-init` — +# a symlink the older s6-overlay v2 layout created; v3.2.0.0 ships `/init` +# directly and `/s6-init` is absent, so the allinone image failed to boot +# with `executable file not found in $PATH`.) `/init` runs the service +# bundle in /etc/s6-overlay/s6-rc.d/ via s6-rc. +ENTRYPOINT ["/init"] diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 912d5fc..2a2ef2c 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -41,6 +41,12 @@ services: - ./Caddyfile:/etc/caddy/Caddyfile:ro env_file: - .env + # T6: the Caddyfile upstream is env-substituted {$RUTSTER_UPSTREAM:127.0.0.1:8080}; + # compose overrides to reach the engine service on the shared `rutster-net` + # network via DNS name. Without this, Caddy proxies to its own loopback + # (its own container's :8080 = nothing) and every /healthz returns 502. + environment: + RUTSTER_UPSTREAM: engine:8080 depends_on: - engine networks: @@ -64,6 +70,11 @@ services: - ./Caddyfile:/etc/rutster/Caddyfile:ro # copied in image already; mount for edit-without-rebuild env_file: - .env + # T8: production default for the EventSink (spec §5.6 / valkey_sink.rs:45). + # Overridable via .env; the smoke writes redis://valkey:6379/ into .env + # (same value — both reach the engine service-name form). + environment: + RUTSTER_VALKEY_URL: redis://valkey:6379/ networks: - rutster-net diff --git a/deploy/s6-rc.d/brain/type b/deploy/s6-rc.d/brain/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/deploy/s6-rc.d/brain/type @@ -0,0 +1 @@ +longrun diff --git a/deploy/s6-rc.d/caddy/type b/deploy/s6-rc.d/caddy/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/deploy/s6-rc.d/caddy/type @@ -0,0 +1 @@ +longrun diff --git a/deploy/s6-rc.d/engine/type b/deploy/s6-rc.d/engine/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/deploy/s6-rc.d/engine/type @@ -0,0 +1 @@ +longrun diff --git a/deploy/s6-rc.d/valkey-database/dependencies.d/engine b/deploy/s6-rc.d/valkey-database/dependencies.d/engine deleted file mode 100644 index d2aea58..0000000 --- a/deploy/s6-rc.d/valkey-database/dependencies.d/engine +++ /dev/null @@ -1 +0,0 @@ -# empty — orders valkey after engine in boot sequence. diff --git a/deploy/s6-rc.d/valkey-database/type b/deploy/s6-rc.d/valkey-database/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/deploy/s6-rc.d/valkey-database/type @@ -0,0 +1 @@ +longrun diff --git a/deploy/smoke/allinone_smoke.py b/deploy/smoke/allinone_smoke.py index d7d3d08..492698f 100755 --- a/deploy/smoke/allinone_smoke.py +++ b/deploy/smoke/allinone_smoke.py @@ -22,6 +22,7 @@ MockTwilioMediaStreamsServer) + crates/rutster-trunk/tests/ws_ping.rs (the connected/start frame sequence — same JSON shapes verbatim). """ +import io import json import os import socket @@ -29,6 +30,7 @@ import ssl import struct import subprocess import sys +import tarfile import tempfile import time from base64 import b64encode @@ -59,6 +61,30 @@ def fail(msg: str) -> "NoReturn": # noqa: F821 — Python 3.11+ syntax sys.exit(1) +def docker_cp_root_cert() -> bytes: + """`docker cp :/data/caddy/pki/authorities/local/root.crt -` + streams a TAR archive on stdout, not raw PEM (T7 fix). Parse the + tar stream + extract the root.crt member's bytes.""" + r = subprocess.run( + ["docker", "cp", + f"{CONTAINER_NAME}:/data/caddy/pki/authorities/local/root.crt", "-"], + capture_output=True, timeout=10, + ) + if r.returncode != 0 or not r.stdout: + fail(f"docker cp root.crt failed: rc={r.returncode}, stderr={r.stderr.decode('utf-8', 'replace')[-500:]}") + # The tar archive's member is named "root.crt" (the basename of the + # source path). Walk the stream + extract that member. + with tarfile.open(fileobj=io.BytesIO(r.stdout)) as tar: + for member in tar.getmembers(): + if member.name == "root.crt" or member.name.endswith("/root.crt"): + f = tar.extractfile(member) + if f is None: + fail(f"docker cp tar stream: root.crt member unreadable") + return f.read() + fail(f"docker cp tar stream: no root.crt member found (members: {[m.name for m in tar.getmembers()]})") + return b"" # unreachable + + def run(cmd, check=True, timeout=30.0) -> str: """Run a command; return stdout. Fail the smoke on non-zero exit (if check).""" log(f"$ {' '.join(cmd)}") @@ -100,7 +126,7 @@ def bootstrap_container() -> None: def wait_for_root_cert() -> bytes: """Poll until Caddy wrote /data/caddy/pki/authorities/local/root.crt, - then docker cp it out. Caddy creates this on first boot with local_certs.""" + then docker cp it out (via docker_cp_root_cert's tar-extract path).""" deadline = time.time() + 30.0 while time.time() < deadline: r = subprocess.run( @@ -109,13 +135,7 @@ def wait_for_root_cert() -> bytes: capture_output=True, timeout=5, ) if r.returncode == 0: - r = subprocess.run( - ["docker", "cp", - f"{CONTAINER_NAME}:/data/caddy/pki/authorities/local/root.crt", "-"], - capture_output=True, timeout=10, - ) - if r.returncode == 0 and r.stdout: - return r.stdout + return docker_cp_root_cert() time.sleep(0.5) fail("Caddy internal CA root cert did not appear within 30s") return b"" # unreachable @@ -199,12 +219,10 @@ def open_wss(): containerized all-in-one.""" sock = socket.create_connection(("localhost", HOST_PORT), timeout=5) ctx = ssl.create_default_context(cafile=None) - r = subprocess.run( - ["docker", "cp", - f"{CONTAINER_NAME}:/data/caddy/pki/authorities/local/root.crt", "-"], - capture_output=True, timeout=10, - ) - ctx.load_verify_locations(cadata=r.stdout.decode("ascii")) + # T7: docker cp streams a tar archive, not raw PEM; use the shared + # docker_cp_root_cert() helper's tarfile extraction path. + root_cert_pem = docker_cp_root_cert() + ctx.load_verify_locations(cadata=root_cert_pem.decode("ascii")) ctx.check_hostname = False # the cert is for "localhost" — Caddy's local_certs issues it ssl_sock = ctx.wrap_socket(sock, server_hostname="localhost") log(f"TLS established (cipher={ssl_sock.cipher()})") diff --git a/deploy/smoke/compose_smoke.sh b/deploy/smoke/compose_smoke.sh index 0cc7571..e7ae5bf 100755 --- a/deploy/smoke/compose_smoke.sh +++ b/deploy/smoke/compose_smoke.sh @@ -9,23 +9,44 @@ # the WS path) — its job is to assert the four-service orchestrator shape # boots + the network_mode: "service:engine" brain->engine tap posture. # -# The ValkeyEventSink-lands-in-stream assertion is slice C's concern. -# Named hook below; do NOT implement it in this slice. +# T8: the slice-C EventSink-lands-in-stream assertion is wired in below +# (ws_handshake.py + XLEN rutster:events). The all-in-one smoke currently +# imports the same WS frames inline; ws_handshake.py is the shared client. set -euo pipefail cd "$(dirname "$0")/.." # deploy/ IMAGES_TAG="${RUTSTER_IMAGES_TAG:-smoke}" + +# T7: compose's `env_file: .env` reads the file, not shell exports. The +# pre-T7 script exported vars via `export RUTSTER_*` — those never reached +# the containers. Create deploy/.env from .env.example + apply smoke +# overrides INTO that file; trap removes it on exit. +cp .env.example .env +trap 'rm -f .env' EXIT + +# Rewrite the matching rows in .env (not the shell env). +sed -i 's|^RUTSTER_DOMAIN=.*|RUTSTER_DOMAIN=localhost|' .env +sed -i 's|^RUTSTER_ACME_EMAIL=.*|RUTSTER_ACME_EMAIL=smoke@example.com|' .env +# RUTSTER_LOCAL_CERTS is commented out in .env.example by default; uncomment + set true. +sed -i 's|^# RUTSTER_LOCAL_CERTS=.*|RUTSTER_LOCAL_CERTS=true|' .env +grep -qE '^RUTSTER_LOCAL_CERTS=' .env || echo 'RUTSTER_LOCAL_CERTS=true' >> .env +sed -i 's|^RUTSTER_HTTP_BIND=.*|RUTSTER_HTTP_BIND=0.0.0.0:8080|' .env +sed -i 's|^RUTSTER_MEDIA_BIND_IP=.*|RUTSTER_MEDIA_BIND_IP=127.0.0.1|' .env +sed -i 's|^RUTSTER_MEDIA_PORT_RANGE=.*|RUTSTER_MEDIA_PORT_RANGE=49160-49170|' .env +sed -i 's|^RUTSTER_DRAIN_DEADLINE_SECS=.*|RUTSTER_DRAIN_DEADLINE_SECS=10|' .env +# RUTSTER_VALKEY_URL + RUTSTER_TAP_URL are commented out in .env.example; uncomment + set. +sed -i 's|^# RUTSTER_VALKEY_URL=.*|RUTSTER_VALKEY_URL=redis://valkey:6379/|' .env +grep -qE '^RUTSTER_VALKEY_URL=' .env || echo 'RUTSTER_VALKEY_URL=redis://valkey:6379/' >> .env +sed -i 's|^RUTSTER_TAP_URL=.*|RUTSTER_TAP_URL=ws://127.0.0.1:8082|' .env +grep -qE '^RUTSTER_TAP_URL=' .env || echo 'RUTSTER_TAP_URL=ws://127.0.0.1:8082' >> .env +# Export too — the script's own docker compose exec / curl calls below use these. export RUTSTER_DOMAIN=localhost export RUTSTER_ACME_EMAIL=smoke@example.com export RUTSTER_LOCAL_CERTS=true -export RUTSTER_HTTP_BIND=0.0.0.0:8080 -export RUTSTER_MEDIA_BIND_IP=127.0.0.1 -export RUTSTER_MEDIA_PORT_RANGE=49160-49170 -export RUTSTER_DRAIN_DEADLINE_SECS=10 -export RUTSTER_TAP_URL=ws://127.0.0.1:8082 +echo "[compose_smoke] .env created from .env.example + smoke overrides applied" echo "[compose_smoke] overriding compose.yaml image tags -> :${IMAGES_TAG}" # Sed the image: tags from :latest to :smoke so CI's locally-built images # are picked up. In-place + restored on exit. @@ -76,16 +97,28 @@ curl --cacert /tmp/rutster-smoke-root.pem -fsS https://localhost/readyz \ || { echo "[compose_smoke] FAIL: /readyz never returned ok"; docker compose logs; exit 1; } echo "[compose_smoke] /readyz ok" -# TODO slice-C: assert lifecycle event in Valkey stream. -# Slice C lands ValkeyEventSink (spec §5.6) + the assert-event-lands-in- -# Valkey-stream smoke addition. The hook here documents the contract: -# Once slice C lands, this smoke runs: -# docker compose exec valkey valkey-cli XLEN rutster:events -# and asserts > 0 after a sim call completes (an EventSink::on_event -# call writes via XADD MAXLEN ~). Until then, the valkey service is dark -# (no consumers) — counting the stream length would always be 0. -# Leaving the named hook so slice C's plan can grep for it. -echo "[compose_smoke] PASS: T2 compose boots + /healthz + /readyz verified" +# Slice C: assert lifecycle event landed in the Valkey stream (valkey_sink.rs:45). +# ws_handshake.py drives one Twilio connected+start WS handshake against the +# engine's /twilio/media-stream path (through Caddy TLS). The engine's +# EventSink::on_event fires XADD MAXLEN ~ into the `rutster:events` stream; +# XLEN asserts >= 1 event landed. +echo "[compose_smoke] driving one WS handshake via ws_handshake.py" +python3 "$(dirname "$0")/ws_handshake.py" \ + --host localhost --port 443 --ca-pem-file /tmp/rutster-smoke-root.pem \ + || { echo "[compose_smoke] FAIL: ws_handshake.py exited non-zero"; docker compose logs; exit 1; } + +# Give the EventSink's async XADD a beat to land. The XADD is fire-and-forget +# from the media thread (spec §5.6 — never block the hot path on the bus). +sleep 1 +EVENTS=$(docker compose exec -T valkey valkey-cli XLEN rutster:events) +if [ -z "$EVENTS" ] || [ "$EVENTS" -lt 1 ]; then + echo "[compose_smoke] FAIL: no lifecycle event in rutster:events (got '$EVENTS')" >&2 + docker compose logs engine + docker compose exec -T valkey valkey-cli XRANGE rutster:events - + 2>&1 | head -40 + exit 1 +fi +echo "[compose_smoke] rutster:events stream has $EVENTS event(s)" +echo "[compose_smoke] PASS: T2 compose boots + /healthz + /readyz + EventSink verified" # Cleanup (compose down; volumes left for warm cache on re-run). docker compose down --volumes --timeout 30 || true diff --git a/deploy/smoke/ws_handshake.py b/deploy/smoke/ws_handshake.py new file mode 100755 index 0000000..8fe1508 --- /dev/null +++ b/deploy/smoke/ws_handshake.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +""" +deploy/smoke/ws_handshake.py — minimal WS client for smoke scripts (T8 factoring). + +Hand-rolled RFC 6455 frame encode/decode + the Twilio Media Streams connected/start +handshake sequence, decoupled from `docker cp` / container orchestration so both +the all-in-one smoke (`allinone_smoke.py`) + the compose smoke (`compose_smoke.sh`, +via `python3 ws_handshake.py --host ... --port ... --ca-pem-file ...`) can drive +one trunk WS handshake + assert the EventSink wrote a lifecycle event. + +No third-party deps — Python 3 + ssl + socket + json (same posture as allinone_smoke). +""" + +import argparse +import json +import os +import socket +import ssl +import struct +import sys + + +def make_ws_frame(payload: bytes, opcode: int = 0x1) -> bytes: + """Encode a client->server WS frame. opcode 0x1=text, 0x2=binary. + Client frames MUST be masked (RFC 6455 §5.3).""" + mask = os.urandom(4) + masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + header = bytearray([0x80 | opcode]) # FIN + opcode + n = len(payload) + if n < 126: + header.append(0x80 | n) + elif n < 65536: + header.append(0x80 | 126) + header.extend(struct.pack(">H", n)) + else: + header.append(0x80 | 127) + header.extend(struct.pack(">Q", n)) + header.extend(mask) + return bytes(header) + masked + + +def read_ws_frame(sock) -> tuple: + """Read one WS frame from the server. Returns (opcode, payload). + Server frames are NOT masked (RFC 6455 §5.1). + Returns (-1, b"") on EOF.""" + h1 = sock.recv(1) + if not h1: + return -1, b"" + fin_opcode = h1[0] + h2 = sock.recv(1)[0] + masked = bool(h2 & 0x80) + length = h2 & 0x7F + if length == 126: + length = struct.unpack(">H", sock.recv(2))[0] + elif length == 127: + length = struct.unpack(">Q", sock.recv(8))[0] + if masked: + mask = sock.recv(4) + payload = b"" + while len(payload) < length: + chunk = sock.recv(length - len(payload)) + if not chunk: + break + payload += chunk + if masked: + payload = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + return fin_opcode & 0x0F, payload + + +def open_wss(host: str, port: int, ca_pem: bytes, path: str = "/twilio/media-stream") -> ssl.SSLSocket: + """Open a wss:// WS handshake to the given path. `ca_pem` is the raw PEM bytes + (PEM, NOT a tar archive — the caller is responsible for extracting it from + wherever it came from). `check_hostname=False` because Caddy's local_certs + issues a cert for the SNI the operator set, not for `localhost`.""" + sock = socket.create_connection((host, port), timeout=5) + ctx = ssl.create_default_context(cafile=None) + ctx.load_verify_locations(cadata=ca_pem.decode("ascii")) + ctx.check_hostname = False + ssl_sock = ctx.wrap_socket(sock, server_hostname=host) + + key = os.urandom(16).hex() + handshake = ( + f"GET {path} HTTP/1.1\r\n" + f"Host: {host}:{port}\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + f"Sec-WebSocket-Version: 13\r\n" + f"\r\n" + ) + ssl_sock.sendall(handshake.encode("ascii")) + buf = b"" + while b"\r\n\r\n" not in buf: + chunk = ssl_sock.recv(4096) + if not chunk: + raise RuntimeError("WS upgrade response closed prematurely") + buf += chunk + if b"101 Switching Protocols" not in buf: + raise RuntimeError(f"WS upgrade did not return 101: {buf[:300]!r}") + return ssl_sock + + +def send_twilio_handshake(sock) -> None: + """Send the Twilio Media Streams connected + start frames (the same + JSON sequence crates/rutster-trunk/tests/ws_ping.rs uses against the + in-process router — here against the real containerized edge->FOB path).""" + connected = json.dumps({ + "event": "connected", + "protocol": "twilio-media-stream", + "version": "1.0.0", + }) + start = json.dumps({ + "event": "start", + "start": {"streamSid": "MZsmoke", "callSid": "CAsmoke"}, + }) + sock.sendall(make_ws_frame(connected.encode("utf-8"))) + sock.sendall(make_ws_frame(start.encode("utf-8"))) + + +def main() -> int: + """CLI entry — used by compose_smoke.sh. Drives one connected+start + handshake, drains a handful of server frames, exits 0 on success. + + Designed to be invoked from a bash smoke script that already extracted + the root cert PEM to a file (the compose smoke does this via + `docker compose exec -T caddy cat ...`).""" + p = argparse.ArgumentParser(description="Minimal WS handshake client for smoke tests.") + p.add_argument("--host", default="localhost") + p.add_argument("--port", type=int, default=443) + p.add_argument("--ca-pem-file", required=True, + help="Path to the Caddy internal CA root cert PEM (NOT a tar archive).") + p.add_argument("--path", default="/twilio/media-stream") + args = p.parse_args() + + with open(args.ca_pem_file, "rb") as f: + ca_pem = f.read() + + sock = open_wss(args.host, args.port, ca_pem, args.path) + send_twilio_handshake(sock) + # Drain a handful of frames (mark/outbound/etc.) — we just need the + # EventSink to fire on_event once; the XLEN assertion in the bash script + # is the real gate, not the frame count here. + import time + deadline = time.time() + 2.0 + received = 0 + while time.time() < deadline: + sock.settimeout(0.5) + try: + op, _ = read_ws_frame(sock) + except (socket.timeout, OSError): + break + if op in (0x1, 0x2): + received += 1 + elif op == -1: + break + sock.close() + print(f"[ws_handshake] sent connected+start; received {received} frames back") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/adr/0011-deployment-topology.md b/docs/adr/0011-deployment-topology.md index 6dec96f..d5a56c1 100644 --- a/docs/adr/0011-deployment-topology.md +++ b/docs/adr/0011-deployment-topology.md @@ -34,13 +34,17 @@ because these invariants are load-bearing for every decision below): 1. Publicly-CA-trusted cert; **no self-signed path exists** (Twilio error 31910). Auto-renewal is availability-critical. 2. wss:// on 443, open to the whole internet: Twilio publishes no egress IPs and offers no - mTLS. Auth is application-layer only (HMAC signatures). + mTLS. Auth is application-layer only (HMAC signatures — **reconstruction helper landed; + signature validation itself NOT yet enforced — the webhook surface is currently + unAuthenticated**). 3. One WS per call, up to 24 h, 50 msg/s/direction at 20 ms cadence; zero tolerance for frame buffering, connection-lifetime caps, or write-side idle timers. Neither Twilio nor Telnyx documents WS keepalive — keepalive is entirely our job. 4. `` URLs carry **no query strings** — wss routing keys on hostname/path only. 5. `X-Twilio-Signature` is HMAC over the URL *as Twilio saw it*: the edge must forward `X-Forwarded-Proto/Host` honestly and the FOB must reconstruct the public URL. + (Reconstruction helper landed; signature validation itself **planned, NOT yet + enforced** — the webhook surface is currently unauthenticated.) 6. Webhook budget: sub-second for UX, 15 s hard cap — no blocking issuance in the handshake path. 7. TLS 1.2+1.3, mainstream ECDHE, never 1.3-only; never pin Twilio certs. diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index b9c16c3..ad5c892 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -16,7 +16,10 @@ Do not put the engine behind an NLB TLS listener. Two disqualifiers, ratified in 2. **No HTTP context**: an NLB adds no `X-Forwarded-Proto/Host`, so the engine cannot reconstruct the public URL that `X-Twilio-Signature` is HMAC'd over ([Twilio's SSL-termination guidance](https://www.twilio.com/en-us/blog/developers/tutorials/building-blocks/handle-ssl-termination-twilio-node-js-helper-library)) - — genuine traffic fails validation. + — genuine traffic fails signature reconstruction (enforcement itself is **planned, + NOT yet enforced** — the webhook surface is currently unauthenticated, but the + reconstruction helper must still receive honest headers so validation lands + correctly when enforcement ships). ## The three mandatory ALB attribute overrides diff --git a/docs/deploy/certificates.md b/docs/deploy/certificates.md index 8d5d15a..1f0f668 100644 --- a/docs/deploy/certificates.md +++ b/docs/deploy/certificates.md @@ -32,7 +32,7 @@ every push) burns all five in minutes. |---|---|---| | **HTTP-01** (default) | Port 80 publicly reachable, single hostname | Zero config beyond the domain + email. | | **DNS-01** | Wildcards (`*.pbx.domain`), CGNAT/behind-NAT hosts, no port 80 | The bundled `rutster-edge` Caddy build ships a curated plugin set: **cloudflare, route53, porkbun, hetzner, desec** (duckdns excluded — no license file). Any other DNS provider requires your own xcaddy build. DNS API credentials live in the container env — scope them to the zone. | -| **BYO-cert** (in-process rustls, Phase 1) | You already have cert distribution (corporate ACME, central wildcard issuance) | Set `RUTSTER_TLS_CERT` / `RUTSTER_TLS_KEY`; the engine serves TLS itself and **hot-reloads on file change without dropping live WS**. You own renewal delivery. | +| **BYO-cert** (in-process rustls, Phase 1) | You already have cert distribution (corporate ACME, central wildcard issuance) | Set `RUTSTER_TLS_CERT` / `RUTSTER_TLS_KEY`; the engine serves TLS itself and **hot-reloads on SIGHUP (`docker kill -s HUP `) without dropping live WS**. A file-watcher is deliberately deferred — SIGHUP is the operator-side signal certbot `--deploy-hook` invokes. You own renewal delivery. | In-binary ACME (Phase 2) is deliberately deferred behind named triggers — among them Let's Encrypt's dns-persist-01 reaching GA, which would dissolve the DNS-plugin matrix entirely. See diff --git a/docs/deploy/quickstart-docker.md b/docs/deploy/quickstart-docker.md index 61da1d8..960ab50 100644 --- a/docs/deploy/quickstart-docker.md +++ b/docs/deploy/quickstart-docker.md @@ -30,6 +30,7 @@ docker run -d --name rutster --restart unless-stopped \ -e RUTSTER_ACME_EMAIL=you@example.com \ -e RUTSTER_MEDIA_ADVERTISED_IP=203.0.113.7 \ -e RUTSTER_MEDIA_PORT_RANGE=49152-49407 \ + -e RUTSTER_TRUSTED_PROXIES=127.0.0.1/32 \ -e RUTSTER_TWILIO_ACCOUNT_SID=ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx \ -e RUTSTER_TWILIO_AUTH_TOKEN=your_auth_token \ -e RUTSTER_TWILIO_MEDIA_BIND=0.0.0.0:8081 \ @@ -98,7 +99,7 @@ Operational notes: | Symptom | Cause / fix | |---|---| | Twilio error **31910** on calls | Your cert isn't publicly trusted or expired — check `/data` volume survived, see [certificates.md](certificates.md) ([twilio.com/docs/api/errors/31910](https://www.twilio.com/docs/api/errors/31910)) | -| Webhook signature validation fails | The edge isn't forwarding `X-Forwarded-Proto/Host` honestly, or `RUTSTER_TRUSTED_PROXIES` doesn't include your proxy — the engine ignores forwarded headers from unlisted sources | +| Webhook signature validation fails | The edge isn't forwarding `X-Forwarded-Proto/Host` honestly, or `RUTSTER_TRUSTED_PROXIES` doesn't include your proxy — the engine ignores forwarded headers from unlisted sources. Note: signature enforcement is **planned, NOT yet enforced** — the webhook surface is currently unauthenticated | | Browser call connects, no audio | Media UDP blocked or `RUTSTER_MEDIA_ADVERTISED_IP` wrong — the SDP advertises that IP; callers send UDP straight to it | | `curl /readyz` returns 503 | Node is draining or at the admission cap (`RUTSTER_MAX_SESSIONS`) — liveness (`/healthz`) stays 200 | | Let's Encrypt rate-limit errors in Caddy logs | You recreated the container without the `/data` volume — [certificates.md](certificates.md) | diff --git a/docs/deploy/reverse-proxies.md b/docs/deploy/reverse-proxies.md index 5c05a3f..98b7b1a 100644 --- a/docs/deploy/reverse-proxies.md +++ b/docs/deploy/reverse-proxies.md @@ -24,7 +24,9 @@ duration. Every snippet below does that. request): `X-Twilio-Signature` is HMAC over the URL as Twilio saw it ([Twilio's SSL-termination guidance](https://www.twilio.com/en-us/blog/developers/tutorials/building-blocks/handle-ssl-termination-twilio-node-js-helper-library)). 4. **Engine-side trust**: set `RUTSTER_TRUSTED_PROXIES` to your proxy's source IP/CIDR — - forwarded headers from unlisted sources are ignored, and signature validation will fail. + forwarded headers from unlisted sources are ignored, and signature reconstruction + (the URL the HMAC will be validated against) will be incorrect. Signature enforcement + itself is **planned, NOT yet enforced** — the webhook surface is currently unauthenticated. 5. **TLS 1.2+1.3, mainstream ECDHE — never 1.3-only** (Twilio's 1.3 client support is undocumented). diff --git a/docs/deploy/topologies.md b/docs/deploy/topologies.md index a69e550..319e241 100644 --- a/docs/deploy/topologies.md +++ b/docs/deploy/topologies.md @@ -102,4 +102,4 @@ away from it: | `:6379` TCP | Valkey, private | Loopback (T1) / compose network (T2) / green zone (T3) | | `:9090` TCP | FOB `/metrics`, internal | `RUTSTER_METRICS_BIND`; never routed through Caddy | | `/data` volume | Caddy | Cert/ACME state — loss class: total inbound outage | -| `/var/lib/valkey` volume | Valkey | Event stream persistence | +| `/var/lib/valkey` volume (T1) / `/data` volume (T2) | Valkey | Event stream persistence — T1 uses the valkey upstream image's default `/var/lib/valkey` (baked into the Dockerfile VOLUME); T2 compose mounts `/data` (upstream `valkey/valkey` image convention per deploy/compose.yaml — `--dir /data` flag) |