slice-5: scalability seams — addressing, admission, drain, events (review B1/M1-M7) #14

Merged
alee merged 13 commits from slice-5/scalability-seams into main 2026-07-05 04:35:43 +00:00
5 changed files with 165 additions and 6 deletions
Showing only changes of commit 2e5a623b17 - Show all commits

View File

@@ -68,6 +68,17 @@ pub fn media_address_config(
Ok(cfg)
}
/// `RUTSTER_MAX_SESSIONS` — admission cap. Default 64 (placeholder until
/// the ADR-0010 benchmark measures the real per-node ceiling).
pub fn max_sessions(raw: Option<String>) -> Result<usize, String> {
match raw {
None => Ok(64),
Some(s) => s
.parse()
.map_err(|e| format!("RUTSTER_MAX_SESSIONS {s:?}: {e}")),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -114,6 +125,22 @@ mod tests {
assert!(cfg.port_range.is_none());
}
#[test]
fn max_sessions_defaults_when_unset() {
assert_eq!(max_sessions(None).unwrap(), 64);
}
#[test]
fn max_sessions_parses_override() {
assert_eq!(max_sessions(Some("128".into())).unwrap(), 128);
}
#[test]
fn max_sessions_rejects_garbage_with_var_name_in_error() {
let err = max_sessions(Some("zero".into())).unwrap_err();
assert!(err.contains("RUTSTER_MAX_SESSIONS"));
}
#[test]
fn media_address_config_parses_all_three() {
let cfg = media_address_config(

View File

@@ -44,7 +44,11 @@ async fn main() {
std::env::var("RUTSTER_MEDIA_PORT_RANGE").ok(),
)
.expect("RUTSTER_MEDIA_* env config invalid");
let opts = rutster::media_thread::MediaThreadOpts { media_cfg };
let opts = rutster::media_thread::MediaThreadOpts {
media_cfg,
max_sessions: rutster::config::max_sessions(std::env::var("RUTSTER_MAX_SESSIONS").ok())
.expect("RUTSTER_MAX_SESSIONS must be a number"),
};
let (app_state, media_thread) = app_state
.spawn_media_thread_with(opts, tokio::runtime::Handle::current())
.expect("media thread spawn");

View File

@@ -44,16 +44,22 @@ const CMD_CHANNEL_CAPACITY: usize = 64;
/// pre-slice-5 single-node dev behavior exactly.
pub struct MediaThreadOpts {
pub media_cfg: rutster_media::MediaAddressConfig,
/// Admission cap (review M2). 64 is a placeholder until the ADR-0010
/// benchmark produces the real per-node number — the gauge below is
/// that benchmark's primary readout.
pub max_sessions: usize,
}
// clippy sees a single-field struct today and suggests `#[derive(Default)]`;
// we keep the manual impl on purpose (see doc comment above) so Task 7's
// `Arc<dyn EventSink>` field doesn't force a churn-y derive → manual flip.
// clippy sees a two-field, all-derivable struct today and suggests
// `#[derive(Default)]`; we keep the manual impl on purpose (see doc comment
// above) so Task 7's `Arc<dyn EventSink>` field doesn't force a churn-y
// derive → manual flip.
#[allow(clippy::derivable_impls)]
impl Default for MediaThreadOpts {
fn default() -> Self {
Self {
media_cfg: rutster_media::MediaAddressConfig::default(),
max_sessions: 64,
}
}
}
@@ -85,6 +91,23 @@ pub enum MediaCmd {
},
/// Graceful shutdown — drain + drop + join.
Shutdown { reply: oneshot::Sender<()> },
/// Cold-path capacity/health snapshot (readyz + the autoscaling
/// signal; review M2). Cheap: counters the loop already maintains.
Stats { reply: oneshot::Sender<MediaStats> },
}
/// What the node tells the platform about itself. `serde::Serialize`
/// because /readyz returns it verbatim as JSON.
#[derive(Debug, Clone, serde::Serialize)]
pub struct MediaStats {
pub sessions: usize,
pub max_sessions: usize,
pub draining: bool,
/// Ticks whose work exceeded META_TICK — the saturation signal.
/// Overload degrades EVERY call at once (missed 20ms deadlines), so
/// this must trend at ~0; an autoscaler scales out on its slope.
pub tick_overruns: u64,
pub last_tick_micros: u64,
}
/// The handle returned to the binary. Clone the `cmd_tx` per-session;
@@ -190,13 +213,29 @@ fn run_media_thread(
media_cmd_tx: mpsc::Sender<MediaCmd>,
) {
let mut sessions: HashMap<ChannelId, ThreadSession> = HashMap::new();
let max_sessions = opts.max_sessions;
let mut tick_overruns: u64 = 0;
let mut last_tick_micros: u64 = 0;
info!("media thread started");
loop {
let tick_started = Instant::now();
// === Step 1: drain ALL pending commands (cold path) BEFORE ticking. ===
while let Ok(cmd) = cmd_rx.try_recv() {
match cmd {
MediaCmd::Register { tap_url, reply } => {
// Admission control (review M2): shed the marginal
// call with a crisp 503 instead of degrading every
// in-flight call's 20ms budget. "node full" is the
// routes-layer contract string.
if sessions.len() >= max_sessions {
let _ = reply.send(Err(format!(
"node full: {} sessions (max {max_sessions})",
sessions.len()
)));
continue;
}
match RtcSession::new_with_config(&opts.media_cfg) {
Ok(session) => {
let id = session.channel_id();
@@ -260,6 +299,15 @@ fn run_media_thread(
let _ = reply.send(());
return;
}
MediaCmd::Stats { reply } => {
let _ = reply.send(MediaStats {
sessions: sessions.len(),
max_sessions,
draining: false, // Task 4 wires the real flag
tick_overruns,
last_tick_micros,
});
}
}
}
@@ -329,8 +377,17 @@ fn run_media_thread(
debug!(channel_id = %id, "session evicted after close");
}
// === Step 3: sleep META_TICK. ===
std::thread::sleep(META_TICK);
// === Step 3: compensated sleep (review M2). ===
// The old fixed sleep(META_TICK) silently stretched the effective
// tick to 10ms + work; under load that pushes every session past
// its 20ms outbound deadline SIMULTANEOUSLY. Sleep only the
// remainder, and count the ticks where there was none to sleep.
let elapsed = tick_started.elapsed();
last_tick_micros = elapsed.as_micros() as u64;
if elapsed >= META_TICK {
tick_overruns += 1;
}
std::thread::sleep(META_TICK.saturating_sub(elapsed));
}
}
@@ -338,6 +395,46 @@ fn run_media_thread(
mod tests {
use super::*;
#[test]
fn register_rejects_when_at_capacity_and_stats_reports() {
let rt = tokio::runtime::Runtime::new().unwrap();
let handle = rt.handle().clone();
let url = url::Url::parse("ws://127.0.0.1:8081/echo").unwrap();
let opts = MediaThreadOpts {
max_sessions: 1,
..Default::default()
};
let thread = MediaThread::spawn(url.clone(), opts, handle).expect("spawn");
let register = |thread: &MediaThread| {
let (reply, rx) = oneshot::channel();
thread
.cmd_tx()
.blocking_send(MediaCmd::Register {
tap_url: url::Url::parse("ws://127.0.0.1:1/echo").unwrap(),
reply,
})
.unwrap();
rx.blocking_recv().expect("reply")
};
assert!(register(&thread).is_ok(), "first session fits");
let err = register(&thread).expect_err("second must be rejected");
assert!(err.contains("node full"), "routes contract string: {err}");
let (reply, rx) = oneshot::channel();
thread
.cmd_tx()
.blocking_send(MediaCmd::Stats { reply })
.unwrap();
let stats = rx.blocking_recv().expect("stats");
assert_eq!(stats.sessions, 1);
assert_eq!(stats.max_sessions, 1);
assert!(!stats.draining);
thread.shutdown();
}
#[test]
fn media_thread_register_and_shutdown_round_trips() {
let rt = tokio::runtime::Runtime::new().unwrap();

View File

@@ -55,6 +55,12 @@ pub async fn create_session(
(StatusCode::OK, body).into_response()
}
Err(e) => {
// Contract strings from the media thread (see media_thread.rs):
// capacity/drain rejections are retryable-elsewhere → 503 so an
// LB retries the next node; everything else stays 500.
if e.contains("node full") || e.contains("draining") {
return (StatusCode::SERVICE_UNAVAILABLE, e).into_response();
}
tracing::error!(error = ?e, "session create failed");
StatusCode::INTERNAL_SERVER_ERROR.into_response()
}

View File

@@ -58,6 +58,31 @@ async fn post_v1_sessions_returns_a_session_id() {
.unwrap();
}
#[tokio::test]
async fn create_session_returns_503_when_node_full() {
use rutster::media_thread::MediaThreadOpts;
let state = rutster::session_map::AppState::default();
let opts = MediaThreadOpts {
max_sessions: 0, // a node that can never admit — the LB-shed path
..Default::default()
};
let (state, _thread) = state
.spawn_media_thread_with(opts, tokio::runtime::Handle::current())
.expect("spawn");
let app = rutster::routes::router(state);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/v1/sessions")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE);
}
#[tokio::test]
async fn get_root_serves_html() {
let (state, media_thread) = app_state_with_thread();