slice-5: scalability seams — addressing, admission, drain, events (review B1/M1-M7) #14

Merged
alee merged 13 commits from slice-5/scalability-seams into main 2026-07-05 04:35:43 +00:00
3 changed files with 164 additions and 7 deletions
Showing only changes of commit 5fd324e971 - Show all commits

View File

@@ -79,6 +79,20 @@ pub fn max_sessions(raw: Option<String>) -> Result<usize, String> {
}
}
/// `RUTSTER_DRAIN_DEADLINE_SECS` — how long shutdown waits for in-flight
/// calls before the hard stop. Default 0 = today's instant shutdown (dev
/// loop); production sets minutes. Calls are non-migratable by design, so
/// drain-then-terminate is the ONLY graceful scale-in shape.
pub fn drain_deadline(raw: Option<String>) -> Result<std::time::Duration, String> {
match raw {
None => Ok(std::time::Duration::ZERO),
Some(s) => s
.parse::<u64>()
.map(std::time::Duration::from_secs)
.map_err(|e| format!("RUTSTER_DRAIN_DEADLINE_SECS {s:?}: {e}")),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -141,6 +155,25 @@ mod tests {
assert!(err.contains("RUTSTER_MAX_SESSIONS"));
}
#[test]
fn drain_deadline_defaults_to_zero_when_unset() {
assert_eq!(drain_deadline(None).unwrap(), std::time::Duration::ZERO);
}
#[test]
fn drain_deadline_parses_seconds() {
assert_eq!(
drain_deadline(Some("300".into())).unwrap(),
std::time::Duration::from_secs(300)
);
}
#[test]
fn drain_deadline_rejects_garbage_with_var_name_in_error() {
let err = drain_deadline(Some("soon".into())).unwrap_err();
assert!(err.contains("RUTSTER_DRAIN_DEADLINE_SECS"));
}
#[test]
fn media_address_config_parses_all_three() {
let cfg = media_address_config(

View File

@@ -58,21 +58,55 @@ async fn main() {
// RUTSTER_TAP_BIND pattern in rutster-brain-realtime).
let addr: SocketAddr = rutster::config::http_bind(std::env::var("RUTSTER_HTTP_BIND").ok())
.expect("RUTSTER_HTTP_BIND must be host:port");
let drain_deadline =
rutster::config::drain_deadline(std::env::var("RUTSTER_DRAIN_DEADLINE_SECS").ok())
.expect("RUTSTER_DRAIN_DEADLINE_SECS must be seconds");
info!(%addr, "listening");
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
// Two-phase shutdown (slice-5, review M1):
// SIGTERM → Drain (stop admitting; keep serving HTTP for in-flight
// calls' DELETE/offer; readyz flips so the LB pulls us) → deadline
// → HTTP quiesce → media_thread.shutdown() (the hard stop).
let (http_stop_tx, http_stop_rx) = tokio::sync::oneshot::channel::<()>();
let drain_cmd_tx = app_state.cmd_tx.clone();
tokio::spawn(async move {
shutdown_signal().await;
if !drain_deadline.is_zero() {
let (reply, drained) = tokio::sync::oneshot::channel();
if drain_cmd_tx
.send(rutster::media_thread::MediaCmd::Drain { reply })
.await
.is_ok()
{
match tokio::time::timeout(drain_deadline, drained).await {
Ok(_) => info!("drain complete; proceeding to shutdown"),
Err(_) => info!(?drain_deadline, "drain deadline hit; shutting down anyway"),
}
}
}
let _ = http_stop_tx.send(());
});
axum::serve(listener, router(app_state))
.with_graceful_shutdown(shutdown_signal())
.with_graceful_shutdown(async {
let _ = http_stop_rx.await;
})
.await
.unwrap();
media_thread.shutdown();
}
/// Ctrl-C / SIGTERM handler (spec §4.5). After the signal, `main` returns
/// from `axum::serve(...)` and calls `media_thread.shutdown()`, which sends
/// `MediaCmd::Shutdown` to the dedicated thread. The thread drains its
/// session map, exits its loop, and is joined. No in-flight call
/// preservation story in the dev loop.
/// Ctrl-C / SIGTERM handler (spec §4.5). This future resolves the instant
/// the signal arrives — it does NOT wait for the drain. The caller (`main`,
/// slice-5 review M1) is the two-phase shutdown: after this resolves, it
/// sends `MediaCmd::Drain` and keeps `axum::serve` running so in-flight
/// calls can still hit DELETE/offer while readyz flips the LB away from
/// this node; once the map empties or `RUTSTER_DRAIN_DEADLINE_SECS` elapses
/// (whichever first — default 0 skips this wait entirely, preserving the
/// old instant-shutdown dev loop), HTTP quiesces and `media_thread.shutdown()`
/// sends `MediaCmd::Shutdown` for the hard stop.
async fn shutdown_signal() {
let ctrl_c = async {
tokio::signal::ctrl_c()

View File

@@ -91,6 +91,12 @@ pub enum MediaCmd {
},
/// Graceful shutdown — drain + drop + join.
Shutdown { reply: oneshot::Sender<()> },
/// Enter drain (review M1): reject new Registers ("draining" → 503),
/// keep ticking existing sessions, fire `reply` when the map empties.
/// The deadline lives with the CALLER (main.rs times out and proceeds
/// to Shutdown) — the thread just reports emptiness; that keeps
/// "how long is too long" an operator policy, not thread logic.
Drain { reply: oneshot::Sender<()> },
/// Cold-path capacity/health snapshot (readyz + the autoscaling
/// signal; review M2). Cheap: counters the loop already maintains.
Stats { reply: oneshot::Sender<MediaStats> },
@@ -216,6 +222,8 @@ fn run_media_thread(
let max_sessions = opts.max_sessions;
let mut tick_overruns: u64 = 0;
let mut last_tick_micros: u64 = 0;
let mut draining = false;
let mut drain_done: Option<oneshot::Sender<()>> = None;
info!("media thread started");
loop {
@@ -225,6 +233,14 @@ fn run_media_thread(
while let Ok(cmd) = cmd_rx.try_recv() {
match cmd {
MediaCmd::Register { tap_url, reply } => {
// Drain (review M1) takes priority over the capacity
// check below: once draining, EVERY new Register is
// shed regardless of headroom — we're bleeding out,
// not admitting.
if draining {
let _ = reply.send(Err("draining: not accepting new sessions".into()));
continue;
}
// Admission control (review M2): shed the marginal
// call with a crisp 503 instead of degrading every
// in-flight call's 20ms budget. "node full" is the
@@ -303,11 +319,19 @@ fn run_media_thread(
let _ = reply.send(MediaStats {
sessions: sessions.len(),
max_sessions,
draining: false, // Task 4 wires the real flag
draining,
tick_overruns,
last_tick_micros,
});
}
MediaCmd::Drain { reply } => {
draining = true;
if sessions.is_empty() {
let _ = reply.send(());
} else {
drain_done = Some(reply);
}
}
}
}
@@ -377,6 +401,14 @@ fn run_media_thread(
debug!(channel_id = %id, "session evicted after close");
}
// Drain completion check — after eviction so a tick that closes
// the last session completes the drain in the same iteration.
if draining && sessions.is_empty() {
if let Some(done) = drain_done.take() {
let _ = done.send(());
}
}
// === Step 3: compensated sleep (review M2). ===
// The old fixed sleep(META_TICK) silently stretched the effective
// tick to 10ms + work; under load that pushes every session past
@@ -435,6 +467,64 @@ mod tests {
thread.shutdown();
}
#[test]
fn drain_rejects_new_sessions_and_completes_when_map_empties() {
let rt = tokio::runtime::Runtime::new().unwrap();
let handle = rt.handle().clone();
let url = url::Url::parse("ws://127.0.0.1:8081/echo").unwrap();
let thread =
MediaThread::spawn(url.clone(), MediaThreadOpts::default(), handle).expect("spawn");
// one live session
let (reply, rx) = oneshot::channel();
thread
.cmd_tx()
.blocking_send(MediaCmd::Register {
tap_url: url::Url::parse("ws://127.0.0.1:1/echo").unwrap(),
reply,
})
.unwrap();
let id = rx.blocking_recv().unwrap().unwrap();
// drain: must NOT complete while the session lives
let (drain_reply, mut drain_rx) = oneshot::channel();
thread
.cmd_tx()
.blocking_send(MediaCmd::Drain { reply: drain_reply })
.unwrap();
std::thread::sleep(Duration::from_millis(50));
assert!(
drain_rx.try_recv().is_err(),
"drain must wait for in-flight sessions"
);
// new registers are shed with the contract string
let (reply, rx) = oneshot::channel();
thread
.cmd_tx()
.blocking_send(MediaCmd::Register {
tap_url: url::Url::parse("ws://127.0.0.1:1/echo").unwrap(),
reply,
})
.unwrap();
let err = rx.blocking_recv().unwrap().expect_err("draining rejects");
assert!(err.contains("draining"), "{err}");
// deleting the last session completes the drain
let (reply, rx) = oneshot::channel();
thread
.cmd_tx()
.blocking_send(MediaCmd::Delete { id, reply })
.unwrap();
rx.blocking_recv().unwrap();
assert!(
drain_rx.blocking_recv().is_ok(),
"drain completes once the map is empty"
);
thread.shutdown();
}
#[test]
fn media_thread_register_and_shutdown_round_trips() {
let rt = tokio::runtime::Runtime::new().unwrap();