diff --git a/tests/soak/core/watchdog.ts b/tests/soak/core/watchdog.ts new file mode 100644 index 0000000..b79fd01 --- /dev/null +++ b/tests/soak/core/watchdog.ts @@ -0,0 +1,44 @@ +/** + * Watchdog — simple per-room timeout detector. + * + * `start()` begins a countdown. `heartbeat()` resets it. If the + * countdown elapses without a heartbeat, `onTimeout` fires once + * (subsequent heartbeats are no-ops after firing, unless `start()` + * is called again). `stop()` cancels any pending timer. + * + * Used by the runner to detect stuck rooms: one watchdog per room, + * scenarios call ctx.heartbeat(roomId) at each progress point, and + * a firing watchdog logs + aborts the run. + */ + +export class Watchdog { + private timer: NodeJS.Timeout | null = null; + private fired = false; + + constructor( + private timeoutMs: number, + private onTimeout: () => void, + ) {} + + start(): void { + this.stop(); + this.fired = false; + this.timer = setTimeout(() => { + if (this.fired) return; + this.fired = true; + this.onTimeout(); + }, this.timeoutMs); + } + + heartbeat(): void { + if (this.fired) return; + this.start(); + } + + stop(): void { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + } +} diff --git a/tests/soak/runner.ts b/tests/soak/runner.ts index ff13380..2c19238 100644 --- a/tests/soak/runner.ts +++ b/tests/soak/runner.ts @@ -17,6 +17,7 @@ import { SessionPool } from './core/session-pool'; import { RoomCoordinator } from './core/room-coordinator'; import { DashboardServer } from './dashboard/server'; import { Screencaster } from './core/screencaster'; +import { Watchdog } from './core/watchdog'; import { getScenario, listScenarios } from './scenarios'; import type { DashboardReporter, ScenarioContext, Session } from './core/types'; @@ -123,6 +124,7 @@ async function main(): Promise { let dashboardServer: DashboardServer | null = null; let dashboard: DashboardReporter = noopDashboard(); + const watchdogs = new Map(); let exitCode = 0; try { const sessions = await pool.acquire(accounts); @@ -179,6 +181,20 @@ async function main(): Promise { } } + // Per-room watchdogs — fire if no heartbeat arrives within 60s. + // Declared at outer scope so the finally block can stop them and + // drain any pending timers before the process exits. + for (let i = 0; i < rooms; i++) { + const roomId = `room-${i}`; + const w = new Watchdog(60_000, () => { + logger.error('watchdog_fired', { room: roomId }); + dashboard.update(roomId, { phase: 'error' }); + abortController.abort(); + }); + w.start(); + watchdogs.set(roomId, w); + } + const ctx: ScenarioContext = { config, sessions, @@ -186,8 +202,9 @@ async function main(): Promise { dashboard, logger, signal: abortController.signal, - heartbeat: () => { - // Task 26 wires per-room watchdogs. No-op until then. + heartbeat: (roomId: string) => { + const w = watchdogs.get(roomId); + if (w) w.heartbeat(); }, }; @@ -214,6 +231,7 @@ async function main(): Promise { }); exitCode = 1; } finally { + for (const w of watchdogs.values()) w.stop(); await screencaster.stopAll(); await pool.release(); if (dashboardServer) { diff --git a/tests/soak/scenarios/shared/multiplayer-game.ts b/tests/soak/scenarios/shared/multiplayer-game.ts index 6c4d51d..09b7162 100644 --- a/tests/soak/scenarios/shared/multiplayer-game.ts +++ b/tests/soak/scenarios/shared/multiplayer-game.ts @@ -49,6 +49,12 @@ export async function runOneMultiplayerGame( const maxDuration = opts.maxDurationMs ?? 5 * 60_000; try { + // Reset every session back to the lobby before starting. + // After the first game ends each session is parked on the + // game_over screen, which hides the lobby's Create Room button. + // goto('/') bounces them back; localStorage-cached auth persists. + await Promise.all(sessions.map((s) => s.bot.goto('/'))); + // Host creates game and announces the code const code = await host.bot.createGame(host.account.username); ctx.coordinator.announce(opts.roomId, code); diff --git a/tests/soak/tests/watchdog.test.ts b/tests/soak/tests/watchdog.test.ts new file mode 100644 index 0000000..ec4f308 --- /dev/null +++ b/tests/soak/tests/watchdog.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Watchdog } from '../core/watchdog'; + +describe('Watchdog', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('fires after timeout if no heartbeat', () => { + const onTimeout = vi.fn(); + const w = new Watchdog(1000, onTimeout); + w.start(); + vi.advanceTimersByTime(1001); + expect(onTimeout).toHaveBeenCalledOnce(); + }); + + it('heartbeat resets the timer', () => { + const onTimeout = vi.fn(); + const w = new Watchdog(1000, onTimeout); + w.start(); + vi.advanceTimersByTime(800); + w.heartbeat(); + vi.advanceTimersByTime(800); + expect(onTimeout).not.toHaveBeenCalled(); + vi.advanceTimersByTime(300); + expect(onTimeout).toHaveBeenCalledOnce(); + }); + + it('stop cancels pending timeout', () => { + const onTimeout = vi.fn(); + const w = new Watchdog(1000, onTimeout); + w.start(); + w.stop(); + vi.advanceTimersByTime(2000); + expect(onTimeout).not.toHaveBeenCalled(); + }); + + it('does not fire twice after stop', () => { + const onTimeout = vi.fn(); + const w = new Watchdog(1000, onTimeout); + w.start(); + vi.advanceTimersByTime(1001); + w.heartbeat(); + vi.advanceTimersByTime(1001); + expect(onTimeout).toHaveBeenCalledOnce(); + }); +});