Watchdog class with 4 Vitest tests (27 total now), wired into ctx.heartbeat in the runner. One watchdog per room with a 60s timeout; firing logs an error, marks the room's dashboard tile as errored, and triggers the abort signal so the scenario unwinds. Watchdogs are explicitly stopped in the runner's finally block so pending timers don't keep the node process alive on exit. Also fixes a multi-game bug discovered during stress scenario verification: after a game ends sessions stay parked on the game_over screen, which hides the lobby and makes a subsequent #create-room-btn click time out. runOneMultiplayerGame now navigates every session to / before each game — localStorage auth persists so nothing re-logs in. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
250 lines
7.7 KiB
TypeScript
250 lines
7.7 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/**
|
|
* Golf Soak Harness — entry point.
|
|
*
|
|
* Usage:
|
|
* TEST_URL=http://localhost:8000 \
|
|
* SOAK_INVITE_CODE=SOAKTEST \
|
|
* bun run soak -- --scenario=populate --rooms=1 --accounts=2 \
|
|
* --cpus-per-room=0 --games-per-room=1 --holes=1 --watch=none
|
|
*/
|
|
|
|
import * as path from 'path';
|
|
import { spawn } from 'child_process';
|
|
import { parseArgs, mergeConfig, CliArgs } from './config';
|
|
import { createLogger } from './core/logger';
|
|
import { SessionPool } from './core/session-pool';
|
|
import { RoomCoordinator } from './core/room-coordinator';
|
|
import { DashboardServer } from './dashboard/server';
|
|
import { Screencaster } from './core/screencaster';
|
|
import { Watchdog } from './core/watchdog';
|
|
import { getScenario, listScenarios } from './scenarios';
|
|
import type { DashboardReporter, ScenarioContext, Session } from './core/types';
|
|
|
|
function noopDashboard(): DashboardReporter {
|
|
return {
|
|
update: () => {},
|
|
log: () => {},
|
|
incrementMetric: () => {},
|
|
};
|
|
}
|
|
|
|
function printScenarioList(): void {
|
|
console.log('Available scenarios:');
|
|
for (const s of listScenarios()) {
|
|
console.log(` ${s.name.padEnd(12)} ${s.description}`);
|
|
console.log(
|
|
` needs: accounts=${s.needs.accounts}, rooms=${s.needs.rooms ?? 1}, cpus=${s.needs.cpusPerRoom ?? 0}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const cli: CliArgs = parseArgs(process.argv.slice(2));
|
|
|
|
if (cli.listOnly) {
|
|
printScenarioList();
|
|
return;
|
|
}
|
|
|
|
if (!cli.scenario) {
|
|
console.error('Error: --scenario=<name> is required. Use --list to see scenarios.');
|
|
process.exit(2);
|
|
}
|
|
|
|
const scenario = getScenario(cli.scenario);
|
|
if (!scenario) {
|
|
console.error(`Error: unknown scenario "${cli.scenario}". Use --list to see scenarios.`);
|
|
process.exit(2);
|
|
}
|
|
|
|
const runId =
|
|
cli.runId ?? `${cli.scenario}-${new Date().toISOString().replace(/[:.]/g, '-')}`;
|
|
const targetUrl = cli.target ?? process.env.TEST_URL ?? 'http://localhost:8000';
|
|
const inviteCode = process.env.SOAK_INVITE_CODE ?? 'SOAKTEST';
|
|
const watch = cli.watch ?? 'dashboard';
|
|
|
|
const logger = createLogger({ runId });
|
|
logger.info('run_start', {
|
|
scenario: scenario.name,
|
|
targetUrl,
|
|
watch,
|
|
cli,
|
|
});
|
|
|
|
// Resolve final config: scenarioDefaults → env → CLI (later wins)
|
|
const config = mergeConfig(
|
|
cli as Record<string, unknown>,
|
|
process.env,
|
|
scenario.defaultConfig,
|
|
);
|
|
|
|
// Ensure core knobs exist, falling back to scenario.needs
|
|
const accounts = Number(config.accounts ?? scenario.needs.accounts);
|
|
const rooms = Number(config.rooms ?? scenario.needs.rooms ?? 1);
|
|
const cpusPerRoom = Number(config.cpusPerRoom ?? scenario.needs.cpusPerRoom ?? 0);
|
|
if (accounts % rooms !== 0) {
|
|
console.error(
|
|
`Error: --accounts=${accounts} does not divide evenly into --rooms=${rooms}`,
|
|
);
|
|
process.exit(2);
|
|
}
|
|
config.accounts = accounts;
|
|
config.rooms = rooms;
|
|
config.cpusPerRoom = cpusPerRoom;
|
|
|
|
if (cli.dryRun) {
|
|
logger.info('dry_run', { config });
|
|
console.log('Dry run OK. Resolved config:');
|
|
console.log(JSON.stringify(config, null, 2));
|
|
return;
|
|
}
|
|
|
|
// Build dependencies
|
|
const credFile = path.resolve(__dirname, '.env.stresstest');
|
|
const headedHostCount = watch === 'tiled' ? rooms : 0;
|
|
const pool = new SessionPool({
|
|
targetUrl,
|
|
inviteCode,
|
|
credFile,
|
|
logger,
|
|
headedHostCount,
|
|
});
|
|
const coordinator = new RoomCoordinator();
|
|
const screencaster = new Screencaster(logger);
|
|
|
|
const abortController = new AbortController();
|
|
|
|
const onSignal = (sig: string) => {
|
|
logger.warn('signal_received', { signal: sig });
|
|
abortController.abort();
|
|
};
|
|
process.on('SIGINT', () => onSignal('SIGINT'));
|
|
process.on('SIGTERM', () => onSignal('SIGTERM'));
|
|
|
|
let dashboardServer: DashboardServer | null = null;
|
|
let dashboard: DashboardReporter = noopDashboard();
|
|
const watchdogs = new Map<string, Watchdog>();
|
|
let exitCode = 0;
|
|
try {
|
|
const sessions = await pool.acquire(accounts);
|
|
logger.info('sessions_acquired', { count: sessions.length });
|
|
|
|
// Build a session lookup for click-to-watch
|
|
const sessionsByKey = new Map<string, Session>();
|
|
for (const s of sessions) sessionsByKey.set(s.key, s);
|
|
|
|
// Dashboard with screencaster handlers now that sessions exist
|
|
if (watch === 'dashboard') {
|
|
const port = Number(config.dashboardPort ?? 7777);
|
|
dashboardServer = new DashboardServer(port, logger, {
|
|
onStartStream: (key) => {
|
|
const session = sessionsByKey.get(key);
|
|
if (!session) {
|
|
logger.warn('stream_start_unknown_session', { sessionKey: key });
|
|
return;
|
|
}
|
|
screencaster
|
|
.start(key, session.page, (jpegBase64) => {
|
|
dashboardServer!.broadcast({ type: 'frame', sessionKey: key, jpegBase64 });
|
|
})
|
|
.catch((err) =>
|
|
logger.error('screencast_start_failed', {
|
|
sessionKey: key,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
}),
|
|
);
|
|
},
|
|
onStopStream: (key) => {
|
|
screencaster.stop(key).catch(() => {
|
|
// best-effort — errors already logged inside Screencaster
|
|
});
|
|
},
|
|
onDisconnect: () => {
|
|
screencaster.stopAll().catch(() => {});
|
|
},
|
|
});
|
|
await dashboardServer.start();
|
|
dashboard = dashboardServer.reporter();
|
|
const url = `http://localhost:${port}`;
|
|
console.log(`Dashboard: ${url}`);
|
|
try {
|
|
const opener =
|
|
process.platform === 'darwin'
|
|
? 'open'
|
|
: process.platform === 'win32'
|
|
? 'start'
|
|
: 'xdg-open';
|
|
spawn(opener, [url], { stdio: 'ignore', detached: true }).unref();
|
|
} catch {
|
|
// If auto-open fails, the URL is already printed.
|
|
}
|
|
}
|
|
|
|
// Per-room watchdogs — fire if no heartbeat arrives within 60s.
|
|
// Declared at outer scope so the finally block can stop them and
|
|
// drain any pending timers before the process exits.
|
|
for (let i = 0; i < rooms; i++) {
|
|
const roomId = `room-${i}`;
|
|
const w = new Watchdog(60_000, () => {
|
|
logger.error('watchdog_fired', { room: roomId });
|
|
dashboard.update(roomId, { phase: 'error' });
|
|
abortController.abort();
|
|
});
|
|
w.start();
|
|
watchdogs.set(roomId, w);
|
|
}
|
|
|
|
const ctx: ScenarioContext = {
|
|
config,
|
|
sessions,
|
|
coordinator,
|
|
dashboard,
|
|
logger,
|
|
signal: abortController.signal,
|
|
heartbeat: (roomId: string) => {
|
|
const w = watchdogs.get(roomId);
|
|
if (w) w.heartbeat();
|
|
},
|
|
};
|
|
|
|
const result = await scenario.run(ctx);
|
|
logger.info('run_complete', {
|
|
gamesCompleted: result.gamesCompleted,
|
|
errors: result.errors.length,
|
|
durationMs: result.durationMs,
|
|
});
|
|
console.log(`Games completed: ${result.gamesCompleted}`);
|
|
console.log(`Errors: ${result.errors.length}`);
|
|
console.log(`Duration: ${(result.durationMs / 1000).toFixed(1)}s`);
|
|
if (result.errors.length > 0) {
|
|
console.log('Errors:');
|
|
for (const e of result.errors) {
|
|
console.log(` ${e.room}: ${e.reason}${e.detail ? ' — ' + e.detail : ''}`);
|
|
}
|
|
exitCode = 1;
|
|
}
|
|
} catch (err) {
|
|
logger.error('run_failed', {
|
|
error: err instanceof Error ? err.message : String(err),
|
|
stack: err instanceof Error ? err.stack : undefined,
|
|
});
|
|
exitCode = 1;
|
|
} finally {
|
|
for (const w of watchdogs.values()) w.stop();
|
|
await screencaster.stopAll();
|
|
await pool.release();
|
|
if (dashboardServer) {
|
|
await dashboardServer.stop();
|
|
}
|
|
}
|
|
|
|
if (abortController.signal.aborted && exitCode === 0) exitCode = 2;
|
|
process.exit(exitCode);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|