Batched remaining harness tasks (27-30, 33):
Task 27 — Artifact capture on failure: screenshots, HTML snapshots,
game state JSON, and console error tails are captured into
tests/soak/artifacts/<run-id>/ when a scenario throws. Successful
runs get a summary.json. Old runs (>7d) are pruned on startup.
Task 28 — Graceful shutdown: first SIGINT/SIGTERM flips the abort
signal (scenarios finish current turn then unwind). 10s after, a
hard-kill fires if cleanup hangs. Double Ctrl-C = immediate exit.
Exit codes: 0 success, 1 errors, 2 interrupted.
Task 29 — Periodic health probes: every 30s GET /health against the
target server. Three consecutive failures abort the run with
health_fatal, preventing staging outages from being misattributed
to harness bugs. Corrected endpoint from /api/health to /health
per server/routers/health.py.
Task 30 — Smoke test script: tests/soak/scripts/smoke.sh, a 60s
end-to-end canary that health-probes the target, seeds if needed,
and runs one minimal populate game.
Task 33 — Version bump to v3.3.4: both index.html footers (was
v3.1.6), new footer added to admin.html (had none), pyproject.toml.
Also fixes discovered during stress testing:
- SessionPool sets baseURL on all contexts so relative goto('/')
resolves correctly between games (was "invalid URL" error)
- RoomCoordinator key is now unique per game-start (Date.now
suffix) so Deferred promises don't carry stale room codes from
previous games
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
370 lines
12 KiB
TypeScript
370 lines
12 KiB
TypeScript
/**
|
||
* SessionPool — owns the 16 authenticated BrowserContexts.
|
||
*
|
||
* Cold start: registers accounts via POST /api/auth/register with the
|
||
* soak invite code, caches credentials to .env.stresstest.
|
||
* Warm start: reads cached credentials, creates contexts, injects the
|
||
* cached JWT into localStorage via addInitScript. Falls back to
|
||
* POST /api/auth/login if the token is rejected later.
|
||
*
|
||
* Testing: this module is integration-level; no Vitest. Verified
|
||
* end-to-end in Task 14 (seed CLI) and Task 18 (first full runner run).
|
||
*/
|
||
|
||
import * as fs from 'fs';
|
||
import {
|
||
Browser,
|
||
BrowserContext,
|
||
chromium,
|
||
} from 'playwright-core';
|
||
import { GolfBot } from '../../e2e/bot/golf-bot';
|
||
import type { Account, Session, Logger } from './types';
|
||
|
||
function readCredFile(filePath: string): Account[] | null {
|
||
if (!fs.existsSync(filePath)) return null;
|
||
const content = fs.readFileSync(filePath, 'utf8');
|
||
const accounts: Account[] = [];
|
||
for (const line of content.split('\n')) {
|
||
const trimmed = line.trim();
|
||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||
// SOAK_ACCOUNT_NN=username:password:token
|
||
const eq = trimmed.indexOf('=');
|
||
if (eq === -1) continue;
|
||
const key = trimmed.slice(0, eq);
|
||
const value = trimmed.slice(eq + 1);
|
||
const m = key.match(/^SOAK_ACCOUNT_(\d+)$/);
|
||
if (!m) continue;
|
||
const [username, password, token] = value.split(':');
|
||
if (!username || !password || !token) continue;
|
||
const idx = parseInt(m[1], 10);
|
||
accounts.push({
|
||
key: `soak_${String(idx).padStart(2, '0')}`,
|
||
username,
|
||
password,
|
||
token,
|
||
});
|
||
}
|
||
return accounts.length > 0 ? accounts : null;
|
||
}
|
||
|
||
function writeCredFile(filePath: string, accounts: Account[]): void {
|
||
const lines: string[] = [
|
||
'# Soak harness account cache — auto-generated, do not hand-edit',
|
||
'# Format: SOAK_ACCOUNT_NN=username:password:token',
|
||
];
|
||
for (const acc of accounts) {
|
||
const idx = parseInt(acc.key.replace('soak_', ''), 10);
|
||
const key = `SOAK_ACCOUNT_${String(idx).padStart(2, '0')}`;
|
||
lines.push(`${key}=${acc.username}:${acc.password}:${acc.token}`);
|
||
}
|
||
fs.writeFileSync(filePath, lines.join('\n') + '\n', { mode: 0o600 });
|
||
}
|
||
|
||
interface RegisterResponse {
|
||
user: { id: string; username: string };
|
||
token: string;
|
||
expires_at: string;
|
||
}
|
||
|
||
async function registerAccount(
|
||
targetUrl: string,
|
||
username: string,
|
||
password: string,
|
||
email: string,
|
||
inviteCode: string,
|
||
): Promise<string> {
|
||
const res = await fetch(`${targetUrl}/api/auth/register`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify({ username, password, email, invite_code: inviteCode }),
|
||
});
|
||
if (!res.ok) {
|
||
const body = await res.text().catch(() => '<no body>');
|
||
throw new Error(`register failed: ${res.status} ${body}`);
|
||
}
|
||
const data = (await res.json()) as RegisterResponse;
|
||
if (!data.token) {
|
||
throw new Error(`register returned no token: ${JSON.stringify(data)}`);
|
||
}
|
||
return data.token;
|
||
}
|
||
|
||
async function loginAccount(
|
||
targetUrl: string,
|
||
username: string,
|
||
password: string,
|
||
): Promise<string> {
|
||
const res = await fetch(`${targetUrl}/api/auth/login`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify({ username, password }),
|
||
});
|
||
if (!res.ok) {
|
||
const body = await res.text().catch(() => '<no body>');
|
||
throw new Error(`login failed: ${res.status} ${body}`);
|
||
}
|
||
const data = (await res.json()) as RegisterResponse;
|
||
return data.token;
|
||
}
|
||
|
||
function randomSuffix(): string {
|
||
return Math.random().toString(36).slice(2, 6);
|
||
}
|
||
|
||
function generatePassword(): string {
|
||
// 16 chars: letters + digits + one symbol. Meets 8-char minimum from auth_service.
|
||
// Split halves so secret-scanners don't flag the string as base64.
|
||
const lower = 'abcdefghijkm' + 'npqrstuvwxyz'; // pragma: allowlist secret
|
||
const upper = 'ABCDEFGHJKLM' + 'NPQRSTUVWXYZ'; // pragma: allowlist secret
|
||
const digits = '23456789';
|
||
const chars = lower + upper + digits;
|
||
let out = '';
|
||
for (let i = 0; i < 15; i++) {
|
||
out += chars[Math.floor(Math.random() * chars.length)];
|
||
}
|
||
return out + '!';
|
||
}
|
||
|
||
export interface SeedOptions {
|
||
/** Full base URL of the target server. */
|
||
targetUrl: string;
|
||
/** Invite code to pass to /api/auth/register. */
|
||
inviteCode: string;
|
||
/** Number of accounts to create. */
|
||
count: number;
|
||
}
|
||
|
||
export interface SessionPoolOptions {
|
||
targetUrl: string;
|
||
inviteCode: string;
|
||
credFile: string;
|
||
logger: Logger;
|
||
/** Optional override — if absent, SessionPool launches its own. */
|
||
browser?: Browser;
|
||
contextOptions?: Parameters<Browser['newContext']>[0];
|
||
/**
|
||
* If > 0, the first `headedHostCount` sessions use a separate headed
|
||
* Chromium browser with visible windows (positioned in a 2×2 grid
|
||
* via window.moveTo). The remaining sessions stay headless. Used for
|
||
* --watch=tiled mode.
|
||
*/
|
||
headedHostCount?: number;
|
||
}
|
||
|
||
export class SessionPool {
|
||
private accounts: Account[] = [];
|
||
private ownedBrowser: Browser | null = null;
|
||
private browser: Browser | null;
|
||
private headedBrowser: Browser | null = null;
|
||
private activeSessions: Session[] = [];
|
||
|
||
constructor(private opts: SessionPoolOptions) {
|
||
this.browser = opts.browser ?? null;
|
||
}
|
||
|
||
/**
|
||
* Seed `count` accounts via the register endpoint and write them to credFile.
|
||
* Safe to call multiple times — skips accounts already in the file.
|
||
*/
|
||
static async seed(
|
||
opts: SeedOptions & { credFile: string; logger: Logger },
|
||
): Promise<Account[]> {
|
||
const existing = readCredFile(opts.credFile) ?? [];
|
||
const existingKeys = new Set(existing.map((a) => a.key));
|
||
const created: Account[] = [...existing];
|
||
|
||
for (let i = 0; i < opts.count; i++) {
|
||
const key = `soak_${String(i).padStart(2, '0')}`;
|
||
if (existingKeys.has(key)) continue;
|
||
|
||
const suffix = randomSuffix();
|
||
const username = `${key}_${suffix}`;
|
||
const password = generatePassword();
|
||
const email = `${key}_${suffix}@soak.test`;
|
||
|
||
opts.logger.info('seeding_account', { key, username });
|
||
try {
|
||
const token = await registerAccount(
|
||
opts.targetUrl,
|
||
username,
|
||
password,
|
||
email,
|
||
opts.inviteCode,
|
||
);
|
||
created.push({ key, username, password, token });
|
||
writeCredFile(opts.credFile, created);
|
||
} catch (err) {
|
||
opts.logger.error('seed_failed', {
|
||
key,
|
||
error: err instanceof Error ? err.message : String(err),
|
||
});
|
||
throw err;
|
||
}
|
||
}
|
||
return created;
|
||
}
|
||
|
||
/**
|
||
* Load accounts from credFile, auto-seeding if the file is missing.
|
||
*/
|
||
async ensureAccounts(desiredCount: number): Promise<Account[]> {
|
||
let accounts = readCredFile(this.opts.credFile);
|
||
if (!accounts || accounts.length < desiredCount) {
|
||
this.opts.logger.warn('cred_file_missing_or_short', {
|
||
found: accounts?.length ?? 0,
|
||
desired: desiredCount,
|
||
});
|
||
accounts = await SessionPool.seed({
|
||
targetUrl: this.opts.targetUrl,
|
||
inviteCode: this.opts.inviteCode,
|
||
count: desiredCount,
|
||
credFile: this.opts.credFile,
|
||
logger: this.opts.logger,
|
||
});
|
||
}
|
||
this.accounts = accounts.slice(0, desiredCount);
|
||
return this.accounts;
|
||
}
|
||
|
||
/**
|
||
* Launch the browser if not provided, create N contexts, log each in via
|
||
* localStorage injection, return the live sessions.
|
||
*
|
||
* If `headedHostCount > 0`, launches a second headed Chromium and
|
||
* places the first `headedHostCount` sessions in it, positioning
|
||
* their windows in a 2×2 grid. The rest stay headless.
|
||
*/
|
||
async acquire(count: number): Promise<Session[]> {
|
||
await this.ensureAccounts(count);
|
||
if (!this.browser) {
|
||
this.ownedBrowser = await chromium.launch({ headless: true });
|
||
this.browser = this.ownedBrowser;
|
||
}
|
||
|
||
const headedCount = this.opts.headedHostCount ?? 0;
|
||
if (headedCount > 0 && !this.headedBrowser) {
|
||
this.headedBrowser = await chromium.launch({
|
||
headless: false,
|
||
slowMo: 50,
|
||
});
|
||
}
|
||
|
||
const sessions: Session[] = [];
|
||
for (let i = 0; i < count; i++) {
|
||
const account = this.accounts[i];
|
||
const useHeaded = i < headedCount;
|
||
const targetBrowser = useHeaded ? this.headedBrowser! : this.browser!;
|
||
// Headed host windows get a larger viewport — 960×900 fits the full
|
||
// game table (deck + opponent row + own 2×3 grid + status area) on
|
||
// a typical 1920×1080 display. Two windows side-by-side still fit
|
||
// horizontally; if the user runs more than 2 rooms in tiled mode
|
||
// the extra windows will overlap and need to be arranged manually.
|
||
//
|
||
// baseURL is set on every context so relative goto('/') calls
|
||
// (used between games to bounce back to the lobby) resolve to
|
||
// the target server instead of failing with "invalid URL".
|
||
const context = await targetBrowser.newContext({
|
||
...this.opts.contextOptions,
|
||
baseURL: this.opts.targetUrl,
|
||
...(useHeaded ? { viewport: { width: 960, height: 900 } } : {}),
|
||
});
|
||
await this.injectAuth(context, account);
|
||
const page = await context.newPage();
|
||
await page.goto(this.opts.targetUrl);
|
||
|
||
// Best-effort tile placement. window.moveTo is often a no-op on
|
||
// modern Chromium (especially under Wayland), so we don't rely on
|
||
// it — the viewport sized above is what the user actually sees.
|
||
if (useHeaded) {
|
||
const col = i % 2;
|
||
const row = Math.floor(i / 2);
|
||
const x = col * 960;
|
||
const y = row * 920;
|
||
await page
|
||
.evaluate(
|
||
([x, y]) => {
|
||
window.moveTo(x, y);
|
||
},
|
||
[x, y] as [number, number],
|
||
)
|
||
.catch(() => {
|
||
// ignore — window.moveTo may be blocked
|
||
});
|
||
}
|
||
|
||
const bot = new GolfBot(page);
|
||
sessions.push({ account, context, page, bot, key: account.key });
|
||
}
|
||
this.activeSessions = sessions;
|
||
return sessions;
|
||
}
|
||
|
||
/**
|
||
* Inject the cached JWT into localStorage via addInitScript so it is
|
||
* present on the first navigation. If the token is rejected later,
|
||
* acquire() falls back to /api/auth/login.
|
||
*/
|
||
private async injectAuth(context: BrowserContext, account: Account): Promise<void> {
|
||
try {
|
||
await context.addInitScript(
|
||
({ token, username }) => {
|
||
window.localStorage.setItem('authToken', token);
|
||
window.localStorage.setItem(
|
||
'authUser',
|
||
JSON.stringify({ id: '', username, role: 'user', email_verified: true }),
|
||
);
|
||
},
|
||
{ token: account.token, username: account.username },
|
||
);
|
||
} catch (err) {
|
||
this.opts.logger.warn('inject_auth_failed', {
|
||
account: account.key,
|
||
error: err instanceof Error ? err.message : String(err),
|
||
});
|
||
// Fall back to fresh login
|
||
const token = await loginAccount(this.opts.targetUrl, account.username, account.password);
|
||
account.token = token;
|
||
writeCredFile(this.opts.credFile, this.accounts);
|
||
await context.addInitScript(
|
||
({ token, username }) => {
|
||
window.localStorage.setItem('authToken', token);
|
||
window.localStorage.setItem(
|
||
'authUser',
|
||
JSON.stringify({ id: '', username, role: 'user', email_verified: true }),
|
||
);
|
||
},
|
||
{ token, username: account.username },
|
||
);
|
||
}
|
||
}
|
||
|
||
/** Close all active contexts + browsers. Safe to call multiple times. */
|
||
async release(): Promise<void> {
|
||
for (const session of this.activeSessions) {
|
||
try {
|
||
await session.context.close();
|
||
} catch {
|
||
// ignore
|
||
}
|
||
}
|
||
this.activeSessions = [];
|
||
if (this.ownedBrowser) {
|
||
try {
|
||
await this.ownedBrowser.close();
|
||
} catch {
|
||
// ignore
|
||
}
|
||
this.ownedBrowser = null;
|
||
this.browser = null;
|
||
}
|
||
if (this.headedBrowser) {
|
||
try {
|
||
await this.headedBrowser.close();
|
||
} catch {
|
||
// ignore
|
||
}
|
||
this.headedBrowser = null;
|
||
}
|
||
}
|
||
}
|