/** * Report test outcomes back to the local LLM router. * * This closes the only loop the router cannot close by itself. Everything it * records on its own is a proxy: structural checks know whether code *parses*, * the local checker guesses whether prose *looks* right. Neither knows whether * the answer did the job. opencode does — it runs your tests. * * Hooks `tool.execute.after`, watches for test/build commands, and POSTs the * pass/fail to the router's /outcome endpoint. The router folds client * outcomes into proficiency in BOTH directions (a client's "succeeded" means * the work worked, unlike a parser's "ok" which only means it parsed), so this * is what eventually makes routing discriminate on quality. * * Install: * mkdir -p ~/.config/opencode/plugins * cp router-outcome.js ~/.config/opencode/plugins/ * * Or per-project, in .opencode/plugins/. * * PARALLEL SESSIONS: the report carries this session's directory, which the * router matches against the working directory it sees in the conversation. * That is exact even with several sessions running. If it cannot match, and * more than one conversation has been routed recently, the router answers 409 * and records nothing rather than guessing — a misattributed failure would * penalize a model for work it never did. */ const ROUTER = process.env.LLM_ROUTER_URL || "http://127.0.0.1:8080"; // Commands whose exit status is a real verdict on the work. Deliberately // narrow: a failing `ls` says nothing about model quality, and a false signal // is worse than no signal — it trains the router on noise. const TEST_COMMAND = new RegExp( [ "\\bpytest\\b", "\\bunittest\\b", "\\bnpm\\s+(run\\s+)?test\\b", "\\bpnpm\\s+(run\\s+)?test\\b", "\\byarn\\s+test\\b", "\\bvitest\\b", "\\bjest\\b", "\\bcargo\\s+(test|check|build)\\b", "\\bgo\\s+(test|build|vet)\\b", "\\bmake\\s+(test|check)\\b", "\\bmvn\\s+test\\b", "\\bgradle\\s+test\\b", "\\btsc\\b", "\\bruff\\b", "\\bmypy\\b", "\\beslint\\b", ].join("|"), ); // Failure signatures, for tools that exit 0 while reporting failures. const FAILURE_TEXT = /\b(\d+\s+failed|FAILED|FAIL\b|Traceback \(most recent call last\)|error(s)?:|panic:|AssertionError|✗|✖)/; function looksFailed(output) { const exit = output?.exitCode ?? output?.exit_code; if (typeof exit === "number" && exit !== 0) return true; const text = `${output?.stdout ?? ""}\n${output?.stderr ?? ""}\n${ typeof output?.output === "string" ? output.output : "" }`; // "0 failed" and "no errors" must not trip the failure regex. if (/\b0 failed\b|\bno errors?\b/i.test(text)) return false; return FAILURE_TEXT.test(text); } function commandOf(input) { const args = input?.args ?? input?.arguments ?? {}; return args.command ?? args.cmd ?? args.script ?? ""; } export const RouterOutcome = async ({ directory }) => { return { "tool.execute.after": async (input, output) => { // Only shell-ish tools carry a command whose exit status is a verdict. const command = commandOf(input); if (!command || !TEST_COMMAND.test(command)) return; const ok = !looksFailed(output); const detail = `${command.slice(0, 120)}${ok ? " — passed" : " — failed"}`; try { const res = await fetch(`${ROUTER}/outcome`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ ok, detail, source: directory }), // The router being down must never break the user's session. signal: AbortSignal.timeout(3000), }); if (res.status === 409) { // Several sessions active and the directory did not match one the // router had seen. Dropping the sample is the correct outcome. console.error( "[router-outcome] ambiguous session; outcome not recorded", ); } else if (!res.ok && res.status !== 404) { console.error(`[router-outcome] ${res.status} reporting outcome`); } } catch (err) { // Swallowed on purpose: a reporting failure is not the user's problem. console.error(`[router-outcome] could not reach ${ROUTER}: ${err.message}`); } }, }; };