6krrt/deploy/opencode-plugin/router-outcome.js

/**
 * Report test outcomes back to the local LLM router.
 *
 * This closes the only loop the router cannot close by itself. Everything it
 * records on its own is a proxy: structural checks know whether code *parses*,
 * the local checker guesses whether prose *looks* right. Neither knows whether
 * the answer did the job. opencode does — it runs your tests.
 *
 * Hooks `tool.execute.after`, watches for test/build commands, and POSTs the
 * pass/fail to the router's /outcome endpoint. The router folds client
 * outcomes into proficiency in BOTH directions (a client's "succeeded" means
 * the work worked, unlike a parser's "ok" which only means it parsed), so this
 * is what eventually makes routing discriminate on quality.
 *
 * Install:
 *   mkdir -p ~/.config/opencode/plugins
 *   cp router-outcome.js ~/.config/opencode/plugins/
 *
 * Or per-project, in .opencode/plugins/.
 *
 * PARALLEL SESSIONS: the report carries this session's directory, which the
 * router matches against the working directory it sees in the conversation.
 * That is exact even with several sessions running. If it cannot match, and
 * more than one conversation has been routed recently, the router answers 409
 * and records nothing rather than guessing — a misattributed failure would
 * penalize a model for work it never did.
 */

const ROUTER = process.env.LLM_ROUTER_URL || "http://127.0.0.1:8080";

// Commands whose exit status is a real verdict on the work. Deliberately
// narrow: a failing `ls` says nothing about model quality, and a false signal
// is worse than no signal — it trains the router on noise.
const TEST_COMMAND = new RegExp(
  [
    "\\bpytest\\b",
    "\\bunittest\\b",
    "\\bnpm\\s+(run\\s+)?test\\b",
    "\\bpnpm\\s+(run\\s+)?test\\b",
    "\\byarn\\s+test\\b",
    "\\bvitest\\b",
    "\\bjest\\b",
    "\\bcargo\\s+(test|check|build)\\b",
    "\\bgo\\s+(test|build|vet)\\b",
    "\\bmake\\s+(test|check)\\b",
    "\\bmvn\\s+test\\b",
    "\\bgradle\\s+test\\b",
    "\\btsc\\b",
    "\\bruff\\b",
    "\\bmypy\\b",
    "\\beslint\\b",
  ].join("|"),
);

// Failure signatures, for tools that exit 0 while reporting failures.
const FAILURE_TEXT =
  /\b(\d+\s+failed|FAILED|FAIL\b|Traceback \(most recent call last\)|error(s)?:|panic:|AssertionError|✗|✖)/;

function looksFailed(output) {
  const exit = output?.exitCode ?? output?.exit_code;
  if (typeof exit === "number" && exit !== 0) return true;
  const text = `${output?.stdout ?? ""}\n${output?.stderr ?? ""}\n${
    typeof output?.output === "string" ? output.output : ""
  }`;
  // "0 failed" and "no errors" must not trip the failure regex.
  if (/\b0 failed\b|\bno errors?\b/i.test(text)) return false;
  return FAILURE_TEXT.test(text);
}

function commandOf(input) {
  const args = input?.args ?? input?.arguments ?? {};
  return args.command ?? args.cmd ?? args.script ?? "";
}

export const RouterOutcome = async ({ directory }) => {
  return {
    "tool.execute.after": async (input, output) => {
      // Only shell-ish tools carry a command whose exit status is a verdict.
      const command = commandOf(input);
      if (!command || !TEST_COMMAND.test(command)) return;

      const ok = !looksFailed(output);
      const detail = `${command.slice(0, 120)}${ok ? " — passed" : " — failed"}`;

      try {
        const res = await fetch(`${ROUTER}/outcome`, {
          method: "POST",
          headers: { "content-type": "application/json" },
          body: JSON.stringify({ ok, detail, source: directory }),
          // The router being down must never break the user's session.
          signal: AbortSignal.timeout(3000),
        });
        if (res.status === 409) {
          // Several sessions active and the directory did not match one the
          // router had seen. Dropping the sample is the correct outcome.
          console.error(
            "[router-outcome] ambiguous session; outcome not recorded",
          );
        } else if (!res.ok && res.status !== 404) {
          console.error(`[router-outcome] ${res.status} reporting outcome`);
        }
      } catch (err) {
        // Swallowed on purpose: a reporting failure is not the user's problem.
        console.error(`[router-outcome] could not reach ${ROUTER}: ${err.message}`);
      }
    },
  };
};