6krrt/deploy/opencode-plugin/guardrails-replay.mjs

import { readFileSync, existsSync, mkdirSync, writeFileSync, rmSync, openSync, fsyncSync, closeSync } from "node:fs";
import { join } from "node:path";
import { mkdtempSync } from "node:fs";
import os from "node:os";
import { Guardrails } from "./guardrails.js";

/**
 * Guardrails Replay CLI
 *
 * Replays a set of tool calls through the guardrails engine to calibrate rules.
 *
 * Usage:
 *   node guardrails-replay.mjs [--fixture FILE] [--url URL] [--directory DIR] [--limit N] [--assume-in-scope WT]
 */

/** Rule IDs that go in the "Always-Scope" table (always fire regardless of boulder). */
const ALWAYS_RULES = new Set(["bash_banned", "bash_protected_port"]);

/** Flush a file path to ensure disk sync. */
function _fsync(path) {
  try {
    const fd = openSync(path, "r");
    fsyncSync(fd);
    closeSync(fd);
  } catch { /* non-fatal */ }
}

/**
 * Extract worktree path from a prompt string.
 *
 * Parses the first `WORKTREE: <path>` line. Path is the first whitespace-delimited
 * token after `WORKTREE: `, with one trailing `.` stripped.
 * Returns null if no WORKTREE line is found.
 */
function parseWorktreePath(prompt) {
  if (typeof prompt !== "string") return null;
  const m = prompt.match(/WORKTREE:\s*(\S+)/m);
  if (!m) return null;
  let path = m[1];
  if (path.endsWith(".")) path = path.slice(0, -1);
  return path;
}

async function main() {
  const args = process.argv.slice(2);
  const options = {
    fixture: null,
    url: null,
    directory: process.cwd(),
    limit: null,
    assumeInScope: null,
    listAll: false,
  };

  for (let i = 0; i < args.length; i++) {
    if (args[i] === "--fixture") options.fixture = args[++i];
    else if (args[i] === "--url") options.url = args[++i];
    else if (args[i] === "--directory") options.directory = args[++i];
    else if (args[i] === "--limit") options.limit = parseInt(args[++i], 10);
    else if (args[i] === "--assume-in-scope") options.assumeInScope = args[++i];
    else if (args[i] === "--list-all") options.listAll = true;
  }

  const directory = options.directory;

  // Create a temp dir for synthetic config/boulder — never under --directory.
  const omoDir = mkdtempSync(join(os.tmpdir(), "guardrails-replay-"));

  // 1. Setup a "report" config (all rules in block mode, no logging)
  const reportConfig = {
    rules: {
      task_needs_agent: "block",
      task_banned_agent: "block",
      task_worktree_line: "block",
      write_outside_worktree: "block",
      bash_main_checkout: "block",
      bash_banned: "block",
      bash_protected_port: "block",
      plan_tick_gate: "block",
    },
    protected_ports: [8080],
  };
  writeFileSync(join(omoDir, "guardrails.json"), JSON.stringify(reportConfig));

  // 2. Load tool calls
  let calls = [];
  if (options.fixture) {
    const raw = readFileSync(options.fixture, "utf-8");
    calls = JSON.parse(raw);
  } else if (options.url) {
    const sessions = await fetchSessions(options.url, directory);
    for (const s of sessions) {
      const messages = await fetchMessages(options.url, s.id, directory);
      for (const m of messages) {
        if (m.parts) {
          for (const part of m.parts) {
            if (part.type === "tool" && part.state?.status === "completed") {
              calls.push({
                sessionID: s.id,
                callID: part.callID || part.partID || part.id || `part_${s.id}_${part.index}`,
                tool: part.tool,
                args: part.state?.input || {},
                prompt: part.state?.input?.prompt,
              });
            }
          }
        }
      }
    }
  }

  if (options.limit) calls = calls.slice(0, options.limit);

  // 3. Build per-session boulders.
  //    Group calls by sessionID. Each session gets its own boulder with
  //    worktree_path derived from its first WORKTREE: line (or --assume-in-scope).
  const sessionCalls = {};
  for (const call of calls) {
    const sid = call.sessionID;
    if (!sessionCalls[sid]) sessionCalls[sid] = [];
    sessionCalls[sid].push(call);
  }

  const sessionBoulders = {};
  for (const [sid, sessionCallsList] of Object.entries(sessionCalls)) {
    // Find first WORKTREE: line from any call in this session.
    let wtPath = null;
    for (const c of sessionCallsList) {
      wtPath = parseWorktreePath(c.prompt ?? "");
      if (wtPath) break;
    }

    // Fallback: --assume-in-scope.
    if (!wtPath) {
      wtPath = options.assumeInScope;
    }

    sessionBoulders[sid] = {
      status: "active",
      session_ids: [sid],
      worktree_path: wtPath || null,
    };
  }

  // Write all session boulders to omoDir *before* Guardrails reads them.
  for (const [sid, boulder] of Object.entries(sessionBoulders)) {
    const boulderPath = join(omoDir, `boulder-${sid}.json`);
    writeFileSync(boulderPath, JSON.stringify(boulder));
  }

  try {
    // 4. Replay — create a fresh Guardrails factory per session so readBoulder
    //    picks up that session's boulder from disk.  Group by err.ruleId:
    //      Always table = bash_banned + bash_protected_port
    //      (B) table    = the other six

    const stats = {
      always: {},
      scoped: {},
    };

    const blockedCalls = [];

    const client = {
      session: {
        get: async ({ path }) => ({ data: { id: path.id, parentID: null } }),
      },
    };

    for (const [sid, sessionCallList] of Object.entries(sessionCalls)) {
      const boulder = sessionBoulders[sid];

      // Write this session's boulder to omoDir *before* creating the factory.
      const boulderPath = join(omoDir, "boulder.json");
      writeFileSync(boulderPath, JSON.stringify(boulder));
      _fsync(boulderPath);

      // Create a fresh Guardrails factory so readBoulder picks up this boulder.
      const { "tool.execute.before": hook } = await Guardrails({
        client,
        directory,
        omoDir: omoDir,
      });

      for (const call of sessionCallList) {
        const input = { sessionID: sid, tool: call.tool, callID: call.callID };
        const output = { args: { ...call.args } };

        try {
          await hook(input, output);

          // Check if prompt was rewritten by task_worktree_line
          if (call.tool === "task" && output.args?.prompt !== call.args?.prompt) {
            updateStat("task_worktree_line", "rewrite", stats);
          }
        } catch (err) {
          const ruleId = err.ruleId;
          if (!ruleId) {
            console.error(`Unexpected error during replay of ${call.callID}:`, err);
            continue;
          }

          const type = "block";
          updateStat(ruleId, type, stats);

          if (options.listAll) {
            const content = call.args?.command || call.args?.description || "";
            const truncated = content.length > 160 ? content.slice(0, 160) : content;
            blockedCalls.push(`${ruleId} | ${call.tool} | ${truncated}`);
          }

          // Collect examples (max 5 per rule)
          const bucket = getBucket(ruleId, stats);
          if (!bucket.examples) bucket.examples = [];
          if (bucket.examples.length < 5) {
            bucket.examples.push(JSON.stringify(call.args));
          }
        }
      }
    }

    // 6. Print Results
    const printTable = (title, data) => {
      console.log(`\n${title}`);
      console.log("Rule | Blocked | Rewritten | Examples");
      console.log("-----|----------|-----------|----------");
      for (const [id, s] of Object.entries(data)) {
        console.log(`${id} | ${s.block} | ${s.rewrite || 0} | ${s.examples ? s.examples.join("; ") : ""}`);
      }
    };

    printTable("Always-Scope Rules", stats.always);
    printTable("(B) Scoped Rules", stats.scoped);

    if (options.listAll && blockedCalls.length > 0) {
      console.log("\nBlocked Calls Detail");
      console.log("Rule | Tool | Command/Description");
      console.log("----|------|---------------------");
      for (const line of blockedCalls) {
        console.log(line);
      }
    }
  } finally {
    rmSync(omoDir, { recursive: true, force: true });
  }
}

/** Get the stat bucket for a ruleId based on grouping. */
function getBucket(ruleId, stats) {
  const bucket = ALWAYS_RULES.has(ruleId)
    ? stats.always
    : stats.scoped;
  if (!bucket[ruleId]) {
    bucket[ruleId] = { block: 0, rewrite: 0, examples: [] };
  }
  return bucket[ruleId];
}

/** Increment a stat counter. */
function updateStat(ruleId, type, stats) {
  const bucket = getBucket(ruleId, stats);
  if (type === "block") bucket.block++;
  if (type === "rewrite") bucket.rewrite++;
}

/** Fetch sessions from the opencode API. */
async function fetchSessions(url, directory) {
  const params = new URLSearchParams({ directory });
  const resp = await fetch(`${url}/session?${params}`);
  if (!resp.ok) throw new Error(`GET ${url}/session failed: ${resp.status} ${resp.statusText}`);
  return resp.json();
}

/** Fetch messages for a session from the opencode API. */
async function fetchMessages(url, sessionId, directory) {
  const params = new URLSearchParams({ directory });
  const resp = await fetch(`${url}/session/${sessionId}/message?${params}`);
  if (!resp.ok) throw new Error(`GET ${url}/session/${sessionId}/message failed: ${resp.status} ${resp.statusText}`);
  return resp.json();
}

main().catch(console.error);