#!/usr/bin/env node // R2 faithful-arm preflight — prove "installed = running as designed" before a // priced run. Targets the R1 finding that the agent logged 102 native `bash` // calls and 0 `ctx_shell` calls, so the heaviest addressable surface in a fix // task (make / reproducer / test logs) never reached the compressor (#361). // // It verifies, on THIS machine, that: // 1. the lean-ctx binary is present (and reports its version), // 2. the resolved pi config suppresses native `bash` (mode=replace or // routeShell), so the agent must use `ctx_shell` — the suppression itself // is the unit-tested invariant `resolveSuppressedBuiltins` in // packages/pi-lean-ctx/extensions/config.ts, // 3. the embedded MCP bridge / session cache is enabled, // 4. the faithful-arm overhead levers are set (rules_injection=off, // tool_profile=minimal, structure_first), // 5. lean-ctx actually compresses shell output (measured: smaller than raw). // // A green run is the precondition devasur asked for: shell routes through // ctx_shell and is metered, not native bash. // // Usage: // node bench/agent-task/r2/preflight.mjs [--config ] // // POSIX shell is assumed for the compression probe (the R2 rail is the // forge-cli / pi runtime on Linux). Exit code 1 if any hard gate fails. import { execFileSync } from "node:child_process"; import { existsSync, readFileSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); const RECOMMENDED_MIN_VERSION = "3.8.6"; // anti-inflation guard + bridged ctx_read // ── tiny PASS/FAIL/WARN reporter ────────────────────────────────────────── let failed = false; const pass = (name, detail) => console.log(` [PASS] ${name.padEnd(24)} ${detail}`); const warn = (name, detail) => console.log(` [WARN] ${name.padEnd(24)} ${detail}`); const fail = (name, detail) => { console.log(` [FAIL] ${name.padEnd(24)} ${detail}`); failed = true; }; const gate = (ok, name, okDetail, failDetail) => ok ? pass(name, okDetail) : fail(name, failDetail); // ── config resolution (mirrors extensions/config.ts precedence) ─────────── function envFlag(name) { const raw = process.env[name]; if (!raw) return false; const v = raw.trim().toLowerCase(); return v === "1" || v === "true" || v === "yes" || v === "on"; } function parseConfigArg() { const i = process.argv.indexOf("--config"); return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : undefined; } function resolveConfigPath() { const explicit = parseConfigArg(); if (explicit) return explicit; const installed = resolve( homedir(), ".pi", "agent", "extensions", "pi-lean-ctx", "config.json", ); if (existsSync(installed)) return installed; // Fall back to this repo's reference config so the preflight self-tests // locally even when the extension is not installed on the dev machine. return resolve(SCRIPT_DIR, "pi-config.json"); } function readConfig(path) { if (!existsSync(path)) return {}; try { const parsed = JSON.parse(readFileSync(path, "utf8")); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {}; } catch (err) { fail("config.json", `invalid JSON at ${path} (${err.message})`); return {}; } } // Effective env value: an explicit process env always wins over the config // `env` block the extension forwards to every lean-ctx subprocess. function effectiveEnv(cfg, key) { return process.env[key] ?? (cfg.env && typeof cfg.env === "object" ? cfg.env[key] : undefined); } function resolveMode(cfg) { const raw = (process.env.LEAN_CTX_PI_MODE ?? cfg.mode ?? "additive").toLowerCase(); return raw === "replace" ? "replace" : "additive"; } function resolveRouteShell(mode, cfg) { if (mode === "replace") return true; if (process.env.LEAN_CTX_PI_ROUTE_SHELL !== undefined) return envFlag("LEAN_CTX_PI_ROUTE_SHELL"); return cfg.routeShell === true; } function resolveEnableMcp(cfg) { return process.env.LEAN_CTX_PI_ENABLE_MCP !== undefined ? envFlag("LEAN_CTX_PI_ENABLE_MCP") : cfg.enableMcp !== false; } function resolveBinary() { return process.env.LEAN_CTX_BIN || "lean-ctx"; } function compareSemver(a, b) { const pa = a.split(".").map(Number); const pb = b.split(".").map(Number); for (let i = 0; i < 3; i++) { const d = (pa[i] || 0) - (pb[i] || 0); if (d !== 0) return d < 0 ? -1 : 1; } return 0; } // ── individual checks ───────────────────────────────────────────────────── function checkBinary(bin) { try { const out = execFileSync(bin, ["--version"], { encoding: "utf8" }).trim(); pass("lean-ctx binary", out); const m = out.match(/(\d+\.\d+\.\d+)/); if (m && compareSemver(m[1], RECOMMENDED_MIN_VERSION) < 0) { warn("version", `${m[1]} < ${RECOMMENDED_MIN_VERSION} — pin a release with the anti-inflation + routeShell fixes`); } return true; } catch (err) { fail("lean-ctx binary", `not runnable (${err.message}) — set LEAN_CTX_BIN or add lean-ctx to PATH`); return false; } } function checkShellRouting(mode, routeShell) { // resolveSuppressedBuiltins (config.ts, unit-tested): replace ⇒ all natives, // additive+routeShell ⇒ just bash, additive ⇒ none. bash gone ⟺ the agent // cannot pick native bash and must use ctx_shell. const bashSuppressed = mode === "replace" || routeShell; gate( bashSuppressed, "shell routing", `native bash suppressed (mode=${mode}, routeShell=${routeShell}) → ctx_shell only`, `native bash still exposed (mode=${mode}, routeShell=${routeShell}) — set "mode":"replace" or "routeShell":true, else the agent reproduces R1's 102 bash / 0 ctx_shell`, ); } function checkBridge(enableMcp) { gate( enableMcp, "session cache", "embedded MCP bridge enabled (unchanged re-reads ~13 tokens)", 'embedded bridge disabled — remove "enableMcp":false / LEAN_CTX_PI_ENABLE_MCP=0', ); } function checkFaithfulLevers(cfg) { const rules = (effectiveEnv(cfg, "LEAN_CTX_RULES_INJECTION") || "").toLowerCase(); gate(rules === "off", "rules_injection", "off (no per-turn rule-file prefix)", `"${rules || "unset"}" — set LEAN_CTX_RULES_INJECTION=off`); const profile = (effectiveEnv(cfg, "LEAN_CTX_TOOL_PROFILE") || "").toLowerCase(); gate(profile === "minimal", "tool_profile", "minimal (6-tool core)", `"${profile || "unset"}" — set LEAN_CTX_TOOL_PROFILE=minimal`); const structureRaw = effectiveEnv(cfg, "LEAN_CTX_STRUCTURE_FIRST"); const structureOn = ["1", "true", "yes", "on"].includes((structureRaw || "").toLowerCase()); gate(structureOn, "structure_first", "on (capability-safe cold-read bias)", `"${structureRaw || "unset"}" — set LEAN_CTX_STRUCTURE_FIRST=1`); } function checkCompression(bin) { // Real proof that shell output is compressed (and therefore metered), without // depending on a footer string: run a log-like command raw vs through // lean-ctx and assert the lean-ctx output is strictly smaller. The generator is // a single `awk` BEGIN loop: it avoids shell command-substitution (so the probe // runs under shell_strict_mode) and `awk` is in the default shell_allowlist — // unlike `seq`, which mode=replace blocks, making the probe false-fail (#361). const cmd = 'awk \'BEGIN { for (i = 1; i <= 80; i++) printf "[INFO] building module %d of 80 ... ok\\n", i }\''; try { const raw = execFileSync("/bin/sh", ["-c", cmd], { encoding: "utf8" }); const compressed = execFileSync(bin, ["-c", cmd], { encoding: "utf8", env: { ...process.env, LEAN_CTX_COMPRESS: "1", LEAN_CTX_SAVINGS_FOOTER: "always" }, }); const pct = raw.length > 0 ? Math.round((1 - compressed.length / raw.length) * 100) : 0; gate( compressed.length < raw.length, "shell compression", `${raw.length} → ${compressed.length} bytes (-${pct}%) via ctx_shell path`, `lean-ctx output (${compressed.length}B) not smaller than raw (${raw.length}B)`, ); } catch (err) { fail("shell compression", `probe failed (${err.message})`); } } function checkProxy(bin) { // Soft: the proxy is usually started just-in-time for the run. Report status // when reachable so a stale/missing proxy is visible, but never gate on it. try { const out = execFileSync(bin, ["proxy", "status"], { encoding: "utf8" }).trim(); const running = /running|listening|:\d+/i.test(out) && !/not running|stopped/i.test(out); (running ? pass : warn)("proxy", out.split("\n")[0] || "status reported"); } catch { warn("proxy", "not running — start with `lean-ctx proxy start --port=4444` before the run"); } } // ── main ────────────────────────────────────────────────────────────────── const configPath = resolveConfigPath(); const cfg = readConfig(configPath); const mode = resolveMode(cfg); const routeShell = resolveRouteShell(mode, cfg); const bin = resolveBinary(); console.log("R2 faithful-arm preflight"); console.log(` config: ${configPath}${existsSync(configPath) ? "" : " (absent — using env + defaults)"}`); console.log(""); checkBinary(bin); checkShellRouting(mode, routeShell); checkBridge(resolveEnableMcp(cfg)); checkFaithfulLevers(cfg); checkCompression(bin); checkProxy(bin); console.log(""); if (failed) { console.log("preflight FAILED — fix the gates above before any priced run."); process.exit(1); } console.log("preflight PASSED — installed = running as designed.");