From 853bf9dcd617913af251b633733fcd0c55db495a Mon Sep 17 00:00:00 2001
From: justrach <54503978+justrach@users.noreply.github.com>
Date: Sat, 4 Jul 2026 19:02:40 +0900
Subject: [PATCH] =?UTF-8?q?feat(scripts):=20bench-ab.sh=20=E2=80=94=20one-?=
 =?UTF-8?q?command=20local=20A/B=20of=20the=20gated=20bench?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Wraps the recipe used for every perf PR this cycle: build the base ref
in a throwaway $HOME worktree (tests/bench misbehave under /tmp roots),
run `zig build bench -- --json` for base and working tree back-to-back
on the same machine, and print the same compare-bench.py table CI posts
on PRs. Defaults to base=HEAD so uncommitted perf work is one command
away from a trustworthy same-machine delta.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 scripts/bench-ab.sh | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100755 scripts/bench-ab.sh

diff --git a/scripts/bench-ab.sh b/scripts/bench-ab.sh
new file mode 100755
index 00000000..b6116cbe
--- /dev/null
+++ b/scripts/bench-ab.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# A/B the gated MCP tool bench: current working tree vs a base ref.
+#
+#   scripts/bench-ab.sh              # base = HEAD (measure uncommitted work)
+#   scripts/bench-ab.sh HEAD~3       # base = any ref
+#   scripts/bench-ab.sh origin/release/0.2.5828
+#
+# Builds the base ref in a throwaway worktree under $HOME (tests and bench
+# misbehave under /tmp roots — see root_policy), runs `zig build bench --
+# --json` for both sides on this machine back-to-back, and prints the same
+# regression table CI posts on PRs (scripts/compare-bench.py, 10% + 50µs
+# thresholds). Only same-machine, same-run deltas are meaningful — never
+# compare against a JSON from another day or box.
+set -euo pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+BASE_REF="${1:-HEAD}"
+BASE_SHA="$(git -C "$REPO_ROOT" rev-parse --short "$BASE_REF")"
+WT="$HOME/.cache/codedb-bench-ab-$$"
+OUT="$(mktemp -d "${TMPDIR:-/tmp}/codedb-bench-ab.XXXXXX")"
+
+cleanup() {
+  git -C "$REPO_ROOT" worktree remove --force "$WT" >/dev/null 2>&1 || true
+  rm -rf "$OUT"
+}
+trap cleanup EXIT
+
+echo "base:  $BASE_REF ($BASE_SHA) in throwaway worktree"
+echo "head:  working tree at $(git -C "$REPO_ROOT" rev-parse --short HEAD)$(git -C "$REPO_ROOT" diff --quiet || echo ' + uncommitted changes')"
+
+git -C "$REPO_ROOT" worktree add --detach "$WT" "$BASE_REF" >/dev/null
+(cd "$WT" && python3 scripts/run-bench-json.py "$OUT/base.json" >/dev/null)
+(cd "$REPO_ROOT" && python3 scripts/run-bench-json.py "$OUT/head.json" >/dev/null)
+
+python3 "$REPO_ROOT/scripts/compare-bench.py" "$OUT/base.json" "$OUT/head.json" \
+  --threshold-pct 10 --min-abs-ns 50000