Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ The portable CLI now includes the local intake layer that service uses first:

```bash
npx proofloop target --url https://your-app.example --write-runner-plan
npx proofloop target --url https://your-app.example --write-browser-smoke --write-runner-plan
npx proofloop target --dir . --write-runner-plan
```

Expand All @@ -32,6 +33,11 @@ detects any already-configured benchmark/browser scripts, writes
`.proofloop/runner/target.plan.json`. It does not invent official scores; missing adapters and
official scorer paths are recorded as blockers.

When `--write-browser-smoke` is provided with `--url` in a repo with `package.json`, Proof Loop
writes `proofloop/browser/live-smoke.spec.ts` and a `proofloop:live-smoke` package script. That turns
basic live URL rendering/clickability into a runnable Playwright task while keeping deeper app flows
and official benchmark scorers explicit.

## Quickstart

```bash
Expand Down Expand Up @@ -147,7 +153,7 @@ script. With neither, it reports `no_gate` with exit code 2. An unconfigured gat
| `proofloop init --agent auto --live` | Add agent docs, manifest, package aliases, workflows, and rubrics. |
| `proofloop doctor [--json]` | Report node/git/agent readiness, manifest/docs/scripts, Playwright/browser readiness, GitHub workflow, UI contracts, and fix commands. |
| `proofloop manifest [--json\|--dense]` | Print project status: stack, commands, proof gates, workflows, UI contracts, blockers. |
| `proofloop target [--url <url>] [--write-runner-plan] [--json]` | Recommend benchmark families from a URL/codebase, detect configured adapters, and write target/runner plan receipts. |
| `proofloop target [--url <url>] [--write-runner-plan] [--write-browser-smoke] [--json]` | Recommend benchmark families from a URL/codebase, detect or scaffold configured adapters, and write target/runner plan receipts. |
| `proofloop docs agents --dense` | Print compact agent workflow instructions. |
| `proofloop ui contract\|component <id>` | Discover stable `data-testid` and `data-proofloop` selectors. |
| `proofloop template --list` / `proofloop template <id> --write` | List or write starter proof-loop templates. |
Expand Down
5 changes: 3 additions & 2 deletions dist/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ exports.runCli = runCli;
* proofloop tooluse <verify|init> expected-tool-use contracts
* proofloop ci install github write the GitHub Actions gate workflow
* proofloop prompt print the one-prompt kickoff
* proofloop target [--url <url>] [--write-runner-plan]
* proofloop target [--url <url>] [--write-runner-plan] [--write-browser-smoke]
* proofloop this-repo [--goal ...] [--write-runner-plan] [--run]
* proofloop manifest|docs|template|workflow|ui|resume|report|charts|receipt|mcp
*
Expand Down Expand Up @@ -96,7 +96,7 @@ function usage() {
" charts latest write local JSON/SVG proof charts",
" receipt verify --file <path> verify app-produced proof receipts",
" runner run|resume|status|report durable append-only task runner with budget and resume",
" target [--url <url>] [--write-runner-plan] recommend benchmark families and write target receipt",
" target [--url <url>] [--write-runner-plan] [--write-browser-smoke] recommend benchmark families and write target receipt",
" mcp start the optional read-only MCP server",
" prompt print the one-prompt kickoff",
" this-repo [--goal <text>] [--write-runner-plan] [--run]",
Expand Down Expand Up @@ -342,6 +342,7 @@ async function runTargetCommand(options, root) {
...(str(options.url) !== undefined ? { url: str(options.url) } : {}),
...(str(options.out) !== undefined ? { outPath: str(options.out) } : {}),
writeRunnerPlan: options["write-runner-plan"] === true || options.runner === true,
writeBrowserSmoke: options["write-browser-smoke"] === true,
json: options.json === true,
dense: options.dense === true,
...(num(options["timeout-ms"]) !== undefined ? { timeoutMs: num(options["timeout-ms"]) } : {}),
Expand Down
3 changes: 3 additions & 0 deletions dist/targetPlan.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ export type ProofloopTargetPlan = {
};
recommendations: ProofloopBenchmarkRecommendation[];
runnerPlan?: ProofloopRunnerPlan;
generatedFiles: string[];
blocked: string[];
nextActions: string[];
honesty: string;
Expand All @@ -52,6 +53,7 @@ export type ProofloopTargetOptions = {
url?: string;
outPath?: string;
writeRunnerPlan?: boolean;
writeBrowserSmoke?: boolean;
json?: boolean;
dense?: boolean;
timeoutMs?: number;
Expand Down Expand Up @@ -79,6 +81,7 @@ export declare function buildProofloopTargetPlan(args: {
root: string;
codebaseSignals?: TargetSignals;
urlSignals?: UrlSignals;
generatedFiles?: string[];
generatedAt?: string;
}): ProofloopTargetPlan;
export declare function classifyBenchmarkFamilies(textInput: string, scripts?: Record<string, string>, hasLiveUrl?: boolean, seedEvidence?: string[]): ProofloopBenchmarkRecommendation[];
Expand Down
42 changes: 41 additions & 1 deletion dist/targetPlan.js
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ async function runProofloopTarget(options) {
}
async function writeProofloopTargetPlan(options) {
const root = (0, node_path_1.resolve)(options.root);
const generatedFiles = options.writeBrowserSmoke && options.url ? writeBrowserSmokeScaffold(root, normalizeUrl(options.url)) : [];
const codebaseSignals = (0, node_fs_1.existsSync)((0, node_path_1.join)(root, "package.json")) ? readCodebaseSignals(root) : undefined;
const urlSignals = options.url ? await readUrlSignals(options.url, options.timeoutMs ?? DEFAULT_TIMEOUT_MS) : undefined;
if (!codebaseSignals && !urlSignals)
Expand All @@ -137,6 +138,7 @@ async function writeProofloopTargetPlan(options) {
root,
codebaseSignals,
urlSignals,
generatedFiles,
});
const planPath = (0, node_path_1.resolve)(root, options.outPath ?? DEFAULT_TARGET_PLAN_PATH);
writeJson(planPath, plan);
Expand Down Expand Up @@ -185,6 +187,7 @@ function buildProofloopTargetPlan(args) {
},
recommendations,
...(runnerPlan.tasks.length > 0 ? { runnerPlan } : {}),
generatedFiles: args.generatedFiles ?? [],
blocked,
nextActions: buildNextActions(recommendations, args.urlSignals, runnerPlan.tasks.length > 0),
honesty: "This is benchmark-family targeting and runnable-plan discovery, not an official benchmark score. Official claims require the configured upstream scorer or an explicitly recorded equivalent judge contract.",
Expand Down Expand Up @@ -251,6 +254,8 @@ function formatProofloopTargetPlanDense(plan, planPath, runnerPlanPath) {
for (const evidence of rec.evidence.slice(0, 3))
lines.push(` evidence=${evidence}`);
}
for (const file of plan.generatedFiles.slice(0, 8))
lines.push(`generated=${file}`);
for (const blocked of plan.blocked.slice(0, 8))
lines.push(`blocked=${blocked}`);
for (const action of plan.nextActions.slice(0, 6))
Expand Down Expand Up @@ -327,9 +332,12 @@ function buildTargetRunnerPlan(root, recommendations, scripts, url) {
}
for (const recommendation of recommendations) {
for (const script of recommendation.configuredScripts) {
const command = `npm run ${quoteNpmScriptName(script.name)}`;
if (tasks.some((task) => task.command === command))
continue;
addTask(tasks, {
id: `benchmark.${toTaskId(recommendation.id)}.${toTaskId(script.name)}`,
command: `npm run ${quoteNpmScriptName(script.name)}`,
command,
env: {
PROOFLOOP_BENCHMARK_FAMILY: recommendation.id,
PROOFLOOP_TARGET_OFFICIAL_SCORE_STATUS: recommendation.officialScoreStatus,
Expand Down Expand Up @@ -503,6 +511,37 @@ function uniqueEvidence(values) {
function hasBrowserScript(scripts) {
return Object.entries(scripts).some(([name, command]) => /\b(browser|playwright|cypress|puppeteer|selenium|webdriver|e2e)\b/i.test(`${name} ${command}`));
}
function writeBrowserSmokeScaffold(root, url) {
const packagePath = (0, node_path_1.join)(root, "package.json");
if (!(0, node_fs_1.existsSync)(packagePath))
return [];
const specPath = (0, node_path_1.join)(root, "proofloop", "browser", "live-smoke.spec.ts");
const pkg = readPackageJson(root);
const scripts = pkg.scripts && typeof pkg.scripts === "object" && !Array.isArray(pkg.scripts) ? { ...pkg.scripts } : {};
scripts["proofloop:live-smoke"] = "playwright test proofloop/browser/live-smoke.spec.ts";
pkg.scripts = scripts;
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(specPath), { recursive: true });
(0, node_fs_1.writeFileSync)(specPath, browserSmokeSpec(url), "utf8");
(0, node_fs_1.writeFileSync)(packagePath, `${JSON.stringify(pkg, null, 2)}\n`, "utf8");
return [specPath, packagePath];
}
function browserSmokeSpec(url) {
return [
"import { expect, test } from '@playwright/test';",
"",
`const targetUrl = process.env.PROOFLOOP_TARGET_URL ?? ${JSON.stringify(url)};`,
"",
"test('ProofLoop live URL smoke', async ({ page }) => {",
" const response = await page.goto(targetUrl, { waitUntil: 'domcontentloaded' });",
" expect(response?.ok(), `expected ${targetUrl} to return a 2xx/3xx response`).toBeTruthy();",
" await expect(page.locator('body')).toBeVisible();",
" await expect(page.locator('body')).not.toHaveText('');",
" const interactive = page.locator('a[href], button, input, textarea, select, [role=\"button\"], [data-testid], [data-proofloop]');",
" expect(await interactive.count(), 'expected at least one interactive or proof-selectable element').toBeGreaterThan(0);",
"});",
"",
].join("\n");
}
function addTask(tasks, task) {
const existing = new Set(tasks.map((entry) => entry.id));
if (!existing.has(task.id)) {
Expand Down Expand Up @@ -541,6 +580,7 @@ function emptyTargetPlan(root, url) {
runnerPlanReady: false,
},
recommendations: [],
generatedFiles: [],
blocked: ["target planning failed before a receipt could be produced"],
nextActions: ["Fix the CLI input or local repo setup, then rerun `npx proofloop target`."],
honesty: "No benchmark proof was produced.",
Expand Down
5 changes: 3 additions & 2 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
* proofloop tooluse <verify|init> expected-tool-use contracts
* proofloop ci install github write the GitHub Actions gate workflow
* proofloop prompt print the one-prompt kickoff
* proofloop target [--url <url>] [--write-runner-plan]
* proofloop target [--url <url>] [--write-runner-plan] [--write-browser-smoke]
* proofloop this-repo [--goal ...] [--write-runner-plan] [--run]
* proofloop manifest|docs|template|workflow|ui|resume|report|charts|receipt|mcp
*
Expand Down Expand Up @@ -112,7 +112,7 @@ function usage(): string {
" charts latest write local JSON/SVG proof charts",
" receipt verify --file <path> verify app-produced proof receipts",
" runner run|resume|status|report durable append-only task runner with budget and resume",
" target [--url <url>] [--write-runner-plan] recommend benchmark families and write target receipt",
" target [--url <url>] [--write-runner-plan] [--write-browser-smoke] recommend benchmark families and write target receipt",
" mcp start the optional read-only MCP server",
" prompt print the one-prompt kickoff",
" this-repo [--goal <text>] [--write-runner-plan] [--run]",
Expand Down Expand Up @@ -392,6 +392,7 @@ async function runTargetCommand(options: Record<string, string | boolean>, root:
...(str(options.url) !== undefined ? { url: str(options.url)! } : {}),
...(str(options.out) !== undefined ? { outPath: str(options.out)! } : {}),
writeRunnerPlan: options["write-runner-plan"] === true || options.runner === true,
writeBrowserSmoke: options["write-browser-smoke"] === true,
json: options.json === true,
dense: options.dense === true,
...(num(options["timeout-ms"]) !== undefined ? { timeoutMs: num(options["timeout-ms"])! } : {}),
Expand Down
44 changes: 43 additions & 1 deletion src/targetPlan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ export type ProofloopTargetPlan = {
};
recommendations: ProofloopBenchmarkRecommendation[];
runnerPlan?: ProofloopRunnerPlan;
generatedFiles: string[];
blocked: string[];
nextActions: string[];
honesty: string;
Expand All @@ -58,6 +59,7 @@ export type ProofloopTargetOptions = {
url?: string;
outPath?: string;
writeRunnerPlan?: boolean;
writeBrowserSmoke?: boolean;
json?: boolean;
dense?: boolean;
timeoutMs?: number;
Expand Down Expand Up @@ -223,6 +225,7 @@ export async function runProofloopTarget(options: ProofloopTargetOptions): Promi

export async function writeProofloopTargetPlan(options: ProofloopTargetOptions): Promise<ProofloopTargetResult> {
const root = resolve(options.root);
const generatedFiles = options.writeBrowserSmoke && options.url ? writeBrowserSmokeScaffold(root, normalizeUrl(options.url)) : [];
const codebaseSignals = existsSync(join(root, "package.json")) ? readCodebaseSignals(root) : undefined;
const urlSignals = options.url ? await readUrlSignals(options.url, options.timeoutMs ?? DEFAULT_TIMEOUT_MS) : undefined;
if (!codebaseSignals && !urlSignals) throw new Error("expected --url <url> or a repo with package.json");
Expand All @@ -231,6 +234,7 @@ export async function writeProofloopTargetPlan(options: ProofloopTargetOptions):
root,
codebaseSignals,
urlSignals,
generatedFiles,
});
const planPath = resolve(root, options.outPath ?? DEFAULT_TARGET_PLAN_PATH);
writeJson(planPath, plan);
Expand All @@ -248,6 +252,7 @@ export function buildProofloopTargetPlan(args: {
root: string;
codebaseSignals?: TargetSignals;
urlSignals?: UrlSignals;
generatedFiles?: string[];
generatedAt?: string;
}): ProofloopTargetPlan {
const root = resolve(args.root);
Expand Down Expand Up @@ -288,6 +293,7 @@ export function buildProofloopTargetPlan(args: {
},
recommendations,
...(runnerPlan.tasks.length > 0 ? { runnerPlan } : {}),
generatedFiles: args.generatedFiles ?? [],
blocked,
nextActions: buildNextActions(recommendations, args.urlSignals, runnerPlan.tasks.length > 0),
honesty: "This is benchmark-family targeting and runnable-plan discovery, not an official benchmark score. Official claims require the configured upstream scorer or an explicitly recorded equivalent judge contract.",
Expand Down Expand Up @@ -362,6 +368,7 @@ export function formatProofloopTargetPlanDense(plan: ProofloopTargetPlan, planPa
lines.push(`family=${rec.id} fit=${rec.fit} confidence=${rec.confidence.toFixed(2)} adapter=${rec.adapterStatus} scorer=${rec.officialScoreStatus}`);
for (const evidence of rec.evidence.slice(0, 3)) lines.push(` evidence=${evidence}`);
}
for (const file of plan.generatedFiles.slice(0, 8)) lines.push(`generated=${file}`);
for (const blocked of plan.blocked.slice(0, 8)) lines.push(`blocked=${blocked}`);
for (const action of plan.nextActions.slice(0, 6)) lines.push(`next=${action}`);
return `${lines.join("\n")}\n`;
Expand Down Expand Up @@ -443,9 +450,11 @@ function buildTargetRunnerPlan(
}
for (const recommendation of recommendations) {
for (const script of recommendation.configuredScripts) {
const command = `npm run ${quoteNpmScriptName(script.name)}`;
if (tasks.some((task) => task.command === command)) continue;
addTask(tasks, {
id: `benchmark.${toTaskId(recommendation.id)}.${toTaskId(script.name)}`,
command: `npm run ${quoteNpmScriptName(script.name)}`,
command,
env: {
PROOFLOOP_BENCHMARK_FAMILY: recommendation.id,
PROOFLOOP_TARGET_OFFICIAL_SCORE_STATUS: recommendation.officialScoreStatus,
Expand Down Expand Up @@ -631,6 +640,38 @@ function hasBrowserScript(scripts: Record<string, string>): boolean {
return Object.entries(scripts).some(([name, command]) => /\b(browser|playwright|cypress|puppeteer|selenium|webdriver|e2e)\b/i.test(`${name} ${command}`));
}

function writeBrowserSmokeScaffold(root: string, url: string): string[] {
const packagePath = join(root, "package.json");
if (!existsSync(packagePath)) return [];
const specPath = join(root, "proofloop", "browser", "live-smoke.spec.ts");
const pkg = readPackageJson(root) as PackageJson & { scripts?: Record<string, string> };
const scripts = pkg.scripts && typeof pkg.scripts === "object" && !Array.isArray(pkg.scripts) ? { ...pkg.scripts } : {};
scripts["proofloop:live-smoke"] = "playwright test proofloop/browser/live-smoke.spec.ts";
pkg.scripts = scripts;
mkdirSync(dirname(specPath), { recursive: true });
writeFileSync(specPath, browserSmokeSpec(url), "utf8");
writeFileSync(packagePath, `${JSON.stringify(pkg, null, 2)}\n`, "utf8");
return [specPath, packagePath];
}

function browserSmokeSpec(url: string): string {
return [
"import { expect, test } from '@playwright/test';",
"",
`const targetUrl = process.env.PROOFLOOP_TARGET_URL ?? ${JSON.stringify(url)};`,
"",
"test('ProofLoop live URL smoke', async ({ page }) => {",
" const response = await page.goto(targetUrl, { waitUntil: 'domcontentloaded' });",
" expect(response?.ok(), `expected ${targetUrl} to return a 2xx/3xx response`).toBeTruthy();",
" await expect(page.locator('body')).toBeVisible();",
" await expect(page.locator('body')).not.toHaveText('');",
" const interactive = page.locator('a[href], button, input, textarea, select, [role=\"button\"], [data-testid], [data-proofloop]');",
" expect(await interactive.count(), 'expected at least one interactive or proof-selectable element').toBeGreaterThan(0);",
"});",
"",
].join("\n");
}

function addTask(tasks: ProofloopRunnerTaskPlan[], task: ProofloopRunnerTaskPlan): void {
const existing = new Set(tasks.map((entry) => entry.id));
if (!existing.has(task.id)) {
Expand Down Expand Up @@ -673,6 +714,7 @@ function emptyTargetPlan(root: string, url: string | undefined): ProofloopTarget
runnerPlanReady: false,
},
recommendations: [],
generatedFiles: [],
blocked: ["target planning failed before a receipt could be produced"],
nextActions: ["Fix the CLI input or local repo setup, then rerun `npx proofloop target`."],
honesty: "No benchmark proof was produced.",
Expand Down
39 changes: 39 additions & 0 deletions tests/targetPlan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,45 @@ describe("proofloop target planner", () => {
expect(result.plan.blocked.join("\n")).toContain("no Playwright/Cypress/browser script");
expect(logs.join("\n")).toContain('"schema": "proofloop-target-plan-v1"');
});

it("can scaffold a Playwright live-smoke adapter for a URL target", async () => {
const root = tempRoot();
writeFileSync(
join(root, "package.json"),
JSON.stringify(
{
name: "live-target-app",
scripts: { test: "node -e 0" },
devDependencies: { "@playwright/test": "1.48.0" },
},
null,
2,
),
"utf8",
);
const url = await startServer("<html><body><a href='/next'>Open</a><button>Run</button><main>Workflow room</main></body></html>");

const result = await runProofloopTarget({
root,
url,
writeBrowserSmoke: true,
writeRunnerPlan: true,
log: () => {},
logError: () => {},
});

const specPath = join(root, "proofloop", "browser", "live-smoke.spec.ts");
const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { scripts: Record<string, string> };
const browser = result.plan.recommendations.find((entry) => entry.id === "live-browser-smoke");

expect(existsSync(specPath)).toBe(true);
expect(readFileSync(specPath, "utf8")).toContain(url);
expect(pkg.scripts["proofloop:live-smoke"]).toBe("playwright test proofloop/browser/live-smoke.spec.ts");
expect(browser?.adapterStatus).toBe("configured");
expect(result.plan.generatedFiles.map((file) => file.replace(/\\/g, "/")).join("\n")).toContain("proofloop/browser/live-smoke.spec.ts");
expect(result.plan.blocked.join("\n")).not.toContain("no Playwright/Cypress/browser script");
expect(result.plan.runnerPlan?.tasks.some((task) => task.command === "npm run proofloop:live-smoke")).toBe(true);
});
});

function startServer(html: string): Promise<string> {
Expand Down
Loading