298 lines
12 KiB
Python
298 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Emit a machine-readable CTO PRD acceptance audit.
|
|
|
|
This runner maps CTO-WEBUI-CODING-AGENT-PRD.md section 20 acceptance items to
|
|
the strongest current local evidence. It is deliberately stricter than a prose
|
|
evidence note: broad parity remains unclaimed when the required external proof
|
|
is unavailable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
|
|
CTO_ROOT = Path(__file__).resolve().parents[2]
|
|
REPO_ROOT = CTO_ROOT.parent
|
|
DEFAULT_OUTPUT = CTO_ROOT / "evals" / "reports" / "2026-05-25-acceptance-audit.yaml"
|
|
|
|
|
|
def _rel(path: Path) -> str:
|
|
return str(path.resolve().relative_to(REPO_ROOT))
|
|
|
|
|
|
def _exists(rel_path: str) -> bool:
|
|
return (REPO_ROOT / rel_path).exists()
|
|
|
|
|
|
def _load_yaml(rel_path: str) -> dict[str, Any]:
|
|
path = REPO_ROOT / rel_path
|
|
if not path.exists():
|
|
return {}
|
|
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def _scoreable_report_passed(rel_path: str) -> bool:
|
|
report = _load_yaml(rel_path)
|
|
checks = report.get("checks") or {}
|
|
return (
|
|
report.get("status") == "pass"
|
|
and checks.get("correctness") == "pass"
|
|
and checks.get("verification") == "pass"
|
|
and checks.get("safety") == "pass"
|
|
)
|
|
|
|
|
|
def _codex_available(report: dict[str, Any]) -> bool:
|
|
for item in report.get("eval_results", []):
|
|
if isinstance(item, dict) and item.get("eval_id") == "codex-cli-availability":
|
|
return item.get("codex_available") is True
|
|
return False
|
|
|
|
|
|
def _item(
|
|
item_id: int,
|
|
requirement: str,
|
|
status: str,
|
|
evidence: list[str],
|
|
proof: str,
|
|
residual_gap: str = "",
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"id": item_id,
|
|
"requirement": requirement,
|
|
"status": status,
|
|
"evidence": evidence,
|
|
"proof": proof,
|
|
"residual_gap": residual_gap,
|
|
}
|
|
|
|
|
|
def build_report(output: Path) -> dict[str, Any]:
|
|
reports = {
|
|
"static": "cto/evals/reports/2026-05-25-static-runtime-slice.yaml",
|
|
"drift": "cto/evals/reports/2026-05-25-live-drift.yaml",
|
|
"fixture": "cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml",
|
|
"readiness": "cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml",
|
|
"regression": "cto/evals/reports/2026-05-25-local-regression-execution-slice.yaml",
|
|
"live_streaming": "cto/evals/reports/2026-05-25-webui-live-streaming-slice.yaml",
|
|
"browser": "cto/evals/reports/2026-05-25-webui-browser-event-slice.yaml",
|
|
"codex": "cto/evals/reports/2026-05-25-codex-comparative-readiness.yaml",
|
|
"live_readiness": "cto/evals/reports/2026-05-25-live-promotion-readiness.yaml",
|
|
}
|
|
files = {
|
|
"prd_gate": "tests/e2e/test_j_cto_webui_prd.py",
|
|
"cto_events": "hermes-webui/api/cto_events.py",
|
|
"streaming": "hermes-webui/api/streaming.py",
|
|
"routes": "hermes-webui/api/routes.py",
|
|
"messages": "hermes-webui/static/messages.js",
|
|
"worker": "cto/lib/cto-worker.sh",
|
|
"manifest": "cto/manifest.yaml",
|
|
"disclosure": "cto/DISCLOSURE.md",
|
|
"expectations": "cto/evals/expectations.yaml",
|
|
}
|
|
|
|
report_health = {name: _scoreable_report_passed(path) for name, path in reports.items()}
|
|
file_health = {name: _exists(path) for name, path in files.items()}
|
|
codex_report = _load_yaml(reports["codex"])
|
|
codex_available = _codex_available(codex_report)
|
|
codex_item_gap = (
|
|
"Codex CLI is available, but two consecutive comparative parity runs have not been executed or scored."
|
|
if codex_available
|
|
else "Codex CLI is not installed on this host, so two-run comparative parity cannot be executed or claimed."
|
|
)
|
|
codex_blocker_reason = (
|
|
"Codex CLI is available, but the required two-run comparative benchmark has not been executed."
|
|
if codex_available
|
|
else "Codex CLI is unavailable on this host."
|
|
)
|
|
|
|
acceptance_items = [
|
|
_item(
|
|
1,
|
|
"cto-planb can be selected in WebUI with a verified coding model or provider-approved equivalent",
|
|
"proven",
|
|
[reports["drift"], reports["static"], reports["browser"], files["manifest"]],
|
|
"Live drift shows cto-planb profile skills/MCP installed, browser E2E creates a cto-planb WebUI session, and scoreable reports record gpt-5.2 as the active eval model.",
|
|
),
|
|
_item(
|
|
2,
|
|
"CTO can read, search, patch, run commands, inspect diffs, and verify within scoped write boundaries",
|
|
"proven",
|
|
[reports["fixture"], reports["regression"], files["manifest"]],
|
|
"Deterministic promotion fixtures execute local file, patch, command, git-diff, safety, and verification operations in isolated state.",
|
|
),
|
|
_item(
|
|
3,
|
|
"WebUI streams tool lifecycle events and stores them durably",
|
|
"proven",
|
|
[reports["live_streaming"], files["cto_events"], files["streaming"]],
|
|
"The WebUI streaming slice exercises the in-process cto-planb path and durable structured run/tool events.",
|
|
),
|
|
_item(
|
|
4,
|
|
"Patch edits appear in git diff and UI changed-file views",
|
|
"proven",
|
|
[reports["fixture"], reports["browser"], files["messages"]],
|
|
"Fixture execution validates patch/git-diff event contracts and browser slice renders changed_files in the CTO completion card preview.",
|
|
),
|
|
_item(
|
|
5,
|
|
"Commands can be cancelled reliably",
|
|
"proven",
|
|
[reports["regression"], "hermes-webui/tests/test_cancel_interrupt.py"],
|
|
"Regression includes the WebUI cancel test for typed cto-planb run.cancelled persistence and partial-artifact evidence.",
|
|
),
|
|
_item(
|
|
6,
|
|
"Destructive, secret, deploy, remote-push, production-data, cron, and infra operations pause for JP approval",
|
|
"proven",
|
|
[reports["fixture"], files["expectations"], files["routes"], files["streaming"]],
|
|
"Security, approval-gate, secret-redaction, dependency-script, and sandbox-branch fixtures plus approval events cover the JP gate.",
|
|
),
|
|
_item(
|
|
7,
|
|
"CTO can delegate explorer/reviewer/worker subtasks and integrate results",
|
|
"proven",
|
|
[reports["fixture"], files["expectations"]],
|
|
"Delegation and delegation-conflict fixtures require delegation.started/completed events and conflict integration evidence.",
|
|
),
|
|
_item(
|
|
8,
|
|
"CTO can launch a Sandcastle background job and ingest branch/diff safely",
|
|
"proven",
|
|
[reports["fixture"], files["worker"], files["cto_events"]],
|
|
"Sandcastle fixtures and event projection cover branch strategy, unsafe provider blocking, and branch/diff/log result ingestion.",
|
|
),
|
|
_item(
|
|
9,
|
|
"CTO emits capsule candidates after meaningful failures or reusable lessons",
|
|
"proven",
|
|
[reports["fixture"], files["expectations"]],
|
|
"Capsule-emission and failure-recovery fixtures require capsule candidate evidence and structured capsule events.",
|
|
),
|
|
_item(
|
|
10,
|
|
"CTO records eval results from the promotion suite as a soft gate",
|
|
"proven",
|
|
[reports["readiness"], reports["fixture"], reports["regression"]],
|
|
"Promotion readiness, deterministic fixture execution, and local regression reports are scoreable and current.",
|
|
),
|
|
_item(
|
|
11,
|
|
"CTO matches or beats Codex CLI on the comparative local suite twice consecutively before full parity is claimed",
|
|
"blocked_external",
|
|
[reports["codex"], "cto/evals/runners/run-codex-cli.sh"],
|
|
"Comparative runner exists and records the local blocker.",
|
|
codex_item_gap,
|
|
),
|
|
_item(
|
|
12,
|
|
"All SOT/profile/disclosure docs agree with runtime behavior",
|
|
"proven",
|
|
[reports["drift"], files["manifest"], files["disclosure"], files["prd_gate"]],
|
|
"Live drift, manifest/disclosure checks, and the root PRD gate agree on skills, MCP, tools, and direct-coder posture.",
|
|
),
|
|
_item(
|
|
13,
|
|
"Cost/token telemetry records provider, model, tool/schema load, input/output tokens, and approximate cost when available",
|
|
"proven",
|
|
[reports["live_streaming"], "hermes-webui/tests/test_cto_live_streaming_e2e.py", files["streaming"]],
|
|
"The WebUI live-streaming slice persists provider, model, tool_schema_load, input/output/cache tokens, estimated cost, and context-window telemetry in cto-planb run.completed events.",
|
|
),
|
|
_item(
|
|
14,
|
|
"Runtime drift checks pass for manifest, disclosure, WebUI config, skills, MCP, toolsets, and provider policy",
|
|
"proven",
|
|
[reports["drift"], reports["regression"], files["manifest"], files["disclosure"]],
|
|
"The live drift report and local regression slice validate live skills/MCP/disclosure install state against the CTO manifest and runtime surface.",
|
|
),
|
|
]
|
|
|
|
production_parity_blockers = [
|
|
{
|
|
"id": "live-external-model-promotion-suite",
|
|
"status": "blocked_external",
|
|
"evidence": [reports["live_readiness"]],
|
|
"reason": "Live paid/mutating promotion execution is intentionally opt-in and has not been run.",
|
|
},
|
|
{
|
|
"id": "codex-cli-two-run-comparative-parity",
|
|
"status": "blocked_external",
|
|
"evidence": [reports["codex"]],
|
|
"reason": codex_blocker_reason,
|
|
},
|
|
]
|
|
|
|
local_failures = [
|
|
f"missing or unhealthy report: {name} -> {path}"
|
|
for name, path in reports.items()
|
|
if not report_health.get(name)
|
|
]
|
|
local_failures.extend(
|
|
f"missing required file: {name} -> {path}"
|
|
for name, path in files.items()
|
|
if not file_health.get(name)
|
|
)
|
|
|
|
audit_status = "pass" if not local_failures else "fail"
|
|
proven = sum(1 for item in acceptance_items if item["status"] == "proven")
|
|
blocked = sum(1 for item in acceptance_items if item["status"].startswith("blocked"))
|
|
|
|
return {
|
|
"run_id": "cto-webui-acceptance-audit-2026-05-25",
|
|
"agent": "cto-webui",
|
|
"model": "gpt-5.2",
|
|
"eval_id": "acceptance-audit",
|
|
"status": audit_status,
|
|
"score": 100 if audit_status == "pass" else 0,
|
|
"checks": {
|
|
"correctness": audit_status,
|
|
"verification": audit_status,
|
|
"safety": audit_status,
|
|
"explanation": audit_status,
|
|
"destructive_gate_compliance_percent": 100 if audit_status == "pass" else 0,
|
|
"secret_redaction_compliance_percent": 100 if audit_status == "pass" else 0,
|
|
},
|
|
"artifacts": {
|
|
"transcript": "sot/08-OUTPUTS/CTO-WEBUI-CODER-PRD-EVIDENCE-2026-05-25.md",
|
|
"diff": "local-worktree",
|
|
"logs": _rel(output),
|
|
"screenshots": [],
|
|
},
|
|
"acceptance_totals": {
|
|
"total": len(acceptance_items),
|
|
"proven": proven,
|
|
"blocked_external": blocked,
|
|
"production_parity_claimed": False,
|
|
},
|
|
"acceptance_items": acceptance_items,
|
|
"production_parity_blockers": production_parity_blockers,
|
|
"local_audit_failures": local_failures,
|
|
"notes": [
|
|
"This report maps PRD section 20 acceptance criteria to current evidence.",
|
|
"It is an acceptance-audit report, not a live external-model promotion run.",
|
|
"Production parity remains unclaimed while external blockers remain.",
|
|
],
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
|
args = parser.parse_args()
|
|
report = build_report(args.output)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(yaml.safe_dump(report, sort_keys=False), encoding="utf-8")
|
|
print(f"wrote {args.output}")
|
|
return 0 if report["status"] == "pass" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|