Tighten CTO live promotion opt-in audit

This commit is contained in:
Svrnty
2026-05-25 13:41:12 -04:00
parent 2beb72064b
commit 0ebd2f69ea
4 changed files with 47 additions and 33 deletions
+16 -4
View File
@@ -25,6 +25,13 @@ FIXTURES = CTO_ROOT / "evals" / "fixtures" / "manifest.yaml"
REQUIRED_LIVE_ACK = "i-understand-this-may-spend-tokens-and-edit-temp-workspaces"
def _artifact_path(path: Path) -> str:
try:
return str(path.relative_to(REPO_ROOT))
except ValueError:
return str(path)
def _run(cmd: list[str], *, cwd: Path, timeout: int = 60) -> dict[str, Any]:
started = time.time()
try:
@@ -79,9 +86,12 @@ def build_report(output: Path) -> dict[str, Any]:
skills = _run(["hermes", "-p", "cto-planb", "skills", "list"], cwd=REPO_ROOT) if hermes_available else None
mcp = _run(["hermes", "-p", "cto-planb", "mcp", "list"], cwd=REPO_ROOT) if hermes_available else None
live_requested = os.environ.get("HERMES_CTO_LIVE_PROMOTION") == "1"
live_ack = os.environ.get("HERMES_CTO_LIVE_PROMOTION_ACK") == REQUIRED_LIVE_ACK
live_requested_raw = os.environ.get("HERMES_CTO_LIVE_PROMOTION", "")
live_ack_raw = os.environ.get("HERMES_CTO_LIVE_PROMOTION_ACK", "")
live_requested = live_requested_raw == "1"
live_ack = live_ack_raw == REQUIRED_LIVE_ACK
live_execution_allowed = live_requested and live_ack
opt_in_state_valid = (not live_requested_raw and not live_ack_raw) or live_execution_allowed
eval_results = [
_result(
@@ -110,13 +120,15 @@ def build_report(output: Path) -> dict[str, Any]:
),
_result(
"live-execution-opt-in-policy",
True,
opt_in_state_valid,
[
"Live paid/mutating promotion execution is disabled unless HERMES_CTO_LIVE_PROMOTION=1",
"HERMES_CTO_LIVE_PROMOTION_ACK must match the required acknowledgement string",
],
live_requested=live_requested,
live_acknowledged=live_ack,
live_execution_allowed=live_execution_allowed,
opt_in_state_valid=opt_in_state_valid,
),
]
all_passed = all(item["status"] == "pass" for item in eval_results)
@@ -149,7 +161,7 @@ def build_report(output: Path) -> dict[str, Any]:
"artifacts": {
"transcript": "sot/08-OUTPUTS/CTO-WEBUI-CODER-PRD-EVIDENCE-2026-05-25.md",
"diff": "local-worktree",
"logs": str(output.relative_to(REPO_ROOT)),
"logs": _artifact_path(output),
"screenshots": [],
},
"eval_results": eval_results,