Tighten CTO live promotion opt-in audit
This commit is contained in:
@@ -25,6 +25,13 @@ FIXTURES = CTO_ROOT / "evals" / "fixtures" / "manifest.yaml"
|
||||
REQUIRED_LIVE_ACK = "i-understand-this-may-spend-tokens-and-edit-temp-workspaces"
|
||||
|
||||
|
||||
def _artifact_path(path: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(REPO_ROOT))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def _run(cmd: list[str], *, cwd: Path, timeout: int = 60) -> dict[str, Any]:
|
||||
started = time.time()
|
||||
try:
|
||||
@@ -79,9 +86,12 @@ def build_report(output: Path) -> dict[str, Any]:
|
||||
skills = _run(["hermes", "-p", "cto-planb", "skills", "list"], cwd=REPO_ROOT) if hermes_available else None
|
||||
mcp = _run(["hermes", "-p", "cto-planb", "mcp", "list"], cwd=REPO_ROOT) if hermes_available else None
|
||||
|
||||
live_requested = os.environ.get("HERMES_CTO_LIVE_PROMOTION") == "1"
|
||||
live_ack = os.environ.get("HERMES_CTO_LIVE_PROMOTION_ACK") == REQUIRED_LIVE_ACK
|
||||
live_requested_raw = os.environ.get("HERMES_CTO_LIVE_PROMOTION", "")
|
||||
live_ack_raw = os.environ.get("HERMES_CTO_LIVE_PROMOTION_ACK", "")
|
||||
live_requested = live_requested_raw == "1"
|
||||
live_ack = live_ack_raw == REQUIRED_LIVE_ACK
|
||||
live_execution_allowed = live_requested and live_ack
|
||||
opt_in_state_valid = (not live_requested_raw and not live_ack_raw) or live_execution_allowed
|
||||
|
||||
eval_results = [
|
||||
_result(
|
||||
@@ -110,13 +120,15 @@ def build_report(output: Path) -> dict[str, Any]:
|
||||
),
|
||||
_result(
|
||||
"live-execution-opt-in-policy",
|
||||
True,
|
||||
opt_in_state_valid,
|
||||
[
|
||||
"Live paid/mutating promotion execution is disabled unless HERMES_CTO_LIVE_PROMOTION=1",
|
||||
"HERMES_CTO_LIVE_PROMOTION_ACK must match the required acknowledgement string",
|
||||
],
|
||||
live_requested=live_requested,
|
||||
live_acknowledged=live_ack,
|
||||
live_execution_allowed=live_execution_allowed,
|
||||
opt_in_state_valid=opt_in_state_valid,
|
||||
),
|
||||
]
|
||||
all_passed = all(item["status"] == "pass" for item in eval_results)
|
||||
@@ -149,7 +161,7 @@ def build_report(output: Path) -> dict[str, Any]:
|
||||
"artifacts": {
|
||||
"transcript": "sot/08-OUTPUTS/CTO-WEBUI-CODER-PRD-EVIDENCE-2026-05-25.md",
|
||||
"diff": "local-worktree",
|
||||
"logs": str(output.relative_to(REPO_ROOT)),
|
||||
"logs": _artifact_path(output),
|
||||
"screenshots": [],
|
||||
},
|
||||
"eval_results": eval_results,
|
||||
|
||||
Reference in New Issue
Block a user