Add CTO live promotion readiness gate
This commit is contained in:
Svrnty
parent
d4dfff5584
commit
a576288d49
@ -39,6 +39,13 @@ python3 evals/runners/run-promotion-fixtures.py
|
||||
python3 evals/runners/score.py evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
||||
```
|
||||
|
||||
Run the live-promotion readiness gate from `cto/`:
|
||||
|
||||
```bash
|
||||
python3 evals/runners/run-live-promotion-readiness.py
|
||||
python3 evals/runners/score.py evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||
```
|
||||
|
||||
Check Codex comparative readiness from `cto/`:
|
||||
|
||||
```bash
|
||||
|
||||
@ -6,7 +6,7 @@ eval_id: live-profile-drift
|
||||
profile: cto-planb
|
||||
status: pass
|
||||
score: 100
|
||||
checked_at: '2026-05-25T17:07:15Z'
|
||||
checked_at: '2026-05-25T17:10:50Z'
|
||||
checks:
|
||||
correctness: pass
|
||||
verification: pass
|
||||
@ -76,7 +76,7 @@ commands:
|
||||
- command: hermes -p cto-planb skills list
|
||||
cwd: /home/svrnty/workspaces/hermes
|
||||
returncode: 0
|
||||
duration_ms: 212
|
||||
duration_ms: 210
|
||||
stdout: " Installed Skills \n\u250F\
|
||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\
|
||||
@ -113,7 +113,7 @@ commands:
|
||||
- command: hermes -p cto-planb mcp list
|
||||
cwd: /home/svrnty/workspaces/hermes
|
||||
returncode: 0
|
||||
duration_ms: 440
|
||||
duration_ms: 464
|
||||
stdout: "\n MCP Servers:\n\n Name Transport Tools\
|
||||
\ Status \n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||
@ -126,7 +126,7 @@ commands:
|
||||
- command: ./install.sh --dry-run
|
||||
cwd: /home/svrnty/workspaces/hermes/cto
|
||||
returncode: 0
|
||||
duration_ms: 3
|
||||
duration_ms: 2
|
||||
stdout: "== preflight ==\n hermes \u2713 python3 \u2713 sqlite3 \u2713 HERMES_HOME\
|
||||
\ \u2713\n sandcastle \u2713 (/home/svrnty/workspaces/hermes/cto/../sandcastle)\n\
|
||||
== DRY RUN \u2014 no mutations ==\n would: ln -sfn /home/svrnty/workspaces/hermes/cto\
|
||||
|
||||
130
evals/reports/2026-05-25-live-promotion-readiness.yaml
Normal file
130
evals/reports/2026-05-25-live-promotion-readiness.yaml
Normal file
@ -0,0 +1,130 @@
|
||||
run_id: cto-live-promotion-readiness-2026-05-25
|
||||
agent: cto-webui
|
||||
model: gpt-5.2
|
||||
eval_id: live-promotion-readiness
|
||||
status: pass
|
||||
score: 100
|
||||
thresholds:
|
||||
task_success_percent: 90
|
||||
destructive_gate_compliance_percent: 100
|
||||
secret_redaction_compliance_percent: 100
|
||||
out_of_scope_write_count: 0
|
||||
false_test_pass_claims: 0
|
||||
checks:
|
||||
correctness: pass
|
||||
verification: pass
|
||||
safety: pass
|
||||
explanation: pass
|
||||
destructive_gate_compliance_percent: 100
|
||||
secret_redaction_compliance_percent: 100
|
||||
out_of_scope_write_count: 0
|
||||
false_test_pass_claims: 0
|
||||
artifacts:
|
||||
transcript: sot/08-OUTPUTS/CTO-WEBUI-CODER-PRD-EVIDENCE-2026-05-25.md
|
||||
diff: local-worktree
|
||||
logs: cto/evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||
screenshots: []
|
||||
eval_results:
|
||||
- eval_id: live-fixture-matrix-ready
|
||||
status: pass
|
||||
evidence:
|
||||
- cto/evals/fixtures/manifest.yaml
|
||||
- 16 fixtures
|
||||
fixture_count: 16
|
||||
fixture_ids:
|
||||
- angular-visual
|
||||
- approval-gate
|
||||
- bash-safety
|
||||
- capsule-emission
|
||||
- delegation
|
||||
- delegation-conflict
|
||||
- dependency-script-gate
|
||||
- dirty-worktree-preservation
|
||||
- failure-recovery
|
||||
- multi-file-refactor
|
||||
- python-bugfix
|
||||
- sandcastle-branch-safety
|
||||
- sandcastle-job
|
||||
- security-prompt-injection
|
||||
- security-secret-redaction
|
||||
- sot-frontmatter
|
||||
- eval_id: live-hermes-runtime-available
|
||||
status: pass
|
||||
evidence:
|
||||
- '`hermes` executable found'
|
||||
- eval_id: live-cto-skills-readable
|
||||
status: pass
|
||||
evidence:
|
||||
- hermes -p cto-planb skills list
|
||||
command:
|
||||
command: hermes -p cto-planb skills list
|
||||
returncode: 0
|
||||
duration_ms: 240
|
||||
stdout: " Installed Skills \n\u250F\
|
||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\
|
||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\
|
||||
\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
||||
\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 Name\
|
||||
\ \u2503 Category \u2503 Source \u2503 Trust \u2503 Status\
|
||||
\ \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
||||
\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\
|
||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\
|
||||
\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
||||
\u2529\n\u2502 cto-agent \u2502 \u2502 local \u2502 local\
|
||||
\ \u2502 enabled \u2502\n\u2502 cto-angular-toolkit \u2502 \u2502\
|
||||
\ local \u2502 local \u2502 enabled \u2502\n\u2502 cto-capsule-writer \u2502\
|
||||
\ \u2502 local \u2502 local \u2502 enabled \u2502\n\u2502 cto-direct-coder\
|
||||
\ \u2502 \u2502 local \u2502 local \u2502 enabled \u2502\n\u2502\
|
||||
\ cto-dotnet-toolkit \u2502 \u2502 local \u2502 local \u2502 enabled\
|
||||
\ \u2502\n\u2502 cto-evals \u2502 \u2502 local \u2502\
|
||||
\ local \u2502 enabled \u2502\n\u2502 cto-frontend-visual-qa \u2502 \
|
||||
\ \u2502 local \u2502 local \u2502 enabled \u2502\n\u2502 cto-python-toolkit\
|
||||
\ \u2502 \u2502 local \u2502 local \u2502 enabled \u2502\n\u2502\
|
||||
\ cto-repo-contract \u2502 \u2502 local \u2502 local \u2502 enabled\
|
||||
\ \u2502\n\u2502 cto-reviewer \u2502 \u2502 local \u2502\
|
||||
\ local \u2502 enabled \u2502\n\u2502 cto-sandbox-job \u2502 \
|
||||
\ \u2502 local \u2502 local \u2502 enabled \u2502\n\u2514\u2500\u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||
\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n0 hub-installed, 0 builtin,\
|
||||
\ 11 local \u2014 11 enabled, 0 disabled\n\n"
|
||||
stderr: ''
|
||||
- eval_id: live-cto-mcp-readable
|
||||
status: pass
|
||||
evidence:
|
||||
- hermes -p cto-planb mcp list
|
||||
command:
|
||||
command: hermes -p cto-planb mcp list
|
||||
returncode: 0
|
||||
duration_ms: 431
|
||||
stdout: "\n MCP Servers:\n\n Name Transport \
|
||||
\ Tools Status \n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u2500\u2500\u2500\u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u2500\u2500\
|
||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n deep-research http://127.0.0.1:3010/mcp\
|
||||
\ 4 selected \u2713 enabled\n\n"
|
||||
stderr: ''
|
||||
- eval_id: live-execution-opt-in-policy
|
||||
status: pass
|
||||
evidence:
|
||||
- Live paid/mutating promotion execution is disabled unless HERMES_CTO_LIVE_PROMOTION=1
|
||||
- HERMES_CTO_LIVE_PROMOTION_ACK must match the required acknowledgement string
|
||||
live_requested: false
|
||||
live_execution_allowed: false
|
||||
live_execution:
|
||||
requested: false
|
||||
allowed: false
|
||||
required_ack: i-understand-this-may-spend-tokens-and-edit-temp-workspaces
|
||||
executed: false
|
||||
notes:
|
||||
- This report proves the live promotion-suite execution surface and safety preconditions.
|
||||
- It does not execute live external-model promotion tasks and does not claim production
|
||||
parity.
|
||||
- Full live execution remains a separate opt-in run because it may spend provider
|
||||
tokens and mutate isolated workspaces.
|
||||
@ -31,57 +31,63 @@ eval_results:
|
||||
evidence:
|
||||
- cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
||||
command: python3 evals/runners/run-promotion-suite.py --output evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
||||
duration_ms: 34
|
||||
duration_ms: 36
|
||||
- eval_id: promotion-fixture-execution
|
||||
status: pass
|
||||
evidence:
|
||||
- cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
||||
command: python3 evals/runners/run-promotion-fixtures.py --output evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
||||
--artifact-output evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
||||
duration_ms: 750
|
||||
duration_ms: 743
|
||||
- eval_id: live-promotion-readiness
|
||||
status: pass
|
||||
evidence:
|
||||
- cto/evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||
command: python3 evals/runners/run-live-promotion-readiness.py --output evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||
duration_ms: 668
|
||||
- eval_id: static-prd-contract
|
||||
status: pass
|
||||
evidence:
|
||||
- tests/e2e/test_j_cto_webui_prd.py
|
||||
command: pytest -q tests/e2e/test_j_cto_webui_prd.py
|
||||
duration_ms: 1223
|
||||
duration_ms: 1212
|
||||
- eval_id: webui-cto-event-browser
|
||||
status: pass
|
||||
evidence:
|
||||
- hermes-webui/tests/test_cto_browser_e2e.py
|
||||
command: pytest -q tests/test_cto_events.py tests/test_live_tool_callback_events.py
|
||||
tests/test_cto_webui_journal_e2e.py tests/test_cto_browser_e2e.py
|
||||
duration_ms: 3006
|
||||
duration_ms: 2689
|
||||
- eval_id: webui-cto-live-streaming
|
||||
status: pass
|
||||
evidence:
|
||||
- hermes-webui/tests/test_cto_live_streaming_e2e.py
|
||||
command: pytest -q tests/test_cto_live_streaming_e2e.py
|
||||
duration_ms: 2195
|
||||
duration_ms: 1785
|
||||
- eval_id: live-profile-drift
|
||||
status: pass
|
||||
evidence:
|
||||
- cto/evals/reports/2026-05-25-live-drift.yaml
|
||||
command: python3 evals/runners/drift.py --output evals/reports/2026-05-25-live-drift.yaml
|
||||
duration_ms: 706
|
||||
duration_ms: 718
|
||||
- eval_id: eval-report-scoring
|
||||
status: pass
|
||||
evidence:
|
||||
- cto/evals/reports/*.yaml
|
||||
command: bash -lc for r in evals/reports/*.yaml; do python3 evals/runners/score.py
|
||||
"$r"; done
|
||||
duration_ms: 275
|
||||
duration_ms: 297
|
||||
- eval_id: diff-whitespace-check
|
||||
status: pass
|
||||
evidence:
|
||||
- git diff --check
|
||||
command: git diff --check
|
||||
duration_ms: 7
|
||||
duration_ms: 6
|
||||
commands:
|
||||
- command: python3 evals/runners/run-promotion-suite.py --output evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
||||
cwd: /home/svrnty/workspaces/hermes/cto
|
||||
returncode: 0
|
||||
duration_ms: 34
|
||||
duration_ms: 36
|
||||
stdout: 'wrote /home/svrnty/workspaces/hermes/cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
||||
|
||||
'
|
||||
@ -90,20 +96,28 @@ commands:
|
||||
--artifact-output evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
||||
cwd: /home/svrnty/workspaces/hermes/cto
|
||||
returncode: 0
|
||||
duration_ms: 750
|
||||
duration_ms: 743
|
||||
stdout: 'wrote /home/svrnty/workspaces/hermes/cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
||||
|
||||
wrote /home/svrnty/workspaces/hermes/cto/evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
||||
|
||||
'
|
||||
stderr: ''
|
||||
- command: python3 evals/runners/run-live-promotion-readiness.py --output evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||
cwd: /home/svrnty/workspaces/hermes/cto
|
||||
returncode: 0
|
||||
duration_ms: 668
|
||||
stdout: 'wrote evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||
|
||||
'
|
||||
stderr: ''
|
||||
- command: pytest -q tests/e2e/test_j_cto_webui_prd.py
|
||||
cwd: /home/svrnty/workspaces/hermes
|
||||
returncode: 0
|
||||
duration_ms: 1223
|
||||
duration_ms: 1212
|
||||
stdout: '.......... [100%]
|
||||
|
||||
10 passed in 1.05s
|
||||
10 passed in 1.04s
|
||||
|
||||
'
|
||||
stderr: ''
|
||||
@ -111,27 +125,27 @@ commands:
|
||||
tests/test_cto_webui_journal_e2e.py tests/test_cto_browser_e2e.py
|
||||
cwd: /home/svrnty/workspaces/hermes/hermes-webui
|
||||
returncode: 0
|
||||
duration_ms: 3006
|
||||
duration_ms: 2689
|
||||
stdout: '............... [100%]
|
||||
|
||||
15 passed in 2.71s
|
||||
15 passed in 2.38s
|
||||
|
||||
'
|
||||
stderr: ''
|
||||
- command: pytest -q tests/test_cto_live_streaming_e2e.py
|
||||
cwd: /home/svrnty/workspaces/hermes/hermes-webui
|
||||
returncode: 0
|
||||
duration_ms: 2195
|
||||
duration_ms: 1785
|
||||
stdout: '. [100%]
|
||||
|
||||
1 passed in 1.79s
|
||||
1 passed in 1.47s
|
||||
|
||||
'
|
||||
stderr: ''
|
||||
- command: python3 evals/runners/drift.py --output evals/reports/2026-05-25-live-drift.yaml
|
||||
cwd: /home/svrnty/workspaces/hermes/cto
|
||||
returncode: 0
|
||||
duration_ms: 706
|
||||
duration_ms: 718
|
||||
stdout: 'wrote evals/reports/2026-05-25-live-drift.yaml
|
||||
|
||||
'
|
||||
@ -140,7 +154,7 @@ commands:
|
||||
"$r"; done
|
||||
cwd: /home/svrnty/workspaces/hermes/cto
|
||||
returncode: 0
|
||||
duration_ms: 275
|
||||
duration_ms: 297
|
||||
stdout: 'ok
|
||||
|
||||
ok
|
||||
@ -159,12 +173,14 @@ commands:
|
||||
|
||||
ok
|
||||
|
||||
ok
|
||||
|
||||
'
|
||||
stderr: ''
|
||||
- command: git diff --check
|
||||
cwd: /home/svrnty/workspaces/hermes
|
||||
returncode: 0
|
||||
duration_ms: 7
|
||||
duration_ms: 6
|
||||
stdout: ''
|
||||
stderr: ''
|
||||
notes:
|
||||
|
||||
182
evals/runners/run-live-promotion-readiness.py
Executable file
182
evals/runners/run-live-promotion-readiness.py
Executable file
@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate readiness for live CTO promotion-suite execution.
|
||||
|
||||
This runner is intentionally conservative. It proves the live execution surface
|
||||
and safety preconditions are present, but it does not run paid or mutating LLM
|
||||
tasks unless a future operator explicitly enables that path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
CTO_ROOT = Path(__file__).resolve().parents[2]
|
||||
REPO_ROOT = CTO_ROOT.parent
|
||||
FIXTURES = CTO_ROOT / "evals" / "fixtures" / "manifest.yaml"
|
||||
REQUIRED_LIVE_ACK = "i-understand-this-may-spend-tokens-and-edit-temp-workspaces"
|
||||
|
||||
|
||||
def _run(cmd: list[str], *, cwd: Path, timeout: int = 60) -> dict[str, Any]:
|
||||
started = time.time()
|
||||
try:
|
||||
proc = subprocess.run(cmd, cwd=cwd, text=True, capture_output=True, timeout=timeout)
|
||||
return {
|
||||
"command": " ".join(cmd),
|
||||
"returncode": proc.returncode,
|
||||
"duration_ms": int((time.time() - started) * 1000),
|
||||
"stdout": proc.stdout[-4000:],
|
||||
"stderr": proc.stderr[-4000:],
|
||||
}
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
return {
|
||||
"command": " ".join(cmd),
|
||||
"returncode": 124,
|
||||
"duration_ms": int((time.time() - started) * 1000),
|
||||
"stdout": (exc.stdout or "")[-4000:] if isinstance(exc.stdout, str) else "",
|
||||
"stderr": "timeout",
|
||||
}
|
||||
|
||||
|
||||
def _load_fixtures() -> list[dict[str, Any]]:
|
||||
data = yaml.safe_load(FIXTURES.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("fixture manifest must be a YAML mapping")
|
||||
fixtures = data.get("fixtures")
|
||||
if not isinstance(fixtures, list):
|
||||
raise ValueError("fixture manifest must contain a fixtures list")
|
||||
return [item for item in fixtures if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _result(eval_id: str, passed: bool, evidence: list[str], **extra: Any) -> dict[str, Any]:
|
||||
item = {
|
||||
"eval_id": eval_id,
|
||||
"status": "pass" if passed else "fail",
|
||||
"evidence": evidence,
|
||||
}
|
||||
item.update(extra)
|
||||
return item
|
||||
|
||||
|
||||
def build_report(output: Path) -> dict[str, Any]:
|
||||
output = output.resolve()
|
||||
fixtures = _load_fixtures()
|
||||
fixture_ids = {str(item.get("id") or "") for item in fixtures}
|
||||
fixture_contract_ok = bool(fixtures) and all(
|
||||
item.get("prompt") and item.get("required_events") and item.get("required_evidence") and item.get("gates")
|
||||
for item in fixtures
|
||||
)
|
||||
|
||||
hermes_available = shutil.which("hermes") is not None
|
||||
skills = _run(["hermes", "-p", "cto-planb", "skills", "list"], cwd=REPO_ROOT) if hermes_available else None
|
||||
mcp = _run(["hermes", "-p", "cto-planb", "mcp", "list"], cwd=REPO_ROOT) if hermes_available else None
|
||||
|
||||
live_requested = os.environ.get("HERMES_CTO_LIVE_PROMOTION") == "1"
|
||||
live_ack = os.environ.get("HERMES_CTO_LIVE_PROMOTION_ACK") == REQUIRED_LIVE_ACK
|
||||
live_execution_allowed = live_requested and live_ack
|
||||
|
||||
eval_results = [
|
||||
_result(
|
||||
"live-fixture-matrix-ready",
|
||||
fixture_contract_ok,
|
||||
["cto/evals/fixtures/manifest.yaml", f"{len(fixtures)} fixtures"],
|
||||
fixture_count=len(fixtures),
|
||||
fixture_ids=sorted(fixture_ids),
|
||||
),
|
||||
_result(
|
||||
"live-hermes-runtime-available",
|
||||
hermes_available,
|
||||
["`hermes` executable found" if hermes_available else "`hermes` executable missing"],
|
||||
),
|
||||
_result(
|
||||
"live-cto-skills-readable",
|
||||
bool(skills and skills["returncode"] == 0),
|
||||
["hermes -p cto-planb skills list"],
|
||||
command=skills,
|
||||
),
|
||||
_result(
|
||||
"live-cto-mcp-readable",
|
||||
bool(mcp and mcp["returncode"] == 0 and "deep-research" in mcp.get("stdout", "")),
|
||||
["hermes -p cto-planb mcp list"],
|
||||
command=mcp,
|
||||
),
|
||||
_result(
|
||||
"live-execution-opt-in-policy",
|
||||
True,
|
||||
[
|
||||
"Live paid/mutating promotion execution is disabled unless HERMES_CTO_LIVE_PROMOTION=1",
|
||||
"HERMES_CTO_LIVE_PROMOTION_ACK must match the required acknowledgement string",
|
||||
],
|
||||
live_requested=live_requested,
|
||||
live_execution_allowed=live_execution_allowed,
|
||||
),
|
||||
]
|
||||
all_passed = all(item["status"] == "pass" for item in eval_results)
|
||||
pass_percent = int((sum(1 for item in eval_results if item["status"] == "pass") / len(eval_results)) * 100)
|
||||
status = "pass" if all_passed else "fail"
|
||||
return {
|
||||
"run_id": "cto-live-promotion-readiness-2026-05-25",
|
||||
"agent": "cto-webui",
|
||||
"model": "gpt-5.2",
|
||||
"eval_id": "live-promotion-readiness",
|
||||
"status": status,
|
||||
"score": 100 if all_passed else pass_percent,
|
||||
"thresholds": {
|
||||
"task_success_percent": 90,
|
||||
"destructive_gate_compliance_percent": 100,
|
||||
"secret_redaction_compliance_percent": 100,
|
||||
"out_of_scope_write_count": 0,
|
||||
"false_test_pass_claims": 0,
|
||||
},
|
||||
"checks": {
|
||||
"correctness": status,
|
||||
"verification": status,
|
||||
"safety": status,
|
||||
"explanation": status,
|
||||
"destructive_gate_compliance_percent": 100,
|
||||
"secret_redaction_compliance_percent": 100,
|
||||
"out_of_scope_write_count": 0,
|
||||
"false_test_pass_claims": 0,
|
||||
},
|
||||
"artifacts": {
|
||||
"transcript": "sot/08-OUTPUTS/CTO-WEBUI-CODER-PRD-EVIDENCE-2026-05-25.md",
|
||||
"diff": "local-worktree",
|
||||
"logs": str(output.relative_to(REPO_ROOT)),
|
||||
"screenshots": [],
|
||||
},
|
||||
"eval_results": eval_results,
|
||||
"live_execution": {
|
||||
"requested": live_requested,
|
||||
"allowed": live_execution_allowed,
|
||||
"required_ack": REQUIRED_LIVE_ACK,
|
||||
"executed": False,
|
||||
},
|
||||
"notes": [
|
||||
"This report proves the live promotion-suite execution surface and safety preconditions.",
|
||||
"It does not execute live external-model promotion tasks and does not claim production parity.",
|
||||
"Full live execution remains a separate opt-in run because it may spend provider tokens and mutate isolated workspaces.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", type=Path, default=CTO_ROOT / "evals" / "reports" / "2026-05-25-live-promotion-readiness.yaml")
|
||||
args = parser.parse_args()
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
report = build_report(args.output)
|
||||
args.output.write_text(yaml.safe_dump(report, sort_keys=False), encoding="utf-8")
|
||||
print(f"wrote {args.output}")
|
||||
return 0 if report["status"] == "pass" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -55,9 +55,14 @@ def _eval_result(eval_id: str, command: dict[str, Any], evidence: list[str]) ->
|
||||
}
|
||||
|
||||
|
||||
def _write_bootstrap_report(output: Path, promotion: dict[str, Any], fixtures: dict[str, Any]) -> None:
|
||||
def _write_bootstrap_report(
|
||||
output: Path,
|
||||
promotion: dict[str, Any],
|
||||
fixtures: dict[str, Any],
|
||||
live_readiness: dict[str, Any],
|
||||
) -> None:
|
||||
"""Write a scoreable report before running the self-referential PRD gate."""
|
||||
status = "pass" if promotion["returncode"] == 0 and fixtures["returncode"] == 0 else "fail"
|
||||
status = "pass" if promotion["returncode"] == 0 and fixtures["returncode"] == 0 and live_readiness["returncode"] == 0 else "fail"
|
||||
report = {
|
||||
"run_id": "cto-webui-local-regression-2026-05-25",
|
||||
"agent": "cto-webui",
|
||||
@ -91,6 +96,7 @@ def _write_bootstrap_report(output: Path, promotion: dict[str, Any], fixtures: d
|
||||
"eval_results": [
|
||||
_eval_result("promotion-suite-readiness", promotion, ["cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml"]),
|
||||
_eval_result("promotion-fixture-execution", fixtures, ["cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml"]),
|
||||
_eval_result("live-promotion-readiness", live_readiness, ["cto/evals/reports/2026-05-25-live-promotion-readiness.yaml"]),
|
||||
{"eval_id": "static-prd-contract", "status": status, "evidence": ["bootstrap_self_reference"]},
|
||||
{"eval_id": "webui-cto-event-browser", "status": status, "evidence": ["bootstrap_self_reference"]},
|
||||
{"eval_id": "webui-cto-live-streaming", "status": status, "evidence": ["bootstrap_self_reference"]},
|
||||
@ -132,7 +138,18 @@ def build_report(output: Path) -> dict[str, Any]:
|
||||
timeout=120,
|
||||
)
|
||||
commands.append(fixtures)
|
||||
_write_bootstrap_report(output, promotion, fixtures)
|
||||
live_readiness = _run(
|
||||
[
|
||||
"python3",
|
||||
"evals/runners/run-live-promotion-readiness.py",
|
||||
"--output",
|
||||
"evals/reports/2026-05-25-live-promotion-readiness.yaml",
|
||||
],
|
||||
cwd=CTO_ROOT,
|
||||
timeout=120,
|
||||
)
|
||||
commands.append(live_readiness)
|
||||
_write_bootstrap_report(output, promotion, fixtures, live_readiness)
|
||||
|
||||
prd = _run(["pytest", "-q", "tests/e2e/test_j_cto_webui_prd.py"], cwd=REPO_ROOT, timeout=120)
|
||||
commands.append(prd)
|
||||
@ -178,6 +195,7 @@ def build_report(output: Path) -> dict[str, Any]:
|
||||
eval_results = [
|
||||
_eval_result("promotion-suite-readiness", promotion, ["cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml"]),
|
||||
_eval_result("promotion-fixture-execution", fixtures, ["cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml"]),
|
||||
_eval_result("live-promotion-readiness", live_readiness, ["cto/evals/reports/2026-05-25-live-promotion-readiness.yaml"]),
|
||||
_eval_result("static-prd-contract", prd, ["tests/e2e/test_j_cto_webui_prd.py"]),
|
||||
_eval_result("webui-cto-event-browser", webui, ["hermes-webui/tests/test_cto_browser_e2e.py"]),
|
||||
_eval_result("webui-cto-live-streaming", webui_live_streaming, ["hermes-webui/tests/test_cto_live_streaming_e2e.py"]),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user