Refresh CTO terminal event eval proof
This commit is contained in:
parent
e5040db9bc
commit
d3e3f70a0b
@ -6,7 +6,7 @@ eval_id: live-profile-drift
|
|||||||
profile: cto-planb
|
profile: cto-planb
|
||||||
status: pass
|
status: pass
|
||||||
score: 100
|
score: 100
|
||||||
checked_at: '2026-05-25T17:20:57Z'
|
checked_at: '2026-05-25T17:21:42Z'
|
||||||
checks:
|
checks:
|
||||||
correctness: pass
|
correctness: pass
|
||||||
verification: pass
|
verification: pass
|
||||||
@ -76,7 +76,7 @@ commands:
|
|||||||
- command: hermes -p cto-planb skills list
|
- command: hermes -p cto-planb skills list
|
||||||
cwd: /home/svrnty/workspaces/hermes
|
cwd: /home/svrnty/workspaces/hermes
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 235
|
duration_ms: 214
|
||||||
stdout: " Installed Skills \n\u250F\
|
stdout: " Installed Skills \n\u250F\
|
||||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
||||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\
|
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\
|
||||||
@ -113,7 +113,7 @@ commands:
|
|||||||
- command: hermes -p cto-planb mcp list
|
- command: hermes -p cto-planb mcp list
|
||||||
cwd: /home/svrnty/workspaces/hermes
|
cwd: /home/svrnty/workspaces/hermes
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 470
|
duration_ms: 430
|
||||||
stdout: "\n MCP Servers:\n\n Name Transport Tools\
|
stdout: "\n MCP Servers:\n\n Name Transport Tools\
|
||||||
\ Status \n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
\ Status \n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||||
\u2500\u2500\u2500\u2500\u2500\u2500 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
\u2500\u2500\u2500\u2500\u2500\u2500 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||||
@ -126,7 +126,7 @@ commands:
|
|||||||
- command: ./install.sh --dry-run
|
- command: ./install.sh --dry-run
|
||||||
cwd: /home/svrnty/workspaces/hermes/cto
|
cwd: /home/svrnty/workspaces/hermes/cto
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 4
|
duration_ms: 3
|
||||||
stdout: "== preflight ==\n hermes \u2713 python3 \u2713 sqlite3 \u2713 HERMES_HOME\
|
stdout: "== preflight ==\n hermes \u2713 python3 \u2713 sqlite3 \u2713 HERMES_HOME\
|
||||||
\ \u2713\n sandcastle \u2713 (/home/svrnty/workspaces/hermes/cto/../sandcastle)\n\
|
\ \u2713\n sandcastle \u2713 (/home/svrnty/workspaces/hermes/cto/../sandcastle)\n\
|
||||||
== DRY RUN \u2014 no mutations ==\n would: ln -sfn /home/svrnty/workspaces/hermes/cto\
|
== DRY RUN \u2014 no mutations ==\n would: ln -sfn /home/svrnty/workspaces/hermes/cto\
|
||||||
|
|||||||
@ -59,7 +59,7 @@ eval_results:
|
|||||||
command:
|
command:
|
||||||
command: hermes -p cto-planb skills list
|
command: hermes -p cto-planb skills list
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 210
|
duration_ms: 215
|
||||||
stdout: " Installed Skills \n\u250F\
|
stdout: " Installed Skills \n\u250F\
|
||||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\
|
||||||
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\
|
\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\
|
||||||
@ -100,7 +100,7 @@ eval_results:
|
|||||||
command:
|
command:
|
||||||
command: hermes -p cto-planb mcp list
|
command: hermes -p cto-planb mcp list
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 443
|
duration_ms: 435
|
||||||
stdout: "\n MCP Servers:\n\n Name Transport \
|
stdout: "\n MCP Servers:\n\n Name Transport \
|
||||||
\ Tools Status \n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
\ Tools Status \n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
||||||
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u2500\u2500\u2500\u2500\u2500\
|
\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u2500\u2500\u2500\u2500\u2500\
|
||||||
|
|||||||
@ -31,26 +31,26 @@ eval_results:
|
|||||||
evidence:
|
evidence:
|
||||||
- cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
- cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
||||||
command: python3 evals/runners/run-promotion-suite.py --output evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
command: python3 evals/runners/run-promotion-suite.py --output evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
||||||
duration_ms: 36
|
duration_ms: 35
|
||||||
- eval_id: promotion-fixture-execution
|
- eval_id: promotion-fixture-execution
|
||||||
status: pass
|
status: pass
|
||||||
evidence:
|
evidence:
|
||||||
- cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
- cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
||||||
command: python3 evals/runners/run-promotion-fixtures.py --output evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
command: python3 evals/runners/run-promotion-fixtures.py --output evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
||||||
--artifact-output evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
--artifact-output evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
||||||
duration_ms: 744
|
duration_ms: 741
|
||||||
- eval_id: live-promotion-readiness
|
- eval_id: live-promotion-readiness
|
||||||
status: pass
|
status: pass
|
||||||
evidence:
|
evidence:
|
||||||
- cto/evals/reports/2026-05-25-live-promotion-readiness.yaml
|
- cto/evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||||
command: python3 evals/runners/run-live-promotion-readiness.py --output evals/reports/2026-05-25-live-promotion-readiness.yaml
|
command: python3 evals/runners/run-live-promotion-readiness.py --output evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||||
duration_ms: 693
|
duration_ms: 687
|
||||||
- eval_id: static-prd-contract
|
- eval_id: static-prd-contract
|
||||||
status: pass
|
status: pass
|
||||||
evidence:
|
evidence:
|
||||||
- tests/e2e/test_j_cto_webui_prd.py
|
- tests/e2e/test_j_cto_webui_prd.py
|
||||||
command: pytest -q tests/e2e/test_j_cto_webui_prd.py
|
command: pytest -q tests/e2e/test_j_cto_webui_prd.py
|
||||||
duration_ms: 1216
|
duration_ms: 1180
|
||||||
- eval_id: webui-cto-event-browser
|
- eval_id: webui-cto-event-browser
|
||||||
status: pass
|
status: pass
|
||||||
evidence:
|
evidence:
|
||||||
@ -59,37 +59,37 @@ eval_results:
|
|||||||
command: pytest -q tests/test_cto_events.py tests/test_live_tool_callback_events.py
|
command: pytest -q tests/test_cto_events.py tests/test_live_tool_callback_events.py
|
||||||
tests/test_cto_webui_journal_e2e.py tests/test_cto_browser_e2e.py tests/test_cancel_interrupt.py
|
tests/test_cto_webui_journal_e2e.py tests/test_cto_browser_e2e.py tests/test_cancel_interrupt.py
|
||||||
tests/test_approval_queue.py
|
tests/test_approval_queue.py
|
||||||
duration_ms: 2364
|
duration_ms: 3186
|
||||||
- eval_id: webui-cto-live-streaming
|
- eval_id: webui-cto-live-streaming
|
||||||
status: pass
|
status: pass
|
||||||
evidence:
|
evidence:
|
||||||
- hermes-webui/tests/test_cto_live_streaming_e2e.py
|
- hermes-webui/tests/test_cto_live_streaming_e2e.py
|
||||||
command: pytest -q tests/test_cto_live_streaming_e2e.py
|
command: pytest -q tests/test_cto_live_streaming_e2e.py
|
||||||
duration_ms: 1220
|
duration_ms: 2097
|
||||||
- eval_id: live-profile-drift
|
- eval_id: live-profile-drift
|
||||||
status: pass
|
status: pass
|
||||||
evidence:
|
evidence:
|
||||||
- cto/evals/reports/2026-05-25-live-drift.yaml
|
- cto/evals/reports/2026-05-25-live-drift.yaml
|
||||||
command: python3 evals/runners/drift.py --output evals/reports/2026-05-25-live-drift.yaml
|
command: python3 evals/runners/drift.py --output evals/reports/2026-05-25-live-drift.yaml
|
||||||
duration_ms: 752
|
duration_ms: 690
|
||||||
- eval_id: eval-report-scoring
|
- eval_id: eval-report-scoring
|
||||||
status: pass
|
status: pass
|
||||||
evidence:
|
evidence:
|
||||||
- cto/evals/reports/*.yaml
|
- cto/evals/reports/*.yaml
|
||||||
command: bash -lc for r in evals/reports/*.yaml; do python3 evals/runners/score.py
|
command: bash -lc for r in evals/reports/*.yaml; do python3 evals/runners/score.py
|
||||||
"$r"; done
|
"$r"; done
|
||||||
duration_ms: 319
|
duration_ms: 291
|
||||||
- eval_id: diff-whitespace-check
|
- eval_id: diff-whitespace-check
|
||||||
status: pass
|
status: pass
|
||||||
evidence:
|
evidence:
|
||||||
- git diff --check
|
- git diff --check
|
||||||
command: git diff --check
|
command: git diff --check
|
||||||
duration_ms: 4
|
duration_ms: 7
|
||||||
commands:
|
commands:
|
||||||
- command: python3 evals/runners/run-promotion-suite.py --output evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
- command: python3 evals/runners/run-promotion-suite.py --output evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
||||||
cwd: /home/svrnty/workspaces/hermes/cto
|
cwd: /home/svrnty/workspaces/hermes/cto
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 36
|
duration_ms: 35
|
||||||
stdout: 'wrote /home/svrnty/workspaces/hermes/cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
stdout: 'wrote /home/svrnty/workspaces/hermes/cto/evals/reports/2026-05-25-promotion-suite-readiness.yaml
|
||||||
|
|
||||||
'
|
'
|
||||||
@ -98,7 +98,7 @@ commands:
|
|||||||
--artifact-output evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
--artifact-output evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
||||||
cwd: /home/svrnty/workspaces/hermes/cto
|
cwd: /home/svrnty/workspaces/hermes/cto
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 744
|
duration_ms: 741
|
||||||
stdout: 'wrote /home/svrnty/workspaces/hermes/cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
stdout: 'wrote /home/svrnty/workspaces/hermes/cto/evals/reports/2026-05-25-promotion-fixture-execution.yaml
|
||||||
|
|
||||||
wrote /home/svrnty/workspaces/hermes/cto/evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
wrote /home/svrnty/workspaces/hermes/cto/evals/artifacts/2026-05-25-promotion-fixture-execution.json
|
||||||
@ -108,7 +108,7 @@ commands:
|
|||||||
- command: python3 evals/runners/run-live-promotion-readiness.py --output evals/reports/2026-05-25-live-promotion-readiness.yaml
|
- command: python3 evals/runners/run-live-promotion-readiness.py --output evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||||
cwd: /home/svrnty/workspaces/hermes/cto
|
cwd: /home/svrnty/workspaces/hermes/cto
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 693
|
duration_ms: 687
|
||||||
stdout: 'wrote evals/reports/2026-05-25-live-promotion-readiness.yaml
|
stdout: 'wrote evals/reports/2026-05-25-live-promotion-readiness.yaml
|
||||||
|
|
||||||
'
|
'
|
||||||
@ -116,10 +116,10 @@ commands:
|
|||||||
- command: pytest -q tests/e2e/test_j_cto_webui_prd.py
|
- command: pytest -q tests/e2e/test_j_cto_webui_prd.py
|
||||||
cwd: /home/svrnty/workspaces/hermes
|
cwd: /home/svrnty/workspaces/hermes
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 1216
|
duration_ms: 1180
|
||||||
stdout: '.......... [100%]
|
stdout: '.......... [100%]
|
||||||
|
|
||||||
10 passed in 1.04s
|
10 passed in 1.00s
|
||||||
|
|
||||||
'
|
'
|
||||||
stderr: ''
|
stderr: ''
|
||||||
@ -128,27 +128,27 @@ commands:
|
|||||||
tests/test_approval_queue.py
|
tests/test_approval_queue.py
|
||||||
cwd: /home/svrnty/workspaces/hermes/hermes-webui
|
cwd: /home/svrnty/workspaces/hermes/hermes-webui
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 2364
|
duration_ms: 3186
|
||||||
stdout: '...................................... [100%]
|
stdout: '...................................... [100%]
|
||||||
|
|
||||||
38 passed in 1.89s
|
38 passed in 2.72s
|
||||||
|
|
||||||
'
|
'
|
||||||
stderr: ''
|
stderr: ''
|
||||||
- command: pytest -q tests/test_cto_live_streaming_e2e.py
|
- command: pytest -q tests/test_cto_live_streaming_e2e.py
|
||||||
cwd: /home/svrnty/workspaces/hermes/hermes-webui
|
cwd: /home/svrnty/workspaces/hermes/hermes-webui
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 1220
|
duration_ms: 2097
|
||||||
stdout: '. [100%]
|
stdout: '. [100%]
|
||||||
|
|
||||||
1 passed in 0.88s
|
1 passed in 1.77s
|
||||||
|
|
||||||
'
|
'
|
||||||
stderr: ''
|
stderr: ''
|
||||||
- command: python3 evals/runners/drift.py --output evals/reports/2026-05-25-live-drift.yaml
|
- command: python3 evals/runners/drift.py --output evals/reports/2026-05-25-live-drift.yaml
|
||||||
cwd: /home/svrnty/workspaces/hermes/cto
|
cwd: /home/svrnty/workspaces/hermes/cto
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 752
|
duration_ms: 690
|
||||||
stdout: 'wrote evals/reports/2026-05-25-live-drift.yaml
|
stdout: 'wrote evals/reports/2026-05-25-live-drift.yaml
|
||||||
|
|
||||||
'
|
'
|
||||||
@ -157,7 +157,7 @@ commands:
|
|||||||
"$r"; done
|
"$r"; done
|
||||||
cwd: /home/svrnty/workspaces/hermes/cto
|
cwd: /home/svrnty/workspaces/hermes/cto
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 319
|
duration_ms: 291
|
||||||
stdout: 'ok
|
stdout: 'ok
|
||||||
|
|
||||||
ok
|
ok
|
||||||
@ -183,7 +183,7 @@ commands:
|
|||||||
- command: git diff --check
|
- command: git diff --check
|
||||||
cwd: /home/svrnty/workspaces/hermes
|
cwd: /home/svrnty/workspaces/hermes
|
||||||
returncode: 0
|
returncode: 0
|
||||||
duration_ms: 4
|
duration_ms: 7
|
||||||
stdout: ''
|
stdout: ''
|
||||||
stderr: ''
|
stderr: ''
|
||||||
notes:
|
notes:
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user