From eefa1567344af0dd1894186a93df92895c91a06c Mon Sep 17 00:00:00 2001 From: Svrnty Date: Sun, 31 May 2026 19:45:28 -0400 Subject: [PATCH] Record Case model admission gate --- WORKBOARD.yaml | 33 +++++++++++-------- .../CTO-CASE-PROVIDER-BUILD-ISSUES.md | 29 ++++++++++++++++ .../CTO-CASE-PROVIDER-BUILD-PRD.md | 9 +++++ tools/validate_cto_child.py | 10 ++++++ 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/WORKBOARD.yaml b/WORKBOARD.yaml index da6c1bc..b10c106 100644 --- a/WORKBOARD.yaml +++ b/WORKBOARD.yaml @@ -8,53 +8,53 @@ items: title: CTO Case Candidate Backend PRD status: validated source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-PRD.md - owner: "" + owner: jp - id: CTO-WORK-003 title: Planning Validator Coverage status: validated source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-ISSUES.md - owner: "" + owner: jp - id: CTO-WORK-004 title: Harness Evidence Interface Contract status: validated source: sot/03-PROTOCOLS/CTO-HARNESS-EVIDENCE-INTERFACE-CONTRACT.md - owner: "" + owner: jp - id: CTO-WORK-005 title: Case Source Admission Record status: validated source: sot/03-PROTOCOLS/CTO-CASE-SOURCE-ADMISSION-RECORD.md - owner: "" + owner: jp - id: CTO-WORK-006 title: Case Adapter Contract And Eligibility Decision status: validated source: sot/03-PROTOCOLS/CTO-CASE-ADAPTER-CONTRACT.md - owner: "" + owner: jp - id: CTO-WORK-007 title: Case Failure Fixture Matrix status: validated source: sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md - owner: "" + owner: jp - id: CTO-WORK-008 title: Staged Proof Gate Records status: validated source: sot/03-PROTOCOLS/CTO-CASE-STAGED-PROOF-GATES.md - owner: "" + owner: jp - id: CTO-WORK-009 title: Stage 1 Gated Case Engine PRD status: validated source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-PRD.md - owner: "" + owner: jp - id: CTO-WORK-010 title: Stage 1 Harness Implementation Route status: validated source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-ISSUES.md - owner: "" + owner: jp - id: CTO-WORK-011 title: Stage 2 Artificial Fixture PRD status: validated source: sot/03-PROTOCOLS/CTO-CASE-STAGE2-ARTIFICIAL-FIXTURE-PRD.md - owner: "" + owner: jp - id: CTO-WORK-012 title: Stage 2 Harness Artificial Fixture Route status: blocked @@ -64,17 +64,17 @@ items: title: Case Provider Admission PRD status: validated source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-PRD.md - owner: "" + owner: jp - id: CTO-WORK-014 title: Hermes Case CLI Task Adapter Route status: validated source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-ISSUES.md - owner: "" + owner: jp - id: CTO-WORK-015 title: Durable Case Provider Build PRD status: validated source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md - owner: "" + owner: jp - id: CTO-WORK-016 title: Real Case Provider Stage 2 Run status: blocked @@ -84,4 +84,9 @@ items: title: Case Provider Timeout Fail-Closed Route status: validated source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md - owner: "" + owner: jp + - id: CTO-WORK-018 + title: Case Model Provider Admission Gate + status: validated + source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md + owner: jp diff --git a/sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md b/sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md index c41c65c..4ba450d 100644 --- a/sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md +++ b/sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md @@ -78,3 +78,32 @@ Validation Evidence: - Hermes commit: `d23c492 Fail closed on Case provider timeout`. - Post-merge `harness/evals/health.sh --json` passed. + +## Real Provider Runtime Evidence - 2026-05-31 + +- Real Case Stage 2 run with `/tmp/workos-case/dist/ca` and `CTO_HARNESS_CASE_TIMEOUT_SECONDS=8` produced report `/home/svrnty/.hermes/profiles/cto-planb/harness-runs/20260531T234205Z-r1-string-slugify-1834617/report.json`. +- Case started provider `anthropic` with model `claude-sonnet-4-6`. +- Adapter-generated `backend/case-data/config.json` contained no model registry, so Case used its built-in default model. +- The run timed out before patch application; tests failed because `strings.py` was unchanged. +- This is an unadmitted external model path for CTO harness proof and must be blocked before Case process start. + +## CTO-WORK-018 - Case Model Provider Admission Gate + +Status: validated. + +Harden the Hermes CTO Case adapter so real Case execution cannot silently fall back to its built-in Anthropic default model. + +Acceptance: + +- Real Case execution requires `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL`. +- Missing model admission writes `backend/provider-model-not-admitted.txt`. +- Missing model admission blocks before `case_process_started`. +- Adapter writes `models.default` into isolated `CASE_DATA_DIR/config.json`. +- `validate-case-provider-adapter.py` checks `model_provider_gate_blocks`. +- `harness/evals/health.sh --json` passes after merge. +- This does not mark real Case Stage 2 as passed. + +Validation Evidence: + +- Hermes commit: `4500082 Gate Case execution on admitted model`. +- Post-merge `harness/evals/health.sh --json` passed. diff --git a/sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md b/sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md index 440853f..3481eda 100644 --- a/sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md +++ b/sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md @@ -82,3 +82,12 @@ Stage 2 moves from provider-unavailable blocked status to a real Case pass repor - Timeout marker: `backend/provider-timeout.txt`. - Timeout validator check: `provider_timeout_fail_closed`. - Hermes evidence commit: `d23c492 Fail closed on Case provider timeout`. + +## Current Model Admission Evidence Addendum - 2026-05-31 + +- Real Case defaulted to provider `anthropic` and model `claude-sonnet-4-6` when no model registry was written. +- Runtime report path: `/home/svrnty/.hermes/profiles/cto-planb/harness-runs/20260531T234205Z-r1-string-slugify-1834617/report.json`. +- Required model admission variables: `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL`. +- Model gate marker: `backend/provider-model-not-admitted.txt`. +- Model gate validator check: `model_provider_gate_blocks`. +- Hermes evidence commit: `4500082 Gate Case execution on admitted model`. diff --git a/tools/validate_cto_child.py b/tools/validate_cto_child.py index dfc990b..4c8e535 100644 --- a/tools/validate_cto_child.py +++ b/tools/validate_cto_child.py @@ -334,12 +334,21 @@ REQUIRED_PROVIDER_BUILD_PRD_PHRASES = [ "backend/provider-timeout.txt", "provider_timeout_fail_closed", "d23c492 Fail closed on Case provider timeout", + "anthropic", + "claude-sonnet-4-6", + "20260531T234205Z-r1-string-slugify-1834617/report.json", + "CTO_HARNESS_CASE_MODEL_PROVIDER", + "CTO_HARNESS_CASE_MODEL", + "backend/provider-model-not-admitted.txt", + "model_provider_gate_blocks", + "4500082 Gate Case execution on admitted model", ] REQUIRED_PROVIDER_BUILD_ISSUE_IDS = [ "CTO-WORK-015", "CTO-WORK-016", "CTO-WORK-017", + "CTO-WORK-018", ] @@ -573,6 +582,7 @@ def main() -> int: "CTO-WORK-015": "validated", "CTO-WORK-016": "blocked", "CTO-WORK-017": "validated", + "CTO-WORK-018": "validated", } for issue_id, expected in expected_statuses.items(): checked.append(f"workboard_status:{issue_id}:{expected}")