Record Case model admission gate

This commit is contained in:
Svrnty 2026-05-31 19:45:28 -04:00
parent 3cd2551e24
commit eefa156734
4 changed files with 67 additions and 14 deletions

View File

@ -8,53 +8,53 @@ items:
title: CTO Case Candidate Backend PRD title: CTO Case Candidate Backend PRD
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-PRD.md source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-PRD.md
owner: "" owner: jp
- id: CTO-WORK-003 - id: CTO-WORK-003
title: Planning Validator Coverage title: Planning Validator Coverage
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-ISSUES.md source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-ISSUES.md
owner: "" owner: jp
- id: CTO-WORK-004 - id: CTO-WORK-004
title: Harness Evidence Interface Contract title: Harness Evidence Interface Contract
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-HARNESS-EVIDENCE-INTERFACE-CONTRACT.md source: sot/03-PROTOCOLS/CTO-HARNESS-EVIDENCE-INTERFACE-CONTRACT.md
owner: "" owner: jp
- id: CTO-WORK-005 - id: CTO-WORK-005
title: Case Source Admission Record title: Case Source Admission Record
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-SOURCE-ADMISSION-RECORD.md source: sot/03-PROTOCOLS/CTO-CASE-SOURCE-ADMISSION-RECORD.md
owner: "" owner: jp
- id: CTO-WORK-006 - id: CTO-WORK-006
title: Case Adapter Contract And Eligibility Decision title: Case Adapter Contract And Eligibility Decision
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-ADAPTER-CONTRACT.md source: sot/03-PROTOCOLS/CTO-CASE-ADAPTER-CONTRACT.md
owner: "" owner: jp
- id: CTO-WORK-007 - id: CTO-WORK-007
title: Case Failure Fixture Matrix title: Case Failure Fixture Matrix
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md source: sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md
owner: "" owner: jp
- id: CTO-WORK-008 - id: CTO-WORK-008
title: Staged Proof Gate Records title: Staged Proof Gate Records
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-STAGED-PROOF-GATES.md source: sot/03-PROTOCOLS/CTO-CASE-STAGED-PROOF-GATES.md
owner: "" owner: jp
- id: CTO-WORK-009 - id: CTO-WORK-009
title: Stage 1 Gated Case Engine PRD title: Stage 1 Gated Case Engine PRD
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-PRD.md source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-PRD.md
owner: "" owner: jp
- id: CTO-WORK-010 - id: CTO-WORK-010
title: Stage 1 Harness Implementation Route title: Stage 1 Harness Implementation Route
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-ISSUES.md source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-ISSUES.md
owner: "" owner: jp
- id: CTO-WORK-011 - id: CTO-WORK-011
title: Stage 2 Artificial Fixture PRD title: Stage 2 Artificial Fixture PRD
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-STAGE2-ARTIFICIAL-FIXTURE-PRD.md source: sot/03-PROTOCOLS/CTO-CASE-STAGE2-ARTIFICIAL-FIXTURE-PRD.md
owner: "" owner: jp
- id: CTO-WORK-012 - id: CTO-WORK-012
title: Stage 2 Harness Artificial Fixture Route title: Stage 2 Harness Artificial Fixture Route
status: blocked status: blocked
@ -64,17 +64,17 @@ items:
title: Case Provider Admission PRD title: Case Provider Admission PRD
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-PRD.md source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-PRD.md
owner: "" owner: jp
- id: CTO-WORK-014 - id: CTO-WORK-014
title: Hermes Case CLI Task Adapter Route title: Hermes Case CLI Task Adapter Route
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-ISSUES.md source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-ISSUES.md
owner: "" owner: jp
- id: CTO-WORK-015 - id: CTO-WORK-015
title: Durable Case Provider Build PRD title: Durable Case Provider Build PRD
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md
owner: "" owner: jp
- id: CTO-WORK-016 - id: CTO-WORK-016
title: Real Case Provider Stage 2 Run title: Real Case Provider Stage 2 Run
status: blocked status: blocked
@ -84,4 +84,9 @@ items:
title: Case Provider Timeout Fail-Closed Route title: Case Provider Timeout Fail-Closed Route
status: validated status: validated
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md
owner: "" owner: jp
- id: CTO-WORK-018
title: Case Model Provider Admission Gate
status: validated
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md
owner: jp

View File

@ -78,3 +78,32 @@ Validation Evidence:
- Hermes commit: `d23c492 Fail closed on Case provider timeout`. - Hermes commit: `d23c492 Fail closed on Case provider timeout`.
- Post-merge `harness/evals/health.sh --json` passed. - Post-merge `harness/evals/health.sh --json` passed.
## Real Provider Runtime Evidence - 2026-05-31
- Real Case Stage 2 run with `/tmp/workos-case/dist/ca` and `CTO_HARNESS_CASE_TIMEOUT_SECONDS=8` produced report `/home/svrnty/.hermes/profiles/cto-planb/harness-runs/20260531T234205Z-r1-string-slugify-1834617/report.json`.
- Case started provider `anthropic` with model `claude-sonnet-4-6`.
- Adapter-generated `backend/case-data/config.json` contained no model registry, so Case used its built-in default model.
- The run timed out before patch application; tests failed because `strings.py` was unchanged.
- This is an unadmitted external model path for CTO harness proof and must be blocked before Case process start.
## CTO-WORK-018 - Case Model Provider Admission Gate
Status: validated.
Harden the Hermes CTO Case adapter so real Case execution cannot silently fall back to its built-in Anthropic default model.
Acceptance:
- Real Case execution requires `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL`.
- Missing model admission writes `backend/provider-model-not-admitted.txt`.
- Missing model admission blocks before `case_process_started`.
- Adapter writes `models.default` into isolated `CASE_DATA_DIR/config.json`.
- `validate-case-provider-adapter.py` checks `model_provider_gate_blocks`.
- `harness/evals/health.sh --json` passes after merge.
- This does not mark real Case Stage 2 as passed.
Validation Evidence:
- Hermes commit: `4500082 Gate Case execution on admitted model`.
- Post-merge `harness/evals/health.sh --json` passed.

View File

@ -82,3 +82,12 @@ Stage 2 moves from provider-unavailable blocked status to a real Case pass repor
- Timeout marker: `backend/provider-timeout.txt`. - Timeout marker: `backend/provider-timeout.txt`.
- Timeout validator check: `provider_timeout_fail_closed`. - Timeout validator check: `provider_timeout_fail_closed`.
- Hermes evidence commit: `d23c492 Fail closed on Case provider timeout`. - Hermes evidence commit: `d23c492 Fail closed on Case provider timeout`.
## Current Model Admission Evidence Addendum - 2026-05-31
- Real Case defaulted to provider `anthropic` and model `claude-sonnet-4-6` when no model registry was written.
- Runtime report path: `/home/svrnty/.hermes/profiles/cto-planb/harness-runs/20260531T234205Z-r1-string-slugify-1834617/report.json`.
- Required model admission variables: `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL`.
- Model gate marker: `backend/provider-model-not-admitted.txt`.
- Model gate validator check: `model_provider_gate_blocks`.
- Hermes evidence commit: `4500082 Gate Case execution on admitted model`.

View File

@ -334,12 +334,21 @@ REQUIRED_PROVIDER_BUILD_PRD_PHRASES = [
"backend/provider-timeout.txt", "backend/provider-timeout.txt",
"provider_timeout_fail_closed", "provider_timeout_fail_closed",
"d23c492 Fail closed on Case provider timeout", "d23c492 Fail closed on Case provider timeout",
"anthropic",
"claude-sonnet-4-6",
"20260531T234205Z-r1-string-slugify-1834617/report.json",
"CTO_HARNESS_CASE_MODEL_PROVIDER",
"CTO_HARNESS_CASE_MODEL",
"backend/provider-model-not-admitted.txt",
"model_provider_gate_blocks",
"4500082 Gate Case execution on admitted model",
] ]
REQUIRED_PROVIDER_BUILD_ISSUE_IDS = [ REQUIRED_PROVIDER_BUILD_ISSUE_IDS = [
"CTO-WORK-015", "CTO-WORK-015",
"CTO-WORK-016", "CTO-WORK-016",
"CTO-WORK-017", "CTO-WORK-017",
"CTO-WORK-018",
] ]
@ -573,6 +582,7 @@ def main() -> int:
"CTO-WORK-015": "validated", "CTO-WORK-015": "validated",
"CTO-WORK-016": "blocked", "CTO-WORK-016": "blocked",
"CTO-WORK-017": "validated", "CTO-WORK-017": "validated",
"CTO-WORK-018": "validated",
} }
for issue_id, expected in expected_statuses.items(): for issue_id, expected in expected_statuses.items():
checked.append(f"workboard_status:{issue_id}:{expected}") checked.append(f"workboard_status:{issue_id}:{expected}")