Record Case model admission gate
This commit is contained in:
parent
3cd2551e24
commit
eefa156734
@ -8,53 +8,53 @@ items:
|
|||||||
title: CTO Case Candidate Backend PRD
|
title: CTO Case Candidate Backend PRD
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-PRD.md
|
source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-PRD.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-003
|
- id: CTO-WORK-003
|
||||||
title: Planning Validator Coverage
|
title: Planning Validator Coverage
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-ISSUES.md
|
source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-ISSUES.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-004
|
- id: CTO-WORK-004
|
||||||
title: Harness Evidence Interface Contract
|
title: Harness Evidence Interface Contract
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-HARNESS-EVIDENCE-INTERFACE-CONTRACT.md
|
source: sot/03-PROTOCOLS/CTO-HARNESS-EVIDENCE-INTERFACE-CONTRACT.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-005
|
- id: CTO-WORK-005
|
||||||
title: Case Source Admission Record
|
title: Case Source Admission Record
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-SOURCE-ADMISSION-RECORD.md
|
source: sot/03-PROTOCOLS/CTO-CASE-SOURCE-ADMISSION-RECORD.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-006
|
- id: CTO-WORK-006
|
||||||
title: Case Adapter Contract And Eligibility Decision
|
title: Case Adapter Contract And Eligibility Decision
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-ADAPTER-CONTRACT.md
|
source: sot/03-PROTOCOLS/CTO-CASE-ADAPTER-CONTRACT.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-007
|
- id: CTO-WORK-007
|
||||||
title: Case Failure Fixture Matrix
|
title: Case Failure Fixture Matrix
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md
|
source: sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-008
|
- id: CTO-WORK-008
|
||||||
title: Staged Proof Gate Records
|
title: Staged Proof Gate Records
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-STAGED-PROOF-GATES.md
|
source: sot/03-PROTOCOLS/CTO-CASE-STAGED-PROOF-GATES.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-009
|
- id: CTO-WORK-009
|
||||||
title: Stage 1 Gated Case Engine PRD
|
title: Stage 1 Gated Case Engine PRD
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-PRD.md
|
source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-PRD.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-010
|
- id: CTO-WORK-010
|
||||||
title: Stage 1 Harness Implementation Route
|
title: Stage 1 Harness Implementation Route
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-ISSUES.md
|
source: sot/03-PROTOCOLS/CTO-CASE-STAGE1-GATED-ENGINE-ISSUES.md
|
||||||
owner: ""
|
owner: jp
|
||||||
|
|
||||||
- id: CTO-WORK-011
|
- id: CTO-WORK-011
|
||||||
title: Stage 2 Artificial Fixture PRD
|
title: Stage 2 Artificial Fixture PRD
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-STAGE2-ARTIFICIAL-FIXTURE-PRD.md
|
source: sot/03-PROTOCOLS/CTO-CASE-STAGE2-ARTIFICIAL-FIXTURE-PRD.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-012
|
- id: CTO-WORK-012
|
||||||
title: Stage 2 Harness Artificial Fixture Route
|
title: Stage 2 Harness Artificial Fixture Route
|
||||||
status: blocked
|
status: blocked
|
||||||
@ -64,17 +64,17 @@ items:
|
|||||||
title: Case Provider Admission PRD
|
title: Case Provider Admission PRD
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-PRD.md
|
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-PRD.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-014
|
- id: CTO-WORK-014
|
||||||
title: Hermes Case CLI Task Adapter Route
|
title: Hermes Case CLI Task Adapter Route
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-ISSUES.md
|
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-ISSUES.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-015
|
- id: CTO-WORK-015
|
||||||
title: Durable Case Provider Build PRD
|
title: Durable Case Provider Build PRD
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md
|
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md
|
||||||
owner: ""
|
owner: jp
|
||||||
- id: CTO-WORK-016
|
- id: CTO-WORK-016
|
||||||
title: Real Case Provider Stage 2 Run
|
title: Real Case Provider Stage 2 Run
|
||||||
status: blocked
|
status: blocked
|
||||||
@ -84,4 +84,9 @@ items:
|
|||||||
title: Case Provider Timeout Fail-Closed Route
|
title: Case Provider Timeout Fail-Closed Route
|
||||||
status: validated
|
status: validated
|
||||||
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md
|
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md
|
||||||
owner: ""
|
owner: jp
|
||||||
|
- id: CTO-WORK-018
|
||||||
|
title: Case Model Provider Admission Gate
|
||||||
|
status: validated
|
||||||
|
source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md
|
||||||
|
owner: jp
|
||||||
|
|||||||
@ -78,3 +78,32 @@ Validation Evidence:
|
|||||||
|
|
||||||
- Hermes commit: `d23c492 Fail closed on Case provider timeout`.
|
- Hermes commit: `d23c492 Fail closed on Case provider timeout`.
|
||||||
- Post-merge `harness/evals/health.sh --json` passed.
|
- Post-merge `harness/evals/health.sh --json` passed.
|
||||||
|
|
||||||
|
## Real Provider Runtime Evidence - 2026-05-31
|
||||||
|
|
||||||
|
- Real Case Stage 2 run with `/tmp/workos-case/dist/ca` and `CTO_HARNESS_CASE_TIMEOUT_SECONDS=8` produced report `/home/svrnty/.hermes/profiles/cto-planb/harness-runs/20260531T234205Z-r1-string-slugify-1834617/report.json`.
|
||||||
|
- Case started provider `anthropic` with model `claude-sonnet-4-6`.
|
||||||
|
- Adapter-generated `backend/case-data/config.json` contained no model registry, so Case used its built-in default model.
|
||||||
|
- The run timed out before patch application; tests failed because `strings.py` was unchanged.
|
||||||
|
- This is an unadmitted external model path for CTO harness proof and must be blocked before Case process start.
|
||||||
|
|
||||||
|
## CTO-WORK-018 - Case Model Provider Admission Gate
|
||||||
|
|
||||||
|
Status: validated.
|
||||||
|
|
||||||
|
Harden the Hermes CTO Case adapter so real Case execution cannot silently fall back to its built-in Anthropic default model.
|
||||||
|
|
||||||
|
Acceptance:
|
||||||
|
|
||||||
|
- Real Case execution requires `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL`.
|
||||||
|
- Missing model admission writes `backend/provider-model-not-admitted.txt`.
|
||||||
|
- Missing model admission blocks before `case_process_started`.
|
||||||
|
- Adapter writes `models.default` into isolated `CASE_DATA_DIR/config.json`.
|
||||||
|
- `validate-case-provider-adapter.py` checks `model_provider_gate_blocks`.
|
||||||
|
- `harness/evals/health.sh --json` passes after merge.
|
||||||
|
- This does not mark real Case Stage 2 as passed.
|
||||||
|
|
||||||
|
Validation Evidence:
|
||||||
|
|
||||||
|
- Hermes commit: `4500082 Gate Case execution on admitted model`.
|
||||||
|
- Post-merge `harness/evals/health.sh --json` passed.
|
||||||
|
|||||||
@ -82,3 +82,12 @@ Stage 2 moves from provider-unavailable blocked status to a real Case pass repor
|
|||||||
- Timeout marker: `backend/provider-timeout.txt`.
|
- Timeout marker: `backend/provider-timeout.txt`.
|
||||||
- Timeout validator check: `provider_timeout_fail_closed`.
|
- Timeout validator check: `provider_timeout_fail_closed`.
|
||||||
- Hermes evidence commit: `d23c492 Fail closed on Case provider timeout`.
|
- Hermes evidence commit: `d23c492 Fail closed on Case provider timeout`.
|
||||||
|
|
||||||
|
## Current Model Admission Evidence Addendum - 2026-05-31
|
||||||
|
|
||||||
|
- Real Case defaulted to provider `anthropic` and model `claude-sonnet-4-6` when no model registry was written.
|
||||||
|
- Runtime report path: `/home/svrnty/.hermes/profiles/cto-planb/harness-runs/20260531T234205Z-r1-string-slugify-1834617/report.json`.
|
||||||
|
- Required model admission variables: `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL`.
|
||||||
|
- Model gate marker: `backend/provider-model-not-admitted.txt`.
|
||||||
|
- Model gate validator check: `model_provider_gate_blocks`.
|
||||||
|
- Hermes evidence commit: `4500082 Gate Case execution on admitted model`.
|
||||||
|
|||||||
@ -334,12 +334,21 @@ REQUIRED_PROVIDER_BUILD_PRD_PHRASES = [
|
|||||||
"backend/provider-timeout.txt",
|
"backend/provider-timeout.txt",
|
||||||
"provider_timeout_fail_closed",
|
"provider_timeout_fail_closed",
|
||||||
"d23c492 Fail closed on Case provider timeout",
|
"d23c492 Fail closed on Case provider timeout",
|
||||||
|
"anthropic",
|
||||||
|
"claude-sonnet-4-6",
|
||||||
|
"20260531T234205Z-r1-string-slugify-1834617/report.json",
|
||||||
|
"CTO_HARNESS_CASE_MODEL_PROVIDER",
|
||||||
|
"CTO_HARNESS_CASE_MODEL",
|
||||||
|
"backend/provider-model-not-admitted.txt",
|
||||||
|
"model_provider_gate_blocks",
|
||||||
|
"4500082 Gate Case execution on admitted model",
|
||||||
]
|
]
|
||||||
|
|
||||||
REQUIRED_PROVIDER_BUILD_ISSUE_IDS = [
|
REQUIRED_PROVIDER_BUILD_ISSUE_IDS = [
|
||||||
"CTO-WORK-015",
|
"CTO-WORK-015",
|
||||||
"CTO-WORK-016",
|
"CTO-WORK-016",
|
||||||
"CTO-WORK-017",
|
"CTO-WORK-017",
|
||||||
|
"CTO-WORK-018",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@ -573,6 +582,7 @@ def main() -> int:
|
|||||||
"CTO-WORK-015": "validated",
|
"CTO-WORK-015": "validated",
|
||||||
"CTO-WORK-016": "blocked",
|
"CTO-WORK-016": "blocked",
|
||||||
"CTO-WORK-017": "validated",
|
"CTO-WORK-017": "validated",
|
||||||
|
"CTO-WORK-018": "validated",
|
||||||
}
|
}
|
||||||
for issue_id, expected in expected_statuses.items():
|
for issue_id, expected in expected_statuses.items():
|
||||||
checked.append(f"workboard_status:{issue_id}:{expected}")
|
checked.append(f"workboard_status:{issue_id}:{expected}")
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user