diff --git a/README.md b/README.md index 7bd4a42..fab07df 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,9 @@ This workspace is registered as a child-local planning workspace. Registration d | |-- CTO-CASE-PROVIDER-ADMISSION-PRD.md | |-- CTO-CASE-PROVIDER-ADMISSION-ISSUES.md | |-- CTO-CASE-PROVIDER-BUILD-PRD.md -| `-- CTO-CASE-PROVIDER-BUILD-ISSUES.md +| |-- CTO-CASE-PROVIDER-BUILD-ISSUES.md +| |-- CTO-CASE-MODEL-PROVIDER-ADMISSION-PRD.md +| `-- CTO-CASE-MODEL-PROVIDER-ADMISSION-ISSUES.md `-- tools/ `-- validate_cto_child.py ``` diff --git a/WORKBOARD.yaml b/WORKBOARD.yaml index b10c106..faecfd0 100644 --- a/WORKBOARD.yaml +++ b/WORKBOARD.yaml @@ -90,3 +90,13 @@ items: status: validated source: sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md owner: jp + - id: CTO-WORK-019 + title: Case Model Provider Admission PRD + status: validated + source: sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-PRD.md + owner: jp + - id: CTO-WORK-020 + title: Admit Case Model Provider For Real Stage 2 + status: blocked + source: sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-ISSUES.md + owner: jp diff --git a/sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-ISSUES.md b/sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-ISSUES.md new file mode 100644 index 0000000..783bb8b --- /dev/null +++ b/sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-ISSUES.md @@ -0,0 +1,56 @@ +--- +title: CTO Case Model Provider Admission Issues +status: draft +lifecycle_classification: sot +owner: jp +created: 2026-05-31 +last_reviewed: 2026-05-31 +core_promotion_status: not-promoted +route: cto +--- + +# CTO Case Model Provider Admission Issues + +Local planning SOT only. Not a Core Protocol. Not active Core authority. + +## CTO-WORK-019 - Case Model Provider Admission PRD + +Status: validated. + +Extract the existing `CTO-WORK-018` harness gate into a first-class model provider admission route. This is the SOT route for deciding which provider/model pair may power real Case Stage 2. + +Acceptance: + +- Records observed fallback provider `anthropic`. +- Records observed fallback model `claude-sonnet-4-6`. +- Requires explicit admitted provider and exact model ID before real Case starts. +- Requires `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL` to match the admission record. +- Requires `backend/provider-model-not-admitted.txt` when admission is missing. +- Requires isolated `CASE_DATA_DIR/config.json` to contain admitted `models.default`. +- Requires negative gates for missing provider/model and unadmitted provider/model. +- Requires no secrets in task file, argv, report, trace, backend logs, SOT, or commits. +- Keeps Case as candidate execution backend, not CTO authority. + +## CTO-WORK-020 - Admit Case Model Provider For Real Stage 2 + +Status: blocked. + +Choose and admit the exact provider/model path for real Case Stage 2, then rerun Stage 2 through the Harness Evidence Interface. + +Acceptance: + +- Admission record names provider, exact model ID, credential source class, allowed network class, approval source, admission timestamp, review trigger, and evidence expectations. +- No provider/model is admitted by default. +- No secret is written to SOT, argv, task file, backend logs, report, trace, or commit. +- `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL` match the admission record. +- Missing or unadmitted provider/model blocks before `case_process_started`. +- Report records `case_model_provider`, `case_model`, and `case_model_admission_status`. +- Real Case Stage 2 produces a pass report only if the admitted provider/model was used. +- Same-run fake baseline comparison remains required. +- No Target Repository path is inspected or copied. + +Blocked by: + +- Human provider approval if an external provider such as Anthropic is selected. +- A Case-compatible local provider route if external providers are not approved. + diff --git a/sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-PRD.md b/sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-PRD.md new file mode 100644 index 0000000..23b278b --- /dev/null +++ b/sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-PRD.md @@ -0,0 +1,90 @@ +--- +title: CTO Case Model Provider Admission PRD +status: draft +lifecycle_classification: sot +owner: jp +created: 2026-05-31 +last_reviewed: 2026-05-31 +core_promotion_status: not-promoted +route: cto +--- + +# CTO Case Model Provider Admission PRD + +Local planning SOT only. Not a Core Protocol. Not active Core authority. + +## Problem Statement + +`CTO-WORK-018` validated a harness gate that blocks missing model configuration, but the CTO route still needs a first-class admission record for the model provider itself. Evidence showed WorkOS Case silently defaulted to provider `anthropic` and model `claude-sonnet-4-6` when the harness did not write a model registry. That path is an unadmitted external model path for CTO proof. + +## Solution + +Extract the model provider decision into a child-local admission route. The route requires an explicit admitted provider/model pair, redacted credential policy, isolated Case config proof, negative gates, and real Stage 2 retry conditions before any real Case run can claim progress. + +## Scope + +- Admit only one named Case model provider and exact model ID at a time. +- Require admission before `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL` may be used for real Case. +- Preserve fail-closed behavior through `backend/provider-model-not-admitted.txt`. +- Require unadmitted provider/model blocks before `case_process_started`. +- Require the adapter to write admitted `models.default` into isolated `CASE_DATA_DIR/config.json`. +- Require provider evidence in `report.json`, backend logs, `trace.jsonl`, and artifact digests. +- Require secret-redaction evidence for task file, argv, report, trace, and backend logs. +- Keep Stage 2 mutation scope limited to copied artificial fixture only. +- Keep executable admission separate from model provider admission. +- Keep `ca run --task --mode unattended` as the only real Case Stage 2 command shape. +- Preserve same-run fake baseline comparison. + +## Non-Goals + +- Do not approve Anthropic, Claude, local inference, or any other provider by default. +- Do not create a broad provider marketplace or registry abstraction. +- Do not store credentials in SOT, task files, argv, commits, reports, traces, or backend logs. +- Do not grant Case CTO authority. +- Do not authorize copied repo, sandbox repo, owned repo, default backend, WebUI product, or Core promotion behavior. +- Do not bypass the Harness Evidence Interface. +- Do not mutate Case source, Cortex Core, vendor source, or target repositories. + +## Acceptance Criteria + +- A model provider admission record names provider, exact model ID, credential source class, allowed network class, approval source, admission timestamp, and expiry or review trigger. +- Missing provider/model admission blocks before `case_process_started`. +- Unadmitted provider/model blocks before `case_process_started`. +- Missing credentials, unexpected fallback model, missing config write, or absent provider evidence blocks. +- Stage 2 report records `case_model_provider`, `case_model`, `case_model_admission_status`, `case_process_started`, `backend_exit_code`, `allowed_writes_passed`, `changed_files`, and `blockers`. +- Real Case Stage 2 cannot pass unless the report proves the admitted provider/model was used. +- Real Case Stage 2 remains blocked unless a pass report exists. +- Fake remains the default validation lane. +- Same-run fake baseline comparison remains required. +- No secrets appear in task file, argv, report, trace, backend logs, SOT, or commits. + +## Validation + +- `python3 tools/validate_cto_child.py` validates this child-local route. +- Hermes focused validation must include `python3 harness/runner/validate-case-provider-adapter.py --harness-root harness --json`. +- Required negative gates: missing provider/model blocks before `case_process_started`; unadmitted provider/model blocks before `case_process_started`; no secrets appear in task file, argv, report, trace, backend logs. +- Real provider validation must include `CTO_HARNESS_ALLOW_CASE=1 CTO_HARNESS_CASE_STAGE=2 CTO_HARNESS_CASE_BIN= CTO_HARNESS_CASE_MODEL_PROVIDER= CTO_HARNESS_CASE_MODEL= harness/evals/run-case.sh r1-string-slugify --engine case --json`. +- Aggregate validation remains `harness/evals/health.sh --json` after focused gates pass. + +## Risks And Dependencies + +- Human approval may be required before any external provider is admitted. +- Local provider use may require a separate Case-compatible provider adapter or credentials path. +- Case defaults may change; model evidence must be read from actual run artifacts, not assumed from docs. +- Provider credentials may be unavailable in the current terminal. +- License status remains unresolved for broader execution modes. + +## Success Definition + +Real Case Stage 2 remains blocked until a named provider/model is admitted, then passes only when the Harness Evidence Interface proves the admitted provider/model executed the copied artificial fixture without forbidden writes, target inspection, fallback model use, or secret leakage. + +## Current Evidence - 2026-05-31 + +- Existing gate: `CTO-WORK-018 - Case Model Provider Admission Gate`. +- Real Case defaulted to provider `anthropic` and model `claude-sonnet-4-6` without an explicit model registry. +- Runtime report path: `/home/svrnty/.hermes/profiles/cto-planb/harness-runs/20260531T234205Z-r1-string-slugify-1834617/report.json`. +- Hermes model gate commit: `4500082 Gate Case execution on admitted model`. +- Model gate variables: `CTO_HARNESS_CASE_MODEL_PROVIDER` and `CTO_HARNESS_CASE_MODEL`. +- Model gate marker: `backend/provider-model-not-admitted.txt`. +- Validator check: `model_provider_gate_blocks`. + diff --git a/tools/validate_cto_child.py b/tools/validate_cto_child.py index 4c8e535..e995ab4 100644 --- a/tools/validate_cto_child.py +++ b/tools/validate_cto_child.py @@ -32,6 +32,8 @@ REQUIRED_FILES = [ "sot/03-PROTOCOLS/CTO-CASE-PROVIDER-ADMISSION-ISSUES.md", "sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-PRD.md", "sot/03-PROTOCOLS/CTO-CASE-PROVIDER-BUILD-ISSUES.md", + "sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-PRD.md", + "sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-ISSUES.md", ] REQUIRED_BRIEF_PHRASES = [ @@ -351,6 +353,42 @@ REQUIRED_PROVIDER_BUILD_ISSUE_IDS = [ "CTO-WORK-018", ] +REQUIRED_MODEL_PROVIDER_ADMISSION_PRD_PHRASES = [ + "Local planning SOT only. Not a Core Protocol. Not active Core authority.", + "CTO-WORK-018", + "anthropic", + "claude-sonnet-4-6", + "unadmitted external model path", + "explicit admitted provider/model pair", + "credential source class", + "allowed network class", + "admission timestamp", + "CTO_HARNESS_CASE_MODEL_PROVIDER", + "CTO_HARNESS_CASE_MODEL", + "backend/provider-model-not-admitted.txt", + "unadmitted provider/model blocks before `case_process_started`", + "models.default", + "CASE_DATA_DIR/config.json", + "ca run --task --mode unattended", + "case_model_provider", + "case_model", + "case_model_admission_status", + "case_process_started", + "allowed_writes_passed", + "changed_files", + "blockers", + "Fake remains the default validation lane.", + "Same-run fake baseline comparison remains required.", + "No secrets appear in task file, argv, report, trace, backend logs, SOT, or commits.", + "4500082 Gate Case execution on admitted model", + "model_provider_gate_blocks", +] + +REQUIRED_MODEL_PROVIDER_ADMISSION_ISSUE_IDS = [ + "CTO-WORK-019", + "CTO-WORK-020", +] + def workboard_status(text: str, issue_id: str) -> str | None: pattern = rf"- id: {re.escape(issue_id)}\n(?: .+\n)*? status: ([^\n]+)" @@ -542,6 +580,28 @@ def main() -> int: if issue_id not in text: errors.append(f"missing_provider_build_issue_id:{issue_id}") + model_provider_admission_prd = ROOT / "sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-PRD.md" + if model_provider_admission_prd.is_file(): + text = model_provider_admission_prd.read_text(encoding="utf-8") + if "core_promotion_status: not-promoted" not in text: + errors.append("model_provider_admission_prd_missing_not_promoted_frontmatter") + for phrase in REQUIRED_MODEL_PROVIDER_ADMISSION_PRD_PHRASES: + checked.append(f"model_provider_admission_prd_phrase:{phrase}") + if phrase not in text: + errors.append(f"missing_model_provider_admission_prd_phrase:{phrase}") + + model_provider_admission_issues = ROOT / "sot/03-PROTOCOLS/CTO-CASE-MODEL-PROVIDER-ADMISSION-ISSUES.md" + if model_provider_admission_issues.is_file(): + text = model_provider_admission_issues.read_text(encoding="utf-8") + if "core_promotion_status: not-promoted" not in text: + errors.append("model_provider_admission_issues_missing_not_promoted_frontmatter") + if "Local planning SOT only. Not a Core Protocol. Not active Core authority." not in text: + errors.append("model_provider_admission_issues_missing_local_planning_notice") + for issue_id in REQUIRED_MODEL_PROVIDER_ADMISSION_ISSUE_IDS: + checked.append(f"model_provider_admission_issue_id:{issue_id}") + if issue_id not in text: + errors.append(f"missing_model_provider_admission_issue_id:{issue_id}") + board = ROOT / "WORKBOARD.yaml" if board.is_file(): text = board.read_text(encoding="utf-8") @@ -565,6 +625,10 @@ def main() -> int: checked.append(f"workboard_id:{issue_id}") if issue_id not in text: errors.append(f"missing_workboard_id:{issue_id}") + for issue_id in REQUIRED_MODEL_PROVIDER_ADMISSION_ISSUE_IDS: + checked.append(f"workboard_id:{issue_id}") + if issue_id not in text: + errors.append(f"missing_workboard_id:{issue_id}") expected_statuses = { "CTO-WORK-002": "validated", "CTO-WORK-003": "validated", @@ -583,6 +647,8 @@ def main() -> int: "CTO-WORK-016": "blocked", "CTO-WORK-017": "validated", "CTO-WORK-018": "validated", + "CTO-WORK-019": "validated", + "CTO-WORK-020": "blocked", } for issue_id, expected in expected_statuses.items(): checked.append(f"workboard_status:{issue_id}:{expected}") @@ -617,6 +683,10 @@ def main() -> int: errors.append("workboard_missing_provider_build_prd_source") if "CTO-CASE-PROVIDER-BUILD-ISSUES.md" not in text: errors.append("workboard_missing_provider_build_issues_source") + if "CTO-CASE-MODEL-PROVIDER-ADMISSION-PRD.md" not in text: + errors.append("workboard_missing_model_provider_admission_prd_source") + if "CTO-CASE-MODEL-PROVIDER-ADMISSION-ISSUES.md" not in text: + errors.append("workboard_missing_model_provider_admission_issues_source") payload = { "ok": not errors,