diff --git a/README.md b/README.md index 62296fb..e29c810 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,8 @@ This workspace is registered as a child-local planning workspace. Registration d | |-- CTO-CASE-CANDIDATE-BACKEND-ISSUES.md | |-- CTO-HARNESS-EVIDENCE-INTERFACE-CONTRACT.md | |-- CTO-CASE-SOURCE-ADMISSION-RECORD.md -| `-- CTO-CASE-ADAPTER-CONTRACT.md +| |-- CTO-CASE-ADAPTER-CONTRACT.md +| `-- CTO-CASE-FAILURE-FIXTURE-MATRIX.md `-- tools/ `-- validate_cto_child.py ``` diff --git a/WORKBOARD.yaml b/WORKBOARD.yaml index 84852e3..4c2e705 100644 --- a/WORKBOARD.yaml +++ b/WORKBOARD.yaml @@ -31,8 +31,8 @@ items: owner: jp - id: CTO-WORK-007 title: Case Failure Fixture Matrix - status: candidate - source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-ISSUES.md + status: validated + source: sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md owner: jp - id: CTO-WORK-008 title: Staged Proof Gate Records diff --git a/sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md b/sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md new file mode 100644 index 0000000..52cefae --- /dev/null +++ b/sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md @@ -0,0 +1,75 @@ +--- +name: cto-case-failure-fixture-matrix +tier: local +status: draft +owner: jp +source: sot/03-PROTOCOLS/CTO-CASE-CANDIDATE-BACKEND-PRD.md +created: 2026-05-31 +last_reviewed: 2026-05-31 +lifecycle_classification: planning +core_promotion_status: not-promoted +description: Child-local failure fixture matrix for later Case adapter fail-closed testing. +--- + +# CTO Case Failure Fixture Matrix + +Local planning SOT only. Not a Core Protocol. Not active Core authority. + +## Purpose + +Define the required failure fixtures for later Case adapter testing before any executable Case integration exists. + +This matrix turns failure-mode language into concrete expected evidence. It prevents the future adapter from proving only the happy path. + +## Non-Authority Notice + +This matrix does not authorize Runtime behavior, WebUI Product behavior, Case execution, real-repo mutation, merge, deploy, push, close, vendor-source mutation, external developer repository mutation, or Core promotion. + +## Matrix Rules + +- Each row maps to the CTO Harness Evidence Interface. +- Each row must produce `report.json`, `events.normalized.jsonl`, `patch.diff`, `test.log`, and `trace.jsonl` when implemented. +- Each row must produce `artifact_digests` and freshness proof when implemented. +- Each row must record a blocker reason in `report.json`. +- Each row must emit a normalized event when the failure class has one. +- Each row must fail closed with `status` set to `fail` or `blocked`. +- Each row must use nonzero exit behavior when the adapter contract marks the failure executable. +- This matrix is planning-only and does not run Case. + +## Required Failure Fixtures + +| Fixture | Trigger | Expected blocker reason | Normalized event | Report status | Exit behavior | Evidence mapping | +| --- | --- | --- | --- | --- | --- | --- | +| no-diff | Backend reports success but no source diff exists when a diff is required. | `no_diff` | `git.diff.checked` | `fail` | nonzero | `patch.diff` exists and is empty; `changed_files` is empty; `blockers` includes `no_diff`. | +| disallowed-file | Backend changes a path outside `allowed_paths`. | `disallowed_file_change` | `git.diff.checked` | `fail` | nonzero | `changed_files` lists offending path; `allowed_writes_passed` is `false`; `patch.diff` captures the change. | +| failed-tests | Verification command exits nonzero. | `verification_failed` | `verification.completed` | `fail` | nonzero | `test.log` captures failing command output; `verification` records command and nonzero exit. | +| missing-test-command | Task contract requires verification but no command is available. | `missing_test_command` | `verification.completed` | `blocked` | nonzero | `test.log` states no command was available; `verification` records missing command. | +| missing-event | Required normalized event is absent or out of order. | `missing_required_event` | `run.completed` | `fail` | nonzero | `events.normalized.jsonl` is present; validator identifies missing or out-of-order event. | +| reviewer-reject | Case reviewer rejects the result. | `reviewer_rejected` | `verification.completed` | `blocked` | nonzero | backend raw logs record reviewer rejection; normalized evidence records blocked result without approving mutation. | +| approval-denied | Human approval is required and denied. | `approval_denied` | `approval.denied` | `blocked` | nonzero | approval event records actor, scope, mutation mode, timestamp, and denial. | +| timeout | Backend or verification exceeds configured timeout. | `timeout` | `run.completed` | `blocked` | nonzero | `trace.jsonl` records timeout point; `report.json` records timeout blocker. | +| dirty-starting-tree | Target repository is dirty before backend work starts when clean start is required. | `dirty_starting_tree` | `task.contract.created` | `blocked` | nonzero | `trace.jsonl` records preflight failure; no mutation occurs. | +| dirty-ending-tree | Run leaves untracked or unintended dirty state after checks. | `dirty_ending_tree` | `run.completed` | `fail` | nonzero | `trace.jsonl` records ending state check; `report.json` names dirty paths. | +| artifact-write-failure | Required artifact cannot be written. | `artifact_write_failure` | `run.completed` | `fail` | nonzero | `report.json` may be partial only if write failure happens after report creation; trace records failed artifact path. | +| provider-unavailable | Backend provider, model, CLI, or dependency is unavailable. | `provider_unavailable` | `run.completed` | `blocked` | nonzero | backend raw logs record provider failure; no source mutation occurs. | + +## Acceptance Rule + +A future Case adapter cannot progress to staged proof until these fixture classes have expected evidence defined in the adapter validator. + +The first executable implementation may start with a subset only if the staged proof gate explicitly marks the remaining rows as blocked and not accepted for default candidacy. + +## Validator Expectations + +Current validation is planning-only and checks this matrix exists with required rows. + +Later adapter validation must execute or simulate each row and verify: + +- blocker reason; +- normalized event; +- report status; +- exit behavior; +- required artifact presence; +- digest presence; +- freshness proof; +- no hidden source mutation. diff --git a/tools/validate_cto_child.py b/tools/validate_cto_child.py index 6fc4a66..c3af202 100644 --- a/tools/validate_cto_child.py +++ b/tools/validate_cto_child.py @@ -21,6 +21,7 @@ REQUIRED_FILES = [ "sot/03-PROTOCOLS/CTO-HARNESS-EVIDENCE-INTERFACE-CONTRACT.md", "sot/03-PROTOCOLS/CTO-CASE-SOURCE-ADMISSION-RECORD.md", "sot/03-PROTOCOLS/CTO-CASE-ADAPTER-CONTRACT.md", + "sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md", ] REQUIRED_BRIEF_PHRASES = [ @@ -119,6 +120,43 @@ REQUIRED_ADAPTER_CONTRACT_PHRASES = [ "This contract deliberately does not create `case-engine.sh`.", ] +REQUIRED_FAILURE_FIXTURE_PHRASES = [ + "Local planning SOT only. Not a Core Protocol. Not active Core authority.", + "This matrix does not authorize Runtime behavior", + "This matrix is planning-only and does not run Case.", + "no-diff", + "disallowed-file", + "failed-tests", + "missing-test-command", + "missing-event", + "reviewer-reject", + "approval-denied", + "timeout", + "dirty-starting-tree", + "dirty-ending-tree", + "artifact-write-failure", + "provider-unavailable", + "`no_diff`", + "`disallowed_file_change`", + "`verification_failed`", + "`missing_test_command`", + "`missing_required_event`", + "`reviewer_rejected`", + "`approval_denied`", + "`dirty_starting_tree`", + "`dirty_ending_tree`", + "`artifact_write_failure`", + "`provider_unavailable`", + "report.json", + "events.normalized.jsonl", + "patch.diff", + "test.log", + "trace.jsonl", + "artifact_digests", + "freshness proof", + "fail closed", +] + def main() -> int: checked: list[str] = [] @@ -196,6 +234,16 @@ def main() -> int: if phrase not in text: errors.append(f"missing_adapter_contract_phrase:{phrase}") + failure_matrix = ROOT / "sot/03-PROTOCOLS/CTO-CASE-FAILURE-FIXTURE-MATRIX.md" + if failure_matrix.is_file(): + text = failure_matrix.read_text(encoding="utf-8") + if "core_promotion_status: not-promoted" not in text: + errors.append("failure_matrix_missing_not_promoted_frontmatter") + for phrase in REQUIRED_FAILURE_FIXTURE_PHRASES: + checked.append(f"failure_matrix_phrase:{phrase}") + if phrase not in text: + errors.append(f"missing_failure_matrix_phrase:{phrase}") + board = ROOT / "WORKBOARD.yaml" if board.is_file(): text = board.read_text(encoding="utf-8") @@ -211,6 +259,8 @@ def main() -> int: errors.append("workboard_missing_source_admission_record_source") if "CTO-CASE-ADAPTER-CONTRACT.md" not in text: errors.append("workboard_missing_adapter_contract_source") + if "CTO-CASE-FAILURE-FIXTURE-MATRIX.md" not in text: + errors.append("workboard_missing_failure_matrix_source") payload = { "ok": not errors,