Svrnty
133 lines
4.5 KiB
Python
133 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Validate the Visual Evidence contract and pure adapter proof."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
|
|
REQUIRED_FIELDS = [
|
|
"producing_package_id",
|
|
"producing_tool_id",
|
|
"capability_surface",
|
|
"source_reference",
|
|
"provider_mode",
|
|
"retention_disclosure",
|
|
"observed_content_summary",
|
|
"extracted_claims",
|
|
"confidence",
|
|
"caveats",
|
|
"timestamp",
|
|
"validation_status",
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
errors: list[str] = []
|
|
required = [
|
|
"docs/VISUAL-EVIDENCE-CONTRACT.md",
|
|
"candidate-manifests/visual-evidence-contract.json",
|
|
"src/svrnty_vision/visual_evidence.py",
|
|
"tests/test_visual_evidence.py",
|
|
]
|
|
for rel in required:
|
|
if not (ROOT / rel).exists():
|
|
errors.append(f"missing:{rel}")
|
|
|
|
manifest = _read_json(ROOT / "candidate-manifests/visual-evidence-contract.json", errors)
|
|
if manifest:
|
|
_expect(manifest.get("required_fields") == REQUIRED_FIELDS, "required_fields_mismatch", errors)
|
|
proof = manifest.get("first_vertical_proof", {})
|
|
_expect(proof.get("source_tool_id") == "vision.image_analyze", "proof_tool_mismatch", errors)
|
|
_expect(proof.get("live_provider_call_required") is False, "live_provider_required", errors)
|
|
handoff = manifest.get("research_handoff", {})
|
|
_expect(handoff.get("research_may_consume_visual_evidence") is True, "research_handoff_missing", errors)
|
|
_expect(handoff.get("vision_may_write_research_capsules") is False, "vision_writes_capsules", errors)
|
|
_expect(handoff.get("vision_may_perform_research_synthesis") is False, "vision_research_synthesis", errors)
|
|
|
|
_check_snippets(
|
|
"docs/VISUAL-EVIDENCE-CONTRACT.md",
|
|
[
|
|
"`producing_package_id`",
|
|
"Research owns synthesis and capsule writing",
|
|
"does not call a live provider",
|
|
],
|
|
errors,
|
|
)
|
|
_validate_adapter(errors)
|
|
|
|
result = {
|
|
"ok": not errors,
|
|
"validator": "visual-evidence-contract-v1",
|
|
"checked": required,
|
|
"errors": errors,
|
|
"warnings": [],
|
|
}
|
|
print(json.dumps(result, indent=2, sort_keys=True))
|
|
return 0 if result["ok"] else 1
|
|
|
|
|
|
def _validate_adapter(errors: list[str]) -> None:
|
|
try:
|
|
from types import SimpleNamespace
|
|
|
|
from svrnty_vision.visual_evidence import visual_evidence_from_vlm_response
|
|
except Exception as exc: # pragma: no cover - validator import report
|
|
errors.append(f"adapter_import_failed:{type(exc).__name__}:{exc}")
|
|
return
|
|
|
|
response = SimpleNamespace(
|
|
rubric_mode="raw",
|
|
justification="",
|
|
model_id="qwen-test",
|
|
raw_scores_json='{"description":"solid red square","objects":["red square"],"detected_text":[]}',
|
|
)
|
|
evidence = visual_evidence_from_vlm_response(
|
|
response,
|
|
source_reference="fixture://red-square.png",
|
|
provider_mode="sovereign",
|
|
retention_disclosure="synchronous_no_async_persistence",
|
|
timestamp="2026-06-06T00:00:00Z",
|
|
)
|
|
dumped = evidence.model_dump()
|
|
for field in REQUIRED_FIELDS:
|
|
if field not in dumped:
|
|
errors.append(f"adapter_missing_field:{field}")
|
|
_expect(dumped.get("producing_package_id") == "visual-perception-package-candidate", "adapter_package_mismatch", errors)
|
|
_expect(dumped.get("producing_tool_id") == "vision.image_analyze", "adapter_tool_mismatch", errors)
|
|
_expect(dumped.get("observed_content_summary") == "solid red square", "adapter_summary_mismatch", errors)
|
|
_expect("description: solid red square" in dumped.get("extracted_claims", []), "adapter_claim_missing", errors)
|
|
|
|
|
|
def _read_json(path: Path, errors: list[str]) -> dict | None:
|
|
if not path.exists():
|
|
return None
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError as exc:
|
|
errors.append(f"{path.relative_to(ROOT)}:json:{exc}")
|
|
return None
|
|
|
|
|
|
def _check_snippets(rel: str, snippets: list[str], errors: list[str]) -> None:
|
|
path = ROOT / rel
|
|
if not path.exists():
|
|
return
|
|
text = path.read_text(encoding="utf-8")
|
|
for snippet in snippets:
|
|
if snippet not in text:
|
|
errors.append(f"{rel}:missing:{snippet}")
|
|
|
|
|
|
def _expect(condition: bool, error: str, errors: list[str]) -> None:
|
|
if not condition:
|
|
errors.append(error)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|