129 lines
4.5 KiB
Python
129 lines
4.5 KiB
Python
"""Eval suite v1 — one assertion per migrated feature.
|
|
|
|
These run on upstream-sync against new upstream tags. They verify the plugin
|
|
contract still holds after upstream changes. Minimal by design (per protocol
|
|
decision Q3): catch gross breakage, evolve as issues surface.
|
|
"""
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
|
|
def test_eval_loader_contract_unchanged():
|
|
"""The 7-method public API is the protocol contract — adding methods needs a PRD bump."""
|
|
import sys
|
|
sys.path.insert(0, str(ROOT.parent / "hermes-webui"))
|
|
try:
|
|
from api.svrnty_plugin_loader import _PluginAPI
|
|
except ImportError:
|
|
import pytest
|
|
pytest.skip("hermes-webui fork not adjacent; loader contract eval skipped")
|
|
api = _PluginAPI()
|
|
required = {"register_route", "register_static", "inject_script",
|
|
"inject_stylesheet", "config_get", "logger",
|
|
"register_audio_attachment_processor"}
|
|
actual = {m for m in dir(api) if not m.startswith("_")}
|
|
assert required == actual, (
|
|
f"public API drift: expected {required}, got {actual}. "
|
|
f"Adding methods requires a Protocol PRD amendment."
|
|
)
|
|
|
|
|
|
def test_eval_audio_processor_signature_unchanged():
|
|
"""The audio_attachment_processor takes attachments → str. Loader hook + plugin agree."""
|
|
from routes import transcribe
|
|
out = transcribe._transcribe_audio_attachments([])
|
|
assert isinstance(out, str), f"audio processor must return str, got {type(out).__name__}"
|
|
|
|
|
|
def test_eval_vault_status_payload_shape():
|
|
"""Vault status returns {'secrets': [{'name': ...}, ...]} — schema lock."""
|
|
import json
|
|
from unittest.mock import MagicMock, patch
|
|
from routes import vault_status
|
|
|
|
class _H:
|
|
def __init__(self):
|
|
self.body = b""
|
|
self.headers = {}
|
|
|
|
def send_response(self, c): pass
|
|
def send_header(self, k, v): self.headers[k] = v
|
|
def end_headers(self): pass
|
|
|
|
@property
|
|
def wfile(self):
|
|
h = self
|
|
class _W:
|
|
def write(self_, b): h.body += b
|
|
return _W()
|
|
|
|
with patch("routes.vault_status.subprocess.run") as run:
|
|
run.return_value = MagicMock(stdout="a\nb\n", returncode=0)
|
|
h = _H()
|
|
vault_status._handle_vault_status(h, None)
|
|
|
|
payload = json.loads(h.body)
|
|
assert "secrets" in payload
|
|
assert all("name" in s for s in payload["secrets"])
|
|
assert payload["secrets"][0]["name"] == "a"
|
|
|
|
|
|
def test_eval_brand_skin_url_contract():
|
|
"""Brand skin URLs MUST be /plugins/svrnty/<asset> per protocol §14 (Q5)."""
|
|
from unittest.mock import MagicMock
|
|
import plugin
|
|
api = MagicMock()
|
|
api.logger.return_value = MagicMock()
|
|
plugin.register(api)
|
|
api.inject_stylesheet.assert_any_call("/plugins/svrnty/app.css")
|
|
api.inject_script.assert_any_call("/plugins/svrnty/app.js")
|
|
api.inject_stylesheet.assert_any_call("/plugins/svrnty/canvas.css")
|
|
api.inject_script.assert_any_call("/plugins/svrnty/canvas.js")
|
|
|
|
|
|
def test_eval_canvas_tool_surface_contract():
|
|
"""Canvas bridge exposes the first CMO-facing Stitch-like tool contract."""
|
|
import json
|
|
from routes import canvas
|
|
|
|
class _H:
|
|
def __init__(self):
|
|
self.body = b""
|
|
self.headers = {}
|
|
|
|
def send_response(self, c): self.status = c
|
|
def send_header(self, k, v): self.headers[k] = v
|
|
def end_headers(self): pass
|
|
|
|
@property
|
|
def wfile(self):
|
|
h = self
|
|
class _W:
|
|
def write(self_, b): h.body += b
|
|
return _W()
|
|
|
|
h = _H()
|
|
canvas._handle_tools(h, None)
|
|
payload = json.loads(h.body)
|
|
names = {tool["name"] for tool in payload["tools"]}
|
|
assert "canvas.create_project" in names
|
|
assert "canvas.command" in names
|
|
assert "canvas.generate_screen" in names
|
|
assert "canvas.design_context" in names
|
|
assert "canvas.update_screen" in names
|
|
assert "canvas.record_variant" in names
|
|
assert "canvas.connect_prototype_edge" in names
|
|
assert "canvas.export_project" in names
|
|
|
|
|
|
def test_eval_connection_map_has_no_forced_internals():
|
|
"""If forced-internal section grows, audit + amend protocol API (Rule 2)."""
|
|
cm = (ROOT / "CONNECTION-MAP.md").read_text()
|
|
# Look for the "None. Plugin uses only the public API." sentinel.
|
|
assert "Plugin uses only the public API" in cm or "0 forced internal" in cm, (
|
|
"Forced internal dependencies detected — review CONNECTION-MAP.md"
|
|
)
|