77 lines
2.6 KiB
Python
77 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Adversarial test (optional, local):
|
|
|
|
Executes the Monty runner against the socket-deny Tool Request and asserts it fails.
|
|
|
|
Expected:
|
|
- runner exits 0 (it writes a Tool Result)
|
|
- Tool Result exit_code in front matter is non-zero
|
|
- stderr artifact contains "[monty-error]" marker
|
|
|
|
Run this ONLY when:
|
|
- Monty is installed, and
|
|
- (optionally) monty-hardened profile is enabled for defense-in-depth.
|
|
|
|
This test is NOT wired into CI by default.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
from tools.validate_common import extract_front_matter, read_text
|
|
|
|
|
|
def parse_tool_result_md(md_path: Path) -> dict:
|
|
md = read_text(str(md_path))
|
|
fm, _ = extract_front_matter(md)
|
|
return fm
|
|
|
|
|
|
def main() -> int:
|
|
tr = Path("tool-exec/examples/TR-monty-socket-deny.md")
|
|
assert tr.exists(), f"Missing test Tool Request: {tr}"
|
|
|
|
with tempfile.TemporaryDirectory(prefix="threegate-sockettest-") as td:
|
|
results_dir = Path(td) / "results"
|
|
results_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Run the Monty tool runner
|
|
import subprocess
|
|
p = subprocess.run(
|
|
["python3", "tool-exec/monty/run_tool_request.py", "--request", str(tr), "--results-dir", str(results_dir)],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert p.returncode == 0, f"Runner should write a Tool Result; got {p.returncode}\nstdout={p.stdout}\nstderr={p.stderr}"
|
|
|
|
# Find the produced Tool Result markdown
|
|
md_files = sorted(results_dir.glob("TS-*.md"))
|
|
assert md_files, "No Tool Result produced."
|
|
md_path = md_files[-1]
|
|
|
|
fm = parse_tool_result_md(md_path)
|
|
exit_code = int(str(fm.get("exit_code", "0")))
|
|
assert exit_code != 0, f"Expected non-zero tool exit_code for socket attempt; got {exit_code}"
|
|
|
|
# Load stderr artifact and check for marker
|
|
artifacts = fm.get("artifacts", "")
|
|
# The validate_common front matter parser likely returns strings; locate stderr artifact by convention
|
|
# We wrote stderr artifact with name <result_id>.stderr.txt
|
|
result_id = fm.get("result_id")
|
|
assert result_id, "Missing result_id in Tool Result"
|
|
stderr_path = results_dir / f"{result_id}.stderr.txt"
|
|
assert stderr_path.exists(), f"Missing stderr artifact: {stderr_path}"
|
|
stderr_txt = stderr_path.read_text(encoding="utf-8", errors="replace")
|
|
assert "[monty-error]" in stderr_txt, f"Expected monty error marker in stderr; got:\n{stderr_txt}"
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|