112 lines
2.6 KiB
Python
112 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Validate a Tool Result against schemas/tool-result.schema.md (schema_version=1).
|
|
|
|
Usage:
|
|
validate_tool_result.py /path/to/result.md
|
|
|
|
Exit codes:
|
|
0 = valid
|
|
2 = invalid
|
|
3 = error
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from typing import List
|
|
|
|
from validate_common import (
|
|
ValidationResult,
|
|
extract_front_matter,
|
|
find_forbidden,
|
|
read_text,
|
|
require_keys,
|
|
require_sections_in_order,
|
|
)
|
|
|
|
REQUIRED_KEYS = [
|
|
"result_type",
|
|
"schema_version",
|
|
"result_id",
|
|
"created_utc",
|
|
"request_id",
|
|
"executor",
|
|
"backend",
|
|
"exit_code",
|
|
"runtime_sec",
|
|
"network_used",
|
|
]
|
|
|
|
REQUIRED_H2 = [
|
|
"## Summary",
|
|
"## Provenance",
|
|
"## Outputs",
|
|
"## Stdout",
|
|
"## Stderr",
|
|
"## Safety Notes",
|
|
]
|
|
|
|
|
|
def validate(path: str) -> ValidationResult:
|
|
errors: List[str] = []
|
|
warnings: List[str] = []
|
|
|
|
md = read_text(path)
|
|
fm, body = extract_front_matter(md)
|
|
|
|
missing = require_keys(fm, REQUIRED_KEYS)
|
|
if missing:
|
|
errors.append(f"Missing required front matter keys: {', '.join(missing)}")
|
|
|
|
if fm.get("result_type") != "tool_result":
|
|
errors.append(f"result_type must be 'tool_result' (got: {fm.get('result_type')!r})")
|
|
|
|
if fm.get("schema_version") != "1":
|
|
errors.append(f"schema_version must be '1' (got: {fm.get('schema_version')!r})")
|
|
|
|
errors.extend(require_sections_in_order(body, REQUIRED_H2))
|
|
|
|
# Safety Notes must include explicit untrusted statement
|
|
if "## Safety Notes" in body:
|
|
if "Untrusted Output Statement" not in body:
|
|
errors.append("Safety Notes must include 'Untrusted Output Statement:'")
|
|
if "Network confirmation" not in body:
|
|
errors.append("Safety Notes must include 'Network confirmation:'")
|
|
|
|
# Forbidden content scan (whole document)
|
|
forbidden_hits = find_forbidden(md)
|
|
if forbidden_hits:
|
|
errors.extend(forbidden_hits)
|
|
|
|
return ValidationResult(ok=(len(errors) == 0), errors=errors, warnings=warnings)
|
|
|
|
|
|
def main() -> int:
|
|
if len(sys.argv) != 2:
|
|
print(__doc__.strip(), file=sys.stderr)
|
|
return 3
|
|
path = sys.argv[1]
|
|
try:
|
|
res = validate(path)
|
|
except Exception as e:
|
|
print(f"ERROR: {e}", file=sys.stderr)
|
|
return 3
|
|
|
|
if res.ok:
|
|
for w in res.warnings:
|
|
print(f"WARNING: {w}", file=sys.stderr)
|
|
print("ACCEPT")
|
|
return 0
|
|
else:
|
|
for e in res.errors:
|
|
print(f"ERROR: {e}", file=sys.stderr)
|
|
for w in res.warnings:
|
|
print(f"WARNING: {w}", file=sys.stderr)
|
|
print("REJECT")
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|