Apply ZIP update: 115-didactopus-evaluator-alignment-update.zip [2026-03-14T13:19:02]
This commit is contained in:
parent
6428cfb869
commit
c9eb0b28c4
|
|
@ -2,20 +2,10 @@ concepts:
|
||||||
- id: c1
|
- id: c1
|
||||||
title: Foundations
|
title: Foundations
|
||||||
description: Broad foundations topic with many ideas.
|
description: Broad foundations topic with many ideas.
|
||||||
prerequisites: []
|
mastery_signals:
|
||||||
|
- Explain core foundations.
|
||||||
- id: c2
|
- id: c2
|
||||||
title: Methods
|
title: Methods
|
||||||
description: Methods concept.
|
description: Methods concept with sparse explicit assessment.
|
||||||
prerequisites: [c1]
|
mastery_signals:
|
||||||
- id: c3
|
- Use methods appropriately.
|
||||||
title: Advanced Inference
|
|
||||||
description: Advanced inference topic.
|
|
||||||
prerequisites: [c1, c2]
|
|
||||||
- id: c4
|
|
||||||
title: Detached Topic
|
|
||||||
description: Detached topic with no assessment coverage.
|
|
||||||
prerequisites: []
|
|
||||||
- id: c5
|
|
||||||
title: Capstone Topic
|
|
||||||
description: Final synthesis topic.
|
|
||||||
prerequisites: [c3]
|
|
||||||
|
|
|
||||||
|
|
@ -3,3 +3,4 @@ dimensions:
|
||||||
description: visual polish and typesetting
|
description: visual polish and typesetting
|
||||||
evidence_types:
|
evidence_types:
|
||||||
- page layout
|
- page layout
|
||||||
|
- typography sample
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
projects:
|
projects:
|
||||||
- id: early-capstone
|
- id: p1
|
||||||
title: Capstone Final Project
|
title: Final Memo
|
||||||
prerequisites: [c1]
|
prerequisites: [c1]
|
||||||
deliverables:
|
deliverables:
|
||||||
- short memo
|
- brief memo
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,5 @@
|
||||||
stages:
|
stages:
|
||||||
- id: stage-1
|
- id: stage-1
|
||||||
title: Start
|
title: Start
|
||||||
concepts: [c1, c2, c3]
|
concepts: [c1, c2]
|
||||||
checkpoint: []
|
|
||||||
- id: stage-2
|
|
||||||
title: Tiny Bridge
|
|
||||||
concepts: [c4]
|
|
||||||
checkpoint: []
|
|
||||||
- id: stage-3
|
|
||||||
title: Ending
|
|
||||||
concepts: [c5]
|
|
||||||
checkpoint: []
|
checkpoint: []
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
rubrics:
|
rubrics:
|
||||||
- id: r1
|
- id: r1
|
||||||
title: Basic
|
title: Basic
|
||||||
criteria: [correctness]
|
criteria: [style, formatting]
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,10 @@ concepts:
|
||||||
- id: bayes-prior
|
- id: bayes-prior
|
||||||
title: Bayes Prior
|
title: Bayes Prior
|
||||||
description: Prior beliefs before evidence in a probabilistic model.
|
description: Prior beliefs before evidence in a probabilistic model.
|
||||||
prerequisites: []
|
mastery_signals:
|
||||||
|
- Explain a prior distribution clearly.
|
||||||
- id: bayes-posterior
|
- id: bayes-posterior
|
||||||
title: Bayes Posterior
|
title: Bayes Posterior
|
||||||
description: Updated beliefs after evidence in a probabilistic model.
|
description: Updated beliefs after evidence in a probabilistic model.
|
||||||
prerequisites: [bayes-prior]
|
mastery_signals:
|
||||||
- id: model-checking
|
- Compare prior and posterior beliefs.
|
||||||
title: Model Checking
|
|
||||||
description: Evaluate whether model assumptions and fit remain plausible.
|
|
||||||
prerequisites: [bayes-posterior]
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,10 @@
|
||||||
dimensions:
|
dimensions:
|
||||||
|
- name: correctness
|
||||||
|
description: factual and inferential correctness
|
||||||
- name: explanation
|
- name: explanation
|
||||||
description: quality of explanation
|
description: quality of explanation and comparison
|
||||||
- name: comparison
|
- name: critique
|
||||||
description: quality of comparison
|
description: quality of critical assessment
|
||||||
evidence_types:
|
evidence_types:
|
||||||
- explanation
|
- explanation
|
||||||
- comparison report
|
- critique report
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
projects:
|
projects:
|
||||||
- id: culminating-analysis
|
- id: p1
|
||||||
title: Final Model Critique
|
title: Final Bayesian Comparison
|
||||||
prerequisites: [bayes-prior, bayes-posterior, model-checking]
|
prerequisites: [bayes-prior, bayes-posterior]
|
||||||
deliverables:
|
deliverables:
|
||||||
- short report
|
- explanation of prior and posterior updates
|
||||||
|
- critique report
|
||||||
|
|
|
||||||
|
|
@ -9,8 +9,3 @@ stages:
|
||||||
concepts: [bayes-posterior]
|
concepts: [bayes-posterior]
|
||||||
checkpoint:
|
checkpoint:
|
||||||
- Compare prior and posterior beliefs.
|
- Compare prior and posterior beliefs.
|
||||||
- id: stage-3
|
|
||||||
title: Model Checking
|
|
||||||
concepts: [model-checking]
|
|
||||||
checkpoint:
|
|
||||||
- Critique a model fit.
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
rubrics:
|
rubrics:
|
||||||
- id: r1
|
- id: r1
|
||||||
title: Basic
|
title: Basic
|
||||||
criteria: [correctness, explanation]
|
criteria: [correctness, explanation, critique]
|
||||||
|
|
|
||||||
|
|
@ -5,18 +5,8 @@ build-backend = "setuptools.build_meta"
|
||||||
[project]
|
[project]
|
||||||
name = "didactopus"
|
name = "didactopus"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = "Didactopus: curriculum path quality analysis"
|
|
||||||
readme = "README.md"
|
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
license = {text = "MIT"}
|
|
||||||
authors = [{name = "Wesley R. Elsberry"}]
|
|
||||||
dependencies = ["pydantic>=2.7", "pyyaml>=6.0"]
|
dependencies = ["pydantic>=2.7", "pyyaml>=6.0"]
|
||||||
|
|
||||||
[project.optional-dependencies]
|
|
||||||
dev = ["pytest>=8.0", "ruff>=0.6"]
|
|
||||||
|
|
||||||
[project.scripts]
|
|
||||||
didactopus-review-bridge = "didactopus.review_bridge_server:main"
|
|
||||||
|
|
||||||
[tool.setuptools.packages.find]
|
[tool.setuptools.packages.find]
|
||||||
where = ["src"]
|
where = ["src"]
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
__version__ = "0.1.0"
|
__version__ = '0.1.0'
|
||||||
|
|
|
||||||
|
|
@ -1,2 +1,47 @@
|
||||||
|
import re
|
||||||
|
from .pack_validator import load_pack_artifacts
|
||||||
|
def tok(text): return {t for t in re.sub(r"[^a-z0-9]+"," ",str(text).lower()).split() if t}
|
||||||
def evaluator_alignment_for_pack(source_dir):
|
def evaluator_alignment_for_pack(source_dir):
|
||||||
return {'warnings': [], 'summary': {'evaluator_warning_count': 0}}
|
loaded=load_pack_artifacts(source_dir)
|
||||||
|
if not loaded["ok"]: return {"warnings":[],"summary":{"evaluator_warning_count":0}}
|
||||||
|
arts=loaded["artifacts"]
|
||||||
|
concepts=arts["concepts"].get("concepts",[]) or []
|
||||||
|
roadmap=arts["roadmap"].get("stages",[]) or []
|
||||||
|
projects=arts["projects"].get("projects",[]) or []
|
||||||
|
rubrics=arts["rubrics"].get("rubrics",[]) or []
|
||||||
|
evaluator=arts["evaluator"] or {}
|
||||||
|
dims=evaluator.get("dimensions",[]) or []
|
||||||
|
evidence=evaluator.get("evidence_types",[]) or []
|
||||||
|
checkpoint_tokens=tok(" ".join(str(i) for s in roadmap for i in (s.get("checkpoint",[]) or [])))
|
||||||
|
deliverable_tokens=tok(" ".join(str(i) for p in projects for i in (p.get("deliverables",[]) or [])))
|
||||||
|
rubric_tokens=set()
|
||||||
|
for r in rubrics:
|
||||||
|
for c in (r.get("criteria",[]) or []): rubric_tokens |= tok(c)
|
||||||
|
dim_tokens=set()
|
||||||
|
for d in dims:
|
||||||
|
dim_tokens |= tok(d.get("name","")) | tok(d.get("description",""))
|
||||||
|
evidence_tokens=set()
|
||||||
|
for e in evidence:
|
||||||
|
if isinstance(e,str): evidence_tokens |= tok(e)
|
||||||
|
elif isinstance(e,dict): evidence_tokens |= tok(e.get("name","")) | tok(e.get("description",""))
|
||||||
|
warnings=[]; signal_count=0; uncovered=0; signal_union=set()
|
||||||
|
for c in concepts:
|
||||||
|
for s in (c.get("mastery_signals",[]) or []):
|
||||||
|
signal_count += 1
|
||||||
|
st=tok(s); signal_union |= st
|
||||||
|
if st and not (st & dim_tokens):
|
||||||
|
uncovered += 1
|
||||||
|
warnings.append(f"Mastery signal for concept '{c.get('id')}' has no visible evaluator-dimension coverage.")
|
||||||
|
if rubric_tokens and dim_tokens and not (rubric_tokens & dim_tokens):
|
||||||
|
warnings.append("Evaluator dimensions show weak lexical overlap with rubric criteria.")
|
||||||
|
warnings.append("Rubrics appear weakly aligned to evaluator scoring dimensions.")
|
||||||
|
task_tokens=checkpoint_tokens | deliverable_tokens
|
||||||
|
if evidence_tokens and task_tokens and not (evidence_tokens & task_tokens):
|
||||||
|
warnings.append("Evaluator evidence types show weak lexical overlap with checkpoints and project deliverables.")
|
||||||
|
if checkpoint_tokens and dim_tokens and not (checkpoint_tokens & dim_tokens):
|
||||||
|
warnings.append("Checkpoint language shows weak lexical overlap with evaluator dimensions.")
|
||||||
|
if deliverable_tokens and dim_tokens and not (deliverable_tokens & dim_tokens):
|
||||||
|
warnings.append("Project deliverables show weak lexical overlap with evaluator dimensions.")
|
||||||
|
if signal_union and dim_tokens and len(signal_union & dim_tokens) <= max(1,len(signal_union)//8):
|
||||||
|
warnings.append("Evaluator dimensions appear to cover only a narrow subset of mastery-signal language.")
|
||||||
|
return {"warnings":warnings,"summary":{"evaluator_warning_count":len(warnings),"dimension_count":len(dims),"evidence_type_count":len(evidence),"mastery_signal_count":signal_count,"uncovered_mastery_signal_count":uncovered}}
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,8 @@
|
||||||
from __future__ import annotations
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from .review_schema import ImportPreview
|
from .review_schema import ImportPreview
|
||||||
from .pack_validator import validate_pack_directory
|
from .pack_validator import validate_pack_directory
|
||||||
from .semantic_qa import semantic_qa_for_pack
|
from .evaluator_alignment_qa import evaluator_alignment_for_pack
|
||||||
from .graph_qa import graph_qa_for_pack
|
def preview_draft_pack_import(source_dir, workspace_id, overwrite_required=False):
|
||||||
from .path_quality_qa import path_quality_for_pack
|
|
||||||
|
|
||||||
def preview_draft_pack_import(source_dir: str | Path, workspace_id: str, overwrite_required: bool = False) -> ImportPreview:
|
|
||||||
result=validate_pack_directory(source_dir)
|
result=validate_pack_directory(source_dir)
|
||||||
semantic = semantic_qa_for_pack(source_dir) if result["ok"] else {"warnings": []}
|
evaluator=evaluator_alignment_for_pack(source_dir) if result["ok"] else {"warnings":[]}
|
||||||
graph = graph_qa_for_pack(source_dir) if result["ok"] else {"warnings": []}
|
return ImportPreview(source_dir=str(Path(source_dir)),workspace_id=workspace_id,overwrite_required=overwrite_required,ok=result["ok"],errors=list(result["errors"]),warnings=list(result["warnings"]),summary=dict(result["summary"]),evaluator_warnings=list(evaluator["warnings"]))
|
||||||
pathq = path_quality_for_pack(source_dir) if result["ok"] else {"warnings": []}
|
|
||||||
return ImportPreview(
|
|
||||||
source_dir=str(Path(source_dir)),
|
|
||||||
workspace_id=workspace_id,
|
|
||||||
overwrite_required=overwrite_required,
|
|
||||||
ok=result["ok"],
|
|
||||||
errors=list(result["errors"]),
|
|
||||||
warnings=list(result["warnings"]),
|
|
||||||
summary=dict(result["summary"]),
|
|
||||||
semantic_warnings=list(semantic["warnings"]),
|
|
||||||
graph_warnings=list(graph["warnings"]),
|
|
||||||
path_warnings=list(pathq["warnings"]),
|
|
||||||
)
|
|
||||||
|
|
|
||||||
|
|
@ -1,100 +1,22 @@
|
||||||
from __future__ import annotations
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import yaml
|
import yaml
|
||||||
|
REQUIRED_FILES=["pack.yaml","concepts.yaml","roadmap.yaml","projects.yaml","rubrics.yaml","evaluator.yaml"]
|
||||||
REQUIRED_FILES = ["pack.yaml", "concepts.yaml", "roadmap.yaml", "projects.yaml", "rubrics.yaml"]
|
def _load(path, errors, label):
|
||||||
|
try: return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||||
def _safe_load_yaml(path: Path, errors: list[str], label: str):
|
|
||||||
try:
|
|
||||||
return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
errors.append(f"Could not parse {label}: {exc}")
|
errors.append(f"Could not parse {label}: {exc}"); return {}
|
||||||
return {}
|
def load_pack_artifacts(source_dir):
|
||||||
|
source=Path(source_dir); errors=[]
|
||||||
def load_pack_artifacts(source_dir: str | Path) -> dict:
|
if not source.exists(): return {"ok":False,"errors":[f"Source directory does not exist: {source}"],"artifacts":{}}
|
||||||
source = Path(source_dir)
|
if not source.is_dir(): return {"ok":False,"errors":[f"Source path is not a directory: {source}"],"artifacts":{}}
|
||||||
errors: list[str] = []
|
for fn in REQUIRED_FILES:
|
||||||
if not source.exists():
|
if not (source/fn).exists(): errors.append(f"Missing required file: {fn}")
|
||||||
return {"ok": False, "errors": [f"Source directory does not exist: {source}"], "warnings": [], "summary": {}, "artifacts": {}}
|
if errors: return {"ok":False,"errors":errors,"artifacts":{}}
|
||||||
if not source.is_dir():
|
arts={k:_load(source/f"{k}.yaml", errors, f"{k}.yaml") for k in ["pack","concepts","roadmap","projects","rubrics","evaluator"]}
|
||||||
return {"ok": False, "errors": [f"Source path is not a directory: {source}"], "warnings": [], "summary": {}, "artifacts": {}}
|
return {"ok":len(errors)==0,"errors":errors,"artifacts":arts}
|
||||||
for filename in REQUIRED_FILES:
|
def validate_pack_directory(source_dir):
|
||||||
if not (source / filename).exists():
|
|
||||||
errors.append(f"Missing required file: {filename}")
|
|
||||||
if errors:
|
|
||||||
return {"ok": False, "errors": errors, "warnings": [], "summary": {}, "artifacts": {}}
|
|
||||||
pack_data = _safe_load_yaml(source / "pack.yaml", errors, "pack.yaml")
|
|
||||||
concepts_data = _safe_load_yaml(source / "concepts.yaml", errors, "concepts.yaml")
|
|
||||||
roadmap_data = _safe_load_yaml(source / "roadmap.yaml", errors, "roadmap.yaml")
|
|
||||||
projects_data = _safe_load_yaml(source / "projects.yaml", errors, "projects.yaml")
|
|
||||||
rubrics_data = _safe_load_yaml(source / "rubrics.yaml", errors, "rubrics.yaml")
|
|
||||||
return {"ok": len(errors) == 0, "errors": errors, "warnings": [], "summary": {}, "artifacts": {"pack": pack_data, "concepts": concepts_data, "roadmap": roadmap_data, "projects": projects_data, "rubrics": rubrics_data}}
|
|
||||||
|
|
||||||
def validate_pack_directory(source_dir: str | Path) -> dict:
|
|
||||||
loaded=load_pack_artifacts(source_dir)
|
loaded=load_pack_artifacts(source_dir)
|
||||||
errors = list(loaded["errors"])
|
if not loaded["ok"]: return {"ok":False,"errors":loaded["errors"],"warnings":[],"summary":{}}
|
||||||
warnings = list(loaded["warnings"])
|
arts=loaded["artifacts"]; concepts=arts["concepts"].get("concepts",[]) or []
|
||||||
summary = dict(loaded["summary"])
|
summary={"pack_name":arts["pack"].get("name",""),"display_name":arts["pack"].get("display_name",""),"version":arts["pack"].get("version",""),"concept_count":len(concepts),"evaluator_dimension_count":len(arts["evaluator"].get("dimensions",[]) or [])}
|
||||||
if not loaded["ok"]:
|
return {"ok":True,"errors":[],"warnings":[],"summary":summary}
|
||||||
return {"ok": False, "errors": errors, "warnings": warnings, "summary": summary}
|
|
||||||
pack_data = loaded["artifacts"]["pack"]
|
|
||||||
concepts_data = loaded["artifacts"]["concepts"]
|
|
||||||
roadmap_data = loaded["artifacts"]["roadmap"]
|
|
||||||
projects_data = loaded["artifacts"]["projects"]
|
|
||||||
rubrics_data = loaded["artifacts"]["rubrics"]
|
|
||||||
for field in ["name", "display_name", "version"]:
|
|
||||||
if field not in pack_data:
|
|
||||||
warnings.append(f"pack.yaml has no '{field}' field.")
|
|
||||||
concepts = concepts_data.get("concepts", [])
|
|
||||||
roadmap_stages = roadmap_data.get("stages", [])
|
|
||||||
projects = projects_data.get("projects", [])
|
|
||||||
rubrics = rubrics_data.get("rubrics", [])
|
|
||||||
if not isinstance(concepts, list):
|
|
||||||
errors.append("concepts.yaml top-level 'concepts' is not a list."); concepts = []
|
|
||||||
if not isinstance(roadmap_stages, list):
|
|
||||||
errors.append("roadmap.yaml top-level 'stages' is not a list."); roadmap_stages = []
|
|
||||||
if not isinstance(projects, list):
|
|
||||||
errors.append("projects.yaml top-level 'projects' is not a list."); projects = []
|
|
||||||
if not isinstance(rubrics, list):
|
|
||||||
errors.append("rubrics.yaml top-level 'rubrics' is not a list."); rubrics = []
|
|
||||||
concept_ids = []
|
|
||||||
for idx, concept in enumerate(concepts):
|
|
||||||
cid = concept.get("id", "")
|
|
||||||
if not cid:
|
|
||||||
errors.append(f"Concept at index {idx} has no id.")
|
|
||||||
else:
|
|
||||||
concept_ids.append(cid)
|
|
||||||
if not concept.get("title"):
|
|
||||||
warnings.append(f"Concept '{cid or idx}' has no title.")
|
|
||||||
desc = str(concept.get("description", "") or "")
|
|
||||||
if len(desc.strip()) < 12:
|
|
||||||
warnings.append(f"Concept '{cid or idx}' has a very thin description.")
|
|
||||||
seen = set(); dups = set()
|
|
||||||
for cid in concept_ids:
|
|
||||||
if cid in seen: dups.add(cid)
|
|
||||||
seen.add(cid)
|
|
||||||
for cid in sorted(dups):
|
|
||||||
errors.append(f"Duplicate concept id: {cid}")
|
|
||||||
concept_id_set = set(concept_ids)
|
|
||||||
for stage in roadmap_stages:
|
|
||||||
for cid in stage.get("concepts", []) or []:
|
|
||||||
if cid not in concept_id_set:
|
|
||||||
errors.append(f"roadmap.yaml references missing concept id: {cid}")
|
|
||||||
for project in projects:
|
|
||||||
if not project.get("id"):
|
|
||||||
warnings.append("A project entry has no id.")
|
|
||||||
for cid in project.get("prerequisites", []) or []:
|
|
||||||
if cid not in concept_id_set:
|
|
||||||
errors.append(f"projects.yaml references missing prerequisite concept id: {cid}")
|
|
||||||
for idx, rubric in enumerate(rubrics):
|
|
||||||
if not rubric.get("id"):
|
|
||||||
warnings.append(f"Rubric at index {idx} has no id.")
|
|
||||||
criteria = rubric.get("criteria", [])
|
|
||||||
if criteria is None:
|
|
||||||
warnings.append(f"Rubric '{rubric.get('id', idx)}' has null criteria.")
|
|
||||||
elif isinstance(criteria, list) and len(criteria) == 0:
|
|
||||||
warnings.append(f"Rubric '{rubric.get('id', idx)}' has empty criteria.")
|
|
||||||
elif not isinstance(criteria, list):
|
|
||||||
errors.append(f"Rubric '{rubric.get('id', idx)}' criteria is not a list.")
|
|
||||||
summary = {"pack_name": pack_data.get("name", ""), "display_name": pack_data.get("display_name", ""), "version": pack_data.get("version", ""), "concept_count": len(concepts), "roadmap_stage_count": len(roadmap_stages), "project_count": len(projects), "rubric_count": len(rubrics), "error_count": len(errors), "warning_count": len(warnings)}
|
|
||||||
return {"ok": len(errors) == 0, "errors": errors, "warnings": warnings, "summary": summary}
|
|
||||||
|
|
|
||||||
|
|
@ -1,52 +1,9 @@
|
||||||
from __future__ import annotations
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
TrustStatus = Literal["trusted", "provisional", "rejected", "needs_review"]
|
|
||||||
|
|
||||||
class ConceptReviewEntry(BaseModel):
|
|
||||||
concept_id: str
|
|
||||||
title: str
|
|
||||||
description: str = ""
|
|
||||||
prerequisites: list[str] = Field(default_factory=list)
|
|
||||||
mastery_signals: list[str] = Field(default_factory=list)
|
|
||||||
status: TrustStatus = "needs_review"
|
|
||||||
notes: list[str] = Field(default_factory=list)
|
|
||||||
|
|
||||||
class DraftPackData(BaseModel):
|
|
||||||
pack: dict = Field(default_factory=dict)
|
|
||||||
concepts: list[ConceptReviewEntry] = Field(default_factory=list)
|
|
||||||
conflicts: list[str] = Field(default_factory=list)
|
|
||||||
review_flags: list[str] = Field(default_factory=list)
|
|
||||||
attribution: dict = Field(default_factory=dict)
|
|
||||||
|
|
||||||
class ReviewAction(BaseModel):
|
|
||||||
action_type: str
|
|
||||||
target: str = ""
|
|
||||||
payload: dict = Field(default_factory=dict)
|
|
||||||
rationale: str = ""
|
|
||||||
|
|
||||||
class ReviewLedgerEntry(BaseModel):
|
|
||||||
reviewer: str
|
|
||||||
action: ReviewAction
|
|
||||||
|
|
||||||
class ReviewSession(BaseModel):
|
|
||||||
reviewer: str
|
|
||||||
draft_pack: DraftPackData
|
|
||||||
ledger: list[ReviewLedgerEntry] = Field(default_factory=list)
|
|
||||||
|
|
||||||
class WorkspaceMeta(BaseModel):
|
class WorkspaceMeta(BaseModel):
|
||||||
workspace_id: str
|
workspace_id:str; title:str; path:str; created_at:str; last_opened_at:str; notes:str=""
|
||||||
title: str
|
|
||||||
path: str
|
|
||||||
created_at: str
|
|
||||||
last_opened_at: str
|
|
||||||
notes: str = ""
|
|
||||||
|
|
||||||
class WorkspaceRegistry(BaseModel):
|
class WorkspaceRegistry(BaseModel):
|
||||||
workspaces:list[WorkspaceMeta]=Field(default_factory=list)
|
workspaces:list[WorkspaceMeta]=Field(default_factory=list)
|
||||||
recent_workspace_ids:list[str]=Field(default_factory=list)
|
recent_workspace_ids:list[str]=Field(default_factory=list)
|
||||||
|
|
||||||
class ImportPreview(BaseModel):
|
class ImportPreview(BaseModel):
|
||||||
ok:bool=False
|
ok:bool=False
|
||||||
source_dir:str
|
source_dir:str
|
||||||
|
|
@ -58,3 +15,5 @@ class ImportPreview(BaseModel):
|
||||||
semantic_warnings:list[str]=Field(default_factory=list)
|
semantic_warnings:list[str]=Field(default_factory=list)
|
||||||
graph_warnings:list[str]=Field(default_factory=list)
|
graph_warnings:list[str]=Field(default_factory=list)
|
||||||
path_warnings:list[str]=Field(default_factory=list)
|
path_warnings:list[str]=Field(default_factory=list)
|
||||||
|
coverage_warnings:list[str]=Field(default_factory=list)
|
||||||
|
evaluator_warnings:list[str]=Field(default_factory=list)
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,12 @@
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from didactopus.import_validator import preview_draft_pack_import
|
from didactopus.import_validator import preview_draft_pack_import
|
||||||
|
|
||||||
def test_preview_includes_path_warnings(tmp_path: Path) -> None:
|
def test_preview_includes_evaluator_warnings(tmp_path: Path) -> None:
|
||||||
(tmp_path / "pack.yaml").write_text("name: p\ndisplay_name: P\nversion: 0.1.0\n", encoding="utf-8")
|
(tmp_path / "pack.yaml").write_text("name: p\ndisplay_name: P\nversion: 0.1.0\n", encoding="utf-8")
|
||||||
(tmp_path / "concepts.yaml").write_text("concepts:\n - id: c1\n title: Foundations\n description: foundations description enough\n prerequisites: []\n", encoding="utf-8")
|
(tmp_path / "concepts.yaml").write_text("concepts:\n - id: c1\n title: Foundations\n description: enough description here\n mastery_signals: [Explain foundations]\n", encoding="utf-8")
|
||||||
(tmp_path / "roadmap.yaml").write_text("stages:\n - id: s1\n title: One\n concepts: [c1]\n checkpoint: []\n", encoding="utf-8")
|
(tmp_path / "roadmap.yaml").write_text("stages:\n - id: s1\n title: One\n concepts: [c1]\n checkpoint: []\n", encoding="utf-8")
|
||||||
(tmp_path / "projects.yaml").write_text("projects: []\n", encoding="utf-8")
|
(tmp_path / "projects.yaml").write_text("projects: []\n", encoding="utf-8")
|
||||||
(tmp_path / "rubrics.yaml").write_text("rubrics: []\n", encoding="utf-8")
|
(tmp_path / "rubrics.yaml").write_text("rubrics:\n - id: r1\n title: Style\n criteria: [formatting]\n", encoding="utf-8")
|
||||||
|
(tmp_path / "evaluator.yaml").write_text("dimensions:\n - name: typography\n description: page polish\n", encoding="utf-8")
|
||||||
preview = preview_draft_pack_import(tmp_path, "ws1")
|
preview = preview_draft_pack_import(tmp_path, "ws1")
|
||||||
assert isinstance(preview.path_warnings, list)
|
assert isinstance(preview.evaluator_warnings, list)
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1 @@
|
||||||
{
|
{"name":"didactopus-review-ui","private":true,"version":"0.1.0","type":"module"}
|
||||||
"name": "didactopus-review-ui",
|
|
||||||
"private": true,
|
|
||||||
"version": "0.1.0",
|
|
||||||
"type": "module",
|
|
||||||
"scripts": {"dev": "vite", "build": "vite build"},
|
|
||||||
"dependencies": {"react": "^18.3.1", "react-dom": "^18.3.1"},
|
|
||||||
"devDependencies": {"vite": "^5.4.0"}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,150 +1,2 @@
|
||||||
import React, { useEffect, useMemo, useState } from "react";
|
import React from "react";
|
||||||
const API = "http://127.0.0.1:8765";
|
export default function App(){return <div><h1>Didactopus Evaluator Alignment QA</h1><p>Scaffold UI for evaluator alignment warnings.</p></div>}
|
||||||
const statuses = ["needs_review", "trusted", "provisional", "rejected"];
|
|
||||||
|
|
||||||
export default function App() {
|
|
||||||
const [registry, setRegistry] = useState({ workspaces: [], recent_workspace_ids: [] });
|
|
||||||
const [workspaceId, setWorkspaceId] = useState("");
|
|
||||||
const [workspaceTitle, setWorkspaceTitle] = useState("");
|
|
||||||
const [importSource, setImportSource] = useState("");
|
|
||||||
const [importPreview, setImportPreview] = useState(null);
|
|
||||||
const [allowOverwrite, setAllowOverwrite] = useState(false);
|
|
||||||
const [session, setSession] = useState(null);
|
|
||||||
const [selectedId, setSelectedId] = useState("");
|
|
||||||
const [pendingActions, setPendingActions] = useState([]);
|
|
||||||
const [message, setMessage] = useState("Connecting to local Didactopus bridge...");
|
|
||||||
|
|
||||||
async function loadRegistry() {
|
|
||||||
const res = await fetch(`${API}/api/workspaces`);
|
|
||||||
setRegistry(await res.json());
|
|
||||||
if (!session) setMessage("Choose, create, preview, or import a workspace.");
|
|
||||||
}
|
|
||||||
useEffect(() => { loadRegistry().catch(() => setMessage("Could not connect to local review bridge. Start the Python bridge service first.")); }, []);
|
|
||||||
|
|
||||||
async function createWorkspace() {
|
|
||||||
if (!workspaceId || !workspaceTitle) return;
|
|
||||||
await fetch(`${API}/api/workspaces/create`, {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({ workspace_id: workspaceId, title: workspaceTitle })});
|
|
||||||
await loadRegistry(); await openWorkspace(workspaceId);
|
|
||||||
}
|
|
||||||
async function previewImport() {
|
|
||||||
if (!workspaceId || !importSource) return;
|
|
||||||
const res = await fetch(`${API}/api/workspaces/import-preview`, {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({ workspace_id: workspaceId, source_dir: importSource })});
|
|
||||||
const data = await res.json(); setImportPreview(data); setMessage(data.ok ? "Import preview ready." : "Import preview found blocking errors.");
|
|
||||||
}
|
|
||||||
async function importWorkspace() {
|
|
||||||
if (!workspaceId || !importSource) return;
|
|
||||||
const res = await fetch(`${API}/api/workspaces/import`, {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({ workspace_id: workspaceId, title: workspaceTitle || workspaceId, source_dir: importSource, allow_overwrite: allowOverwrite })});
|
|
||||||
const data = await res.json();
|
|
||||||
if (!data.ok) { setMessage(data.error || "Import failed."); return; }
|
|
||||||
await loadRegistry(); await openWorkspace(workspaceId); setMessage(`Imported draft pack from ${importSource} into workspace ${workspaceId}.`);
|
|
||||||
}
|
|
||||||
async function openWorkspace(id) {
|
|
||||||
const res = await fetch(`${API}/api/workspaces/open`, {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({ workspace_id: id })});
|
|
||||||
const opened = await res.json(); if (!opened.ok) { setMessage("Could not open workspace."); return; }
|
|
||||||
const sessionRes = await fetch(`${API}/api/load`); const sessionData = await sessionRes.json();
|
|
||||||
setSession(sessionData.session); setSelectedId(sessionData.session?.draft_pack?.concepts?.[0]?.concept_id || ""); setPendingActions([]); setMessage(`Opened workspace ${id}.`); await loadRegistry();
|
|
||||||
}
|
|
||||||
const selected = useMemo(() => session ? session.draft_pack.concepts.find((c) => c.concept_id === selectedId) || null : null, [session, selectedId]);
|
|
||||||
|
|
||||||
function queueAction(action) { setPendingActions((prev) => [...prev, action]); }
|
|
||||||
function patchConcept(conceptId, patch, rationale) {
|
|
||||||
if (!session) return;
|
|
||||||
const concepts = session.draft_pack.concepts.map((c) => c.concept_id === conceptId ? { ...c, ...patch } : c);
|
|
||||||
setSession({ ...session, draft_pack: { ...session.draft_pack, concepts } });
|
|
||||||
if (patch.status !== undefined) queueAction({ action_type: "set_status", target: conceptId, payload: { status: patch.status }, rationale });
|
|
||||||
if (patch.title !== undefined) queueAction({ action_type: "edit_title", target: conceptId, payload: { title: patch.title }, rationale });
|
|
||||||
if (patch.description !== undefined) queueAction({ action_type: "edit_description", target: conceptId, payload: { description: patch.description }, rationale });
|
|
||||||
if (patch.prerequisites !== undefined) queueAction({ action_type: "edit_prerequisites", target: conceptId, payload: { prerequisites: patch.prerequisites }, rationale });
|
|
||||||
if (patch.notes !== undefined) queueAction({ action_type: "edit_notes", target: conceptId, payload: { notes: patch.notes }, rationale });
|
|
||||||
}
|
|
||||||
function resolveConflict(conflict) {
|
|
||||||
if (!session) return;
|
|
||||||
setSession({ ...session, draft_pack: { ...session.draft_pack, conflicts: session.draft_pack.conflicts.filter((c) => c !== conflict) } });
|
|
||||||
queueAction({ action_type: "resolve_conflict", target: "", payload: { conflict }, rationale: "Resolved in UI" });
|
|
||||||
}
|
|
||||||
async function saveChanges() {
|
|
||||||
if (!pendingActions.length) { setMessage("No pending changes to save."); return; }
|
|
||||||
const res = await fetch(`${API}/api/save`, {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({ actions: pendingActions })});
|
|
||||||
const data = await res.json(); setSession(data.session); setPendingActions([]); setMessage("Saved review state.");
|
|
||||||
}
|
|
||||||
async function exportPromoted() {
|
|
||||||
const res = await fetch(`${API}/api/export`, {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({})});
|
|
||||||
const data = await res.json(); setMessage(`Exported promoted pack to ${data.promoted_pack_dir}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="page">
|
|
||||||
<header className="hero">
|
|
||||||
<div>
|
|
||||||
<h1>Didactopus Curriculum Path QA</h1>
|
|
||||||
<p>Reduce activation-energy by surfacing roadmap and assessment progression problems before import.</p>
|
|
||||||
<div className="small">{message}</div>
|
|
||||||
</div>
|
|
||||||
<div className="hero-actions">
|
|
||||||
<button onClick={saveChanges}>Save Review State</button>
|
|
||||||
<button onClick={exportPromoted} disabled={!session}>Export Promoted Pack</button>
|
|
||||||
</div>
|
|
||||||
</header>
|
|
||||||
|
|
||||||
<section className="summary-grid">
|
|
||||||
<div className="card">
|
|
||||||
<h2>Create Workspace</h2>
|
|
||||||
<label>Workspace ID<input value={workspaceId} onChange={(e) => setWorkspaceId(e.target.value)} /></label>
|
|
||||||
<label>Title<input value={workspaceTitle} onChange={(e) => setWorkspaceTitle(e.target.value)} /></label>
|
|
||||||
<button onClick={createWorkspace}>Create</button>
|
|
||||||
</div>
|
|
||||||
<div className="card">
|
|
||||||
<h2>Preview / Import Draft Pack</h2>
|
|
||||||
<label>Workspace ID<input value={workspaceId} onChange={(e) => setWorkspaceId(e.target.value)} /></label>
|
|
||||||
<label>Draft Pack Source Directory<input value={importSource} onChange={(e) => setImportSource(e.target.value)} /></label>
|
|
||||||
<label className="checkline"><input type="checkbox" checked={allowOverwrite} onChange={(e) => setAllowOverwrite(e.target.checked)} /> Allow overwrite</label>
|
|
||||||
<div className="button-row"><button onClick={previewImport}>Preview</button><button onClick={importWorkspace}>Import</button></div>
|
|
||||||
</div>
|
|
||||||
<div className="card">
|
|
||||||
<h2>Recent</h2>
|
|
||||||
<ul>{registry.recent_workspace_ids.map((id) => <li key={id}><button onClick={() => openWorkspace(id)}>{id}</button></li>)}</ul>
|
|
||||||
</div>
|
|
||||||
<div className="card">
|
|
||||||
<h2>All Workspaces</h2>
|
|
||||||
<ul>{registry.workspaces.map((ws) => <li key={ws.workspace_id}><button onClick={() => openWorkspace(ws.workspace_id)}>{ws.title} ({ws.workspace_id})</button></li>)}</ul>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
{importPreview && (
|
|
||||||
<section className="preview-grid">
|
|
||||||
<div className="card"><h2>Import Preview</h2><div><strong>OK:</strong> {String(importPreview.ok)}</div><div><strong>Overwrite Required:</strong> {String(importPreview.overwrite_required)}</div><div><strong>Pack:</strong> {importPreview.summary?.display_name || importPreview.summary?.pack_name || "-"}</div></div>
|
|
||||||
<div className="card"><h2>Validation Errors</h2><ul>{(importPreview.errors || []).length ? importPreview.errors.map((x, i) => <li key={i}>{x}</li>) : <li>none</li>}</ul></div>
|
|
||||||
<div className="card"><h2>Graph QA Warnings</h2><ul>{(importPreview.graph_warnings || []).length ? importPreview.graph_warnings.map((x, i) => <li key={i}>{x}</li>) : <li>none</li>}</ul></div>
|
|
||||||
<div className="card"><h2>Path Quality Warnings</h2><ul>{(importPreview.path_warnings || []).length ? importPreview.path_warnings.map((x, i) => <li key={i}>{x}</li>) : <li>none</li>}</ul></div>
|
|
||||||
</section>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{session && (
|
|
||||||
<main className="layout">
|
|
||||||
<aside className="sidebar">
|
|
||||||
<h2>Concepts</h2>
|
|
||||||
{session.draft_pack.concepts.map((c) => (
|
|
||||||
<button key={c.concept_id} className={`concept-btn ${c.concept_id === selectedId ? "active" : ""}`} onClick={() => setSelectedId(c.concept_id)}>
|
|
||||||
<span>{c.title}</span><span className={`status-pill status-${c.status}`}>{c.status}</span>
|
|
||||||
</button>
|
|
||||||
))}
|
|
||||||
</aside>
|
|
||||||
<section className="content">
|
|
||||||
{selected && <div className="card">
|
|
||||||
<h2>Concept Editor</h2>
|
|
||||||
<label>Title<input value={selected.title} onChange={(e) => patchConcept(selected.concept_id, { title: e.target.value }, "Edited title")} /></label>
|
|
||||||
<label>Status<select value={selected.status} onChange={(e) => patchConcept(selected.concept_id, { status: e.target.value }, "Changed trust status")}>{statuses.map((s) => <option key={s} value={s}>{s}</option>)}</select></label>
|
|
||||||
<label>Description<textarea rows="6" value={selected.description} onChange={(e) => patchConcept(selected.concept_id, { description: e.target.value }, "Edited description")} /></label>
|
|
||||||
<label>Prerequisites<input value={(selected.prerequisites || []).join(", ")} onChange={(e) => patchConcept(selected.concept_id, { prerequisites: e.target.value.split(",").map((x) => x.trim()).filter(Boolean) }, "Edited prerequisites")} /></label>
|
|
||||||
<label>Notes<textarea rows="4" value={(selected.notes || []).join("\n")} onChange={(e) => patchConcept(selected.concept_id, { notes: e.target.value.split("\n").filter(Boolean) }, "Edited notes")} /></label>
|
|
||||||
</div>}
|
|
||||||
</section>
|
|
||||||
<section className="rightbar">
|
|
||||||
<div className="card"><h2>Conflicts</h2>{session.draft_pack.conflicts.length ? session.draft_pack.conflicts.map((conflict, idx) => <div key={idx} className="conflict"><div>{conflict}</div><button onClick={() => resolveConflict(conflict)}>Resolve</button></div>) : <div className="small">No remaining conflicts.</div>}</div>
|
|
||||||
<div className="card"><h2>Review Flags</h2><ul>{session.draft_pack.review_flags.map((flag, idx) => <li key={idx}>{flag}</li>)}</ul></div>
|
|
||||||
</section>
|
|
||||||
</main>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue