From c049fc4d862fd181749307c71e95b312f2d9c9e7 Mon Sep 17 00:00:00 2001 From: welsberr Date: Sat, 14 Mar 2026 13:29:55 -0400 Subject: [PATCH] Apply ZIP update: 130-didactopus-semantic-qa-update.zip [2026-03-14T13:19:13] --- bad-generated-pack/concepts.yaml | 40 +++++-------------- bad-generated-pack/roadmap.yaml | 14 +++---- bad-generated-pack/rubrics.yaml | 6 ++- docs/faq.md | 32 +++++++-------- generated-pack/concepts.yaml | 6 +++ generated-pack/projects.yaml | 9 ++++- generated-pack/roadmap.yaml | 9 +++-- generated-pack/rubrics.yaml | 7 +++- pyproject.toml | 2 +- src/didactopus/import_validator.py | 3 -- src/didactopus/review_bridge_server.py | 2 +- src/didactopus/review_schema.py | 1 - src/didactopus/semantic_qa.py | 13 +++++- tests/test_import_validator.py | 8 ++-- tests/test_pack_validator.py | 4 +- webui/src/App.jsx | 15 ++++--- webui/src/styles.css | 2 + .../bayes-intro/draft_pack/concepts.yaml | 2 + 18 files changed, 94 insertions(+), 81 deletions(-) diff --git a/bad-generated-pack/concepts.yaml b/bad-generated-pack/concepts.yaml index a9c483b..1c3e587 100644 --- a/bad-generated-pack/concepts.yaml +++ b/bad-generated-pack/concepts.yaml @@ -1,33 +1,13 @@ concepts: - - id: a - title: Foundations - description: Introductory foundations concept with broad scope. - prerequisites: [c] - - id: b - title: Advanced Analysis - description: Advanced analysis of results in a difficult domain. - prerequisites: [a] - - id: c - title: Methods - description: Methods and procedures for analysis in context. - prerequisites: [b] - - id: isolated - title: Isolated Topic - description: A topic disconnected from the rest of the graph. + - id: prior-and-posterior + title: Prior and Posterior + description: Beliefs before and after evidence. prerequisites: [] - - id: bottleneck - title: Core Bottleneck - description: Central concept required by many dependents in the pack. + - id: posterior-analysis + title: Posterior Analysis + description: Beliefs before and after evidence. + prerequisites: [] + - id: statistics-and-probability + title: Statistics and Probability + description: General overview. prerequisites: [] - - id: d1 - title: Dependent One - description: Depends on a single core bottleneck concept. - prerequisites: [bottleneck] - - id: d2 - title: Dependent Two - description: Depends on a single core bottleneck concept. - prerequisites: [bottleneck] - - id: d3 - title: Dependent Three - description: Depends on a single core bottleneck concept. - prerequisites: [bottleneck] diff --git a/bad-generated-pack/roadmap.yaml b/bad-generated-pack/roadmap.yaml index 0b378df..1b1c7b4 100644 --- a/bad-generated-pack/roadmap.yaml +++ b/bad-generated-pack/roadmap.yaml @@ -1,13 +1,9 @@ stages: - id: stage-1 title: Foundations - concepts: [a, bottleneck] + concepts: + - statistics-and-probability - id: stage-2 - title: Advanced Analysis - concepts: [b, d1] - - id: stage-3 - title: Methods - concepts: [c, d2, d3] - - id: stage-4 - title: Detached Topic - concepts: [isolated] + title: Advanced Inference + concepts: + - posterior-analysis diff --git a/bad-generated-pack/rubrics.yaml b/bad-generated-pack/rubrics.yaml index 14571f5..cd1f79a 100644 --- a/bad-generated-pack/rubrics.yaml +++ b/bad-generated-pack/rubrics.yaml @@ -1 +1,5 @@ -rubrics: [] +rubrics: + - id: basic-rubric + title: Basic Rubric + criteria: + - correctness diff --git a/docs/faq.md b/docs/faq.md index d5ff140..d4b8bde 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,24 +1,24 @@ # FAQ -## Why add graph-aware analysis? +## Why add semantic QA? -Because Didactopus is fundamentally concerned with mastery paths and dependency structure. -A pack can look fine in text form and still behave badly as a learning graph. - -## What kinds of issues can it find? - -Examples: -- cycles -- isolated concepts -- dependency bottlenecks -- packs that are too flat -- packs that are too deep +Because a pack can be structurally valid and still be awkward or misleading as a +learning domain. ## How does this help with the activation-energy problem? -It reduces the chance that a user imports a pack and only later discovers that the -prerequisite structure is confusing or fragile. +It catches likely high-level issues earlier, so users do not have to discover +them only after they have already committed to review or study. -## Does this replace human review? +## Does semantic QA prove that a pack is good? -No. It gives the reviewer better signals earlier. +No. It is a heuristic curation aid. + +## What kinds of problems can it flag? + +Examples: +- duplicate or near-duplicate concepts +- over-broad concepts +- abrupt stage transitions +- weak prerequisite structure +- descriptions that are too similar to each other diff --git a/generated-pack/concepts.yaml b/generated-pack/concepts.yaml index c7635ba..3a65b82 100644 --- a/generated-pack/concepts.yaml +++ b/generated-pack/concepts.yaml @@ -3,13 +3,19 @@ concepts: title: Bayes Prior description: Prior beliefs before evidence in a probabilistic model. prerequisites: [] + mastery_signals: + - Explain a prior distribution. - id: bayes-posterior title: Bayes Posterior description: Updated beliefs after evidence in a probabilistic model. prerequisites: - bayes-prior + mastery_signals: + - Compare prior and posterior beliefs. - id: model-checking title: Model Checking description: Evaluate whether model assumptions and fit remain plausible. prerequisites: - bayes-posterior + mastery_signals: + - Critique a model fit. diff --git a/generated-pack/projects.yaml b/generated-pack/projects.yaml index 4d053f9..4c01d81 100644 --- a/generated-pack/projects.yaml +++ b/generated-pack/projects.yaml @@ -1 +1,8 @@ -projects: [] +projects: + - id: compare-beliefs + title: Compare Prior and Posterior + prerequisites: + - bayes-prior + - bayes-posterior + deliverables: + - short report diff --git a/generated-pack/roadmap.yaml b/generated-pack/roadmap.yaml index d196c42..8b2f6ca 100644 --- a/generated-pack/roadmap.yaml +++ b/generated-pack/roadmap.yaml @@ -1,10 +1,13 @@ stages: - id: stage-1 title: Prior Beliefs - concepts: [bayes-prior] + concepts: + - bayes-prior - id: stage-2 title: Posterior Updating - concepts: [bayes-posterior] + concepts: + - bayes-posterior - id: stage-3 title: Model Checking - concepts: [model-checking] + concepts: + - model-checking diff --git a/generated-pack/rubrics.yaml b/generated-pack/rubrics.yaml index 14571f5..594266b 100644 --- a/generated-pack/rubrics.yaml +++ b/generated-pack/rubrics.yaml @@ -1 +1,6 @@ -rubrics: [] +rubrics: + - id: basic-rubric + title: Basic Rubric + criteria: + - correctness + - explanation diff --git a/pyproject.toml b/pyproject.toml index 054ecf5..ff216ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "didactopus" version = "0.1.0" -description = "Didactopus: graph-aware prerequisite analysis" +description = "Didactopus: semantic QA layer for domain packs" readme = "README.md" requires-python = ">=3.10" license = {text = "MIT"} diff --git a/src/didactopus/import_validator.py b/src/didactopus/import_validator.py index 5b60934..8bde4f5 100644 --- a/src/didactopus/import_validator.py +++ b/src/didactopus/import_validator.py @@ -3,12 +3,10 @@ from pathlib import Path from .review_schema import ImportPreview from .pack_validator import validate_pack_directory from .semantic_qa import semantic_qa_for_pack -from .graph_qa import graph_qa_for_pack def preview_draft_pack_import(source_dir: str | Path, workspace_id: str, overwrite_required: bool = False) -> ImportPreview: result = validate_pack_directory(source_dir) semantic = semantic_qa_for_pack(source_dir) if result["ok"] else {"warnings": [], "summary": {}} - graph = graph_qa_for_pack(source_dir) if result["ok"] else {"warnings": [], "summary": {}} preview = ImportPreview( source_dir=str(Path(source_dir)), workspace_id=workspace_id, @@ -18,6 +16,5 @@ def preview_draft_pack_import(source_dir: str | Path, workspace_id: str, overwri warnings=list(result["warnings"]), summary=dict(result["summary"]), semantic_warnings=list(semantic["warnings"]), - graph_warnings=list(graph["warnings"]), ) return preview diff --git a/src/didactopus/review_bridge_server.py b/src/didactopus/review_bridge_server.py index ac00de8..98c7351 100644 --- a/src/didactopus/review_bridge_server.py +++ b/src/didactopus/review_bridge_server.py @@ -120,7 +120,7 @@ class ReviewBridgeHandler(BaseHTTPRequestHandler): json_response(self, 404, {"error": "not found"}) def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Didactopus local review bridge server with graph QA") + parser = argparse.ArgumentParser(description="Didactopus local review bridge server with semantic QA") parser.add_argument("--config", default="configs/config.example.yaml") return parser diff --git a/src/didactopus/review_schema.py b/src/didactopus/review_schema.py index e02ca0e..352940e 100644 --- a/src/didactopus/review_schema.py +++ b/src/didactopus/review_schema.py @@ -56,4 +56,3 @@ class ImportPreview(BaseModel): warnings: list[str] = Field(default_factory=list) summary: dict = Field(default_factory=dict) semantic_warnings: list[str] = Field(default_factory=list) - graph_warnings: list[str] = Field(default_factory=list) diff --git a/src/didactopus/semantic_qa.py b/src/didactopus/semantic_qa.py index c20fd05..b6be58d 100644 --- a/src/didactopus/semantic_qa.py +++ b/src/didactopus/semantic_qa.py @@ -1,4 +1,5 @@ from __future__ import annotations +from pathlib import Path import re from difflib import SequenceMatcher from .pack_validator import load_pack_artifacts @@ -14,7 +15,7 @@ def similarity(a: str, b: str) -> float: def token_set(text: str) -> set[str]: return {t for t in normalize_title(text).split() if t} -def semantic_qa_for_pack(source_dir) -> dict: +def semantic_qa_for_pack(source_dir: str | Path) -> dict: loaded = load_pack_artifacts(source_dir) if not loaded["ok"]: return {"warnings": [], "summary": {"semantic_warning_count": 0}} @@ -25,6 +26,7 @@ def semantic_qa_for_pack(source_dir) -> dict: warnings: list[str] = [] + # Near-duplicate titles for i in range(len(concepts)): for j in range(i + 1, len(concepts)): a = concepts[i] @@ -33,6 +35,7 @@ def semantic_qa_for_pack(source_dir) -> dict: if sim >= 0.86 and a.get("id") != b.get("id"): warnings.append(f"Near-duplicate concept titles: '{a.get('title')}' vs '{b.get('title')}'") + # Over-broad titles for concept in concepts: title = concept.get("title", "") toks = token_set(title) @@ -41,6 +44,7 @@ def semantic_qa_for_pack(source_dir) -> dict: if " and " in title.lower(): warnings.append(f"Concept '{title}' is compound and may combine multiple ideas.") + # Similar descriptions for i in range(len(concepts)): for j in range(i + 1, len(concepts)): da = str(concepts[i].get("description", "") or "") @@ -52,12 +56,14 @@ def semantic_qa_for_pack(source_dir) -> dict: f"Concept descriptions are very similar: '{concepts[i].get('title')}' vs '{concepts[j].get('title')}'" ) + # Thin prerequisite chains on advanced-sounding concepts for concept in concepts: title = normalize_title(concept.get("title", "")) prereqs = concept.get("prerequisites", []) or [] if any(h in title for h in ["advanced", "posterior", "model", "inference", "analysis"]) and len(prereqs) == 0: warnings.append(f"Concept '{concept.get('title')}' looks advanced but has no prerequisites.") + # Missing bridge concepts between roadmap stages concept_by_id = {c.get("id"): c for c in concepts if c.get("id")} for idx in range(len(roadmap) - 1): current_stage = roadmap[idx] @@ -78,5 +84,8 @@ def semantic_qa_for_pack(source_dir) -> dict: return { "warnings": warnings, - "summary": {"semantic_warning_count": len(warnings), "pack_name": pack.get("name", "")}, + "summary": { + "semantic_warning_count": len(warnings), + "pack_name": pack.get("name", ""), + }, } diff --git a/tests/test_import_validator.py b/tests/test_import_validator.py index 7065540..758607f 100644 --- a/tests/test_import_validator.py +++ b/tests/test_import_validator.py @@ -1,14 +1,14 @@ from pathlib import Path from didactopus.import_validator import preview_draft_pack_import -def test_preview_includes_graph_warnings(tmp_path: Path) -> None: +def test_preview_includes_semantic_warnings(tmp_path: Path) -> None: (tmp_path / "pack.yaml").write_text("name: p\ndisplay_name: P\nversion: 0.1.0\n", encoding="utf-8") (tmp_path / "concepts.yaml").write_text( - "concepts:\n - id: a\n title: A\n description: description long enough\n prerequisites: [b]\n - id: b\n title: B\n description: description long enough\n prerequisites: [a]\n", + "concepts:\n - id: c1\n title: Prior and Posterior\n description: Beliefs before and after evidence.\n - id: c2\n title: Posterior Analysis\n description: Beliefs before and after evidence.\n", encoding="utf-8" ) - (tmp_path / "roadmap.yaml").write_text("stages:\n - id: s1\n title: One\n concepts: [a,b]\n", encoding="utf-8") + (tmp_path / "roadmap.yaml").write_text("stages:\n - id: s1\n title: Foundations\n concepts: [c1]\n - id: s2\n title: Advanced Inference\n concepts: [c2]\n", encoding="utf-8") (tmp_path / "projects.yaml").write_text("projects: []\n", encoding="utf-8") (tmp_path / "rubrics.yaml").write_text("rubrics: []\n", encoding="utf-8") preview = preview_draft_pack_import(tmp_path, "ws1") - assert isinstance(preview.graph_warnings, list) + assert isinstance(preview.semantic_warnings, list) diff --git a/tests/test_pack_validator.py b/tests/test_pack_validator.py index 85a120c..82c5d40 100644 --- a/tests/test_pack_validator.py +++ b/tests/test_pack_validator.py @@ -5,7 +5,7 @@ def test_valid_pack(tmp_path: Path) -> None: (tmp_path / "pack.yaml").write_text("name: p\ndisplay_name: P\nversion: 0.1.0\n", encoding="utf-8") (tmp_path / "concepts.yaml").write_text("concepts:\n - id: c1\n title: C1\n description: A full enough description.\n", encoding="utf-8") (tmp_path / "roadmap.yaml").write_text("stages:\n - id: s1\n title: S1\n concepts: [c1]\n", encoding="utf-8") - (tmp_path / "projects.yaml").write_text("projects: []\n", encoding="utf-8") - (tmp_path / "rubrics.yaml").write_text("rubrics: []\n", encoding="utf-8") + (tmp_path / "projects.yaml").write_text("projects:\n - id: p1\n title: P1\n prerequisites: [c1]\n", encoding="utf-8") + (tmp_path / "rubrics.yaml").write_text("rubrics:\n - id: r1\n title: R1\n criteria: [correctness]\n", encoding="utf-8") result = validate_pack_directory(tmp_path) assert result["ok"] is True diff --git a/webui/src/App.jsx b/webui/src/App.jsx index 2018a53..9909b6a 100644 --- a/webui/src/App.jsx +++ b/webui/src/App.jsx @@ -153,10 +153,10 @@ export default function App() {
-

Didactopus Graph QA

+

Didactopus Semantic QA

Reduce the activation-energy hump from generated draft packs to curated review workspaces - by surfacing prerequisite-graph problems before import. + by surfacing semantic curation issues before import.

{message}
@@ -202,19 +202,22 @@ export default function App() {
Pack: {importPreview.summary?.display_name || importPreview.summary?.pack_name || "-"}
Version: {importPreview.summary?.version || "-"}
Concepts: {importPreview.summary?.concept_count ?? "-"}
+
Roadmap Stages: {importPreview.summary?.roadmap_stage_count ?? "-"}
+
Projects: {importPreview.summary?.project_count ?? "-"}
+
Rubrics: {importPreview.summary?.rubric_count ?? "-"}

Validation Errors

+

Validation Warnings

+ +
+

Semantic QA Warnings

-
-

Graph QA Warnings

- -
)} diff --git a/webui/src/styles.css b/webui/src/styles.css index 0fd2538..c657761 100644 --- a/webui/src/styles.css +++ b/webui/src/styles.css @@ -18,6 +18,7 @@ button:hover { border-color: var(--accent); } .preview-grid { margin-top: 16px; display: grid; grid-template-columns: repeat(4, 1fr); gap: 16px; } .layout { margin-top: 16px; display: grid; grid-template-columns: 290px 1fr 360px; gap: 16px; } .card { background: var(--card); border: 1px solid var(--border); border-radius: 18px; padding: 16px; } +.semantic-card { grid-column: span 4; } .sidebar, .content, .rightbar { display: flex; flex-direction: column; gap: 16px; } .concept-btn { width: 100%; text-align: left; display: flex; justify-content: space-between; gap: 8px; margin-bottom: 10px; } .concept-btn.active { border-color: var(--accent); box-shadow: 0 0 0 2px rgba(45,108,223,0.08); } @@ -36,5 +37,6 @@ ul { padding-left: 18px; } .checkline input { width: auto; margin-top: 0; } @media (max-width: 1100px) { .summary-grid, .preview-grid { grid-template-columns: repeat(2, 1fr); } + .semantic-card { grid-column: span 2; } .layout { grid-template-columns: 1fr; } } diff --git a/workspaces/bayes-intro/draft_pack/concepts.yaml b/workspaces/bayes-intro/draft_pack/concepts.yaml index eb91593..21a89c1 100644 --- a/workspaces/bayes-intro/draft_pack/concepts.yaml +++ b/workspaces/bayes-intro/draft_pack/concepts.yaml @@ -3,3 +3,5 @@ concepts: title: Descriptive Statistics description: Measures of center and spread in descriptive data analysis. prerequisites: [] + mastery_signals: + - Explain mean, median, and variance.