diff --git a/docs_check.py b/docs_check.py index 1af80201..d17bc0d3 100644 --- a/docs_check.py +++ b/docs_check.py @@ -1,233 +1,170 @@ -"""Docs integrity checker for PR #561.""" +"""Mintlify docs integrity checker — run before merging any docs PR.""" +from __future__ import annotations + +import glob import json import os import re -import glob +import sys +from typing import Any, Callable, cast -DOCS_DIR = "docs" +DOCS = "docs" +ALL_MD: list[str] = glob.glob(f"{DOCS}/**/*.md", recursive=True) -results = {"pass": [], "fail": []} +failures: list[str] = [] -def ok(msg): - results["pass"].append(msg) - print(f" PASS {msg}") +def check(label: str) -> Callable[[Callable[[], list[str]]], None]: + """Decorator: run a check function, print result, collect failures.""" + def decorator(fn: Callable[[], list[str]]) -> None: + issues = fn() + if issues: + print(f"FAIL {label}") + for msg in issues: + print(f" - {msg}") + failures.extend(issues) + else: + print(f"pass {label}") + return decorator -def fail(msg): - results["fail"].append(msg) - print(f" FAIL {msg}") +def read(path: str) -> str: + return open(path, encoding="utf-8").read() -# ── 1. docs.json valid JSON ─────────────────────────────────────────────────── -print("\n[1] docs.json validity") -try: - with open(os.path.join(DOCS_DIR, "docs.json"), encoding="utf-8") as f: - cfg = json.load(f) - ok("docs.json is valid JSON") -except Exception as e: - fail(f"docs.json parse error: {e}") - cfg = {} +# ── 1. docs.json is valid JSON ──────────────────────────────────────────────── +@check("docs.json is valid JSON") +def _() -> list[str]: + try: + json.loads(read(f"{DOCS}/docs.json")) + return [] + except Exception as e: + return [str(e)] -# ── 2. All nav pages exist on disk ──────────────────────────────────────────── -print("\n[2] Nav pages exist on disk") +# ── 2. Every nav page exists on disk ───────────────────────────────────────── +@check("All nav pages exist on disk") +def _() -> list[str]: + cfg: Any = json.loads(read(f"{DOCS}/docs.json")) - -def collect_pages(obj): - pages = [] - if isinstance(obj, dict): - if "pages" in obj: - for p in obj["pages"]: - if isinstance(p, str): - pages.append(p) + def nav_pages(obj: Any) -> list[str]: + """Recursively collect strings from 'pages' arrays only.""" + result: list[str] = [] + if isinstance(obj, dict): + d = cast(dict[str, Any], obj) + for page in cast(list[Any], d.get("pages", [])): + if isinstance(page, str): + result.append(page) else: - pages.extend(collect_pages(p)) - for v in obj.values(): - if isinstance(v, (dict, list)): - pages.extend(collect_pages(v)) - elif isinstance(obj, list): - for item in obj: - pages.extend(collect_pages(item)) - return pages + result.extend(nav_pages(page)) + for v in d.values(): + if isinstance(v, (dict, list)): + result.extend(nav_pages(v)) + elif isinstance(obj, list): + for item in cast(list[Any], obj): + result.extend(nav_pages(item)) + return result + + return [ + f"missing: {p}" + for p in set(nav_pages(cfg)) + if not p.startswith("http") and not os.path.exists(f"{DOCS}/{p}.md") + ] -nav_pages = [p for p in set(collect_pages(cfg)) if not p.startswith("http")] -missing_pages = [] -for p in sorted(nav_pages): - path = os.path.join(DOCS_DIR, p + ".md") - if not os.path.exists(path): - missing_pages.append(p) +# ── 3. Internal Card hrefs resolve ─────────────────────────────────────────── +@check("All internal Card hrefs resolve") +def _() -> list[str]: + issues: list[str] = [] + for fpath in ALL_MD: + for m in re.finditer(r'href=["\'](?!http)([^"\'#]+)["\']', read(fpath)): + href = m.group(1).strip() + target = os.path.normpath(os.path.join(os.path.dirname(fpath), href)) + ".md" + if not os.path.exists(target): + issues.append(f"{fpath}: href '{href}'") + return issues -if missing_pages: - for m in missing_pages: - fail(f"Nav page missing: {m}") + +# ── 4. No stale repo URLs ───────────────────────────────────────────────────── +@check("No stale repo URLs") +def _() -> list[str]: + stale = ["Hawksight-AI/semantica", "semantica-dev/semantica"] + files = ALL_MD + [f"{DOCS}/docs.json"] + return [ + f"{f}: '{pat}'" + for f in files + for pat in stale + if pat in read(f) + ] + + +# ── 5. All reference pages have frontmatter ─────────────────────────────────── +@check("All reference pages have frontmatter") +def _() -> list[str]: + return [ + os.path.basename(f) + for f in glob.glob(f"{DOCS}/reference/*.md") + if not read(f).startswith("---") + ] + + +# ── 6. No known-wrong class names ──────────────────────────────────────────── +@check("No known-wrong class names in reference pages") +def _() -> list[str]: + banned: list[tuple[str, str]] = [ + (r"\bBaseIngestor\b", "docs/architecture.md"), + (r"\bBaseExtractor\b", "docs/architecture.md"), + (r"\bBasePlugin\b", "docs/architecture.md"), + (r"\bDataNormalizer\b", "docs/reference/normalize.md"), + (r"\bEntityResolver\b", "docs/reference/deduplication.md"), + (r"\bDeductiveEngine\b", "docs/reference/reasoning.md"), + (r"\bAbductiveEngine\b", "docs/reference/reasoning.md"), + (r"\bGraphMLExporter\b", "docs/reference/export.md"), + (r"\bArangoExporter\b(?!.*AQL)", "docs/reference/export.md"), + (r"(? list[str]: + issues: list[str] = [] + for fpath in ALL_MD: + in_block = False + for i, line in enumerate(open(fpath, encoding="utf-8"), 1): + if line.strip().startswith("```"): + in_block = not in_block + if in_block and re.search(r":\s*(list|dict|tuple|set)\[", line): + issues.append(f"{fpath}:{i}: {line.rstrip()}") + return issues + + +# ── 8. index.md covers all 27 modules ──────────────────────────────────────── +@check("All 27 modules present in index.md") +def _() -> list[str]: + modules = [ + "semantica.ingest", "semantica.parse", "semantica.split", "semantica.normalize", + "semantica.semantic_extract", "semantica.kg", "semantica.ontology", "semantica.reasoning", + "semantica.embeddings", "semantica.vector_store", "semantica.graph_store", + "semantica.triplet_store", "semantica.context", "semantica.provenance", + "semantica.change_management", "semantica.deduplication", "semantica.conflicts", + "semantica.export", "semantica.visualization", "semantica.pipeline", "semantica.seed", + "semantica.llms", "semantica.mcp_server", "semantica.explorer", "semantica.evals", + "semantica.utils", "semantica.core", + ] + index = read(f"{DOCS}/index.md") + return [m for m in modules if m not in index] + + +# ── Summary ─────────────────────────────────────────────────────────────────── +print() +if failures: + print(f"FAILED {len(failures)} issue(s) found") + sys.exit(1) else: - ok(f"All {len(nav_pages)} nav pages exist on disk") - - -# ── 3. Internal Card hrefs resolve (page-relative) ─────────────────────────── -print("\n[3] Internal Card hrefs") -broken_hrefs = [] - -for fpath in glob.glob(DOCS_DIR + "/**/*.md", recursive=True): - file_dir = os.path.dirname(fpath) - with open(fpath, encoding="utf-8") as f: - content = f.read() - for m in re.finditer(r'href=["\'](?!http)([^"\'#]+)["\']', content): - href = m.group(1).strip() - if not href: - continue - # Resolve relative to the file's directory (mirrors browser URL resolution) - resolved = os.path.normpath(os.path.join(file_dir, href)) - target_md = resolved + ".md" - if not os.path.exists(target_md): - rel = fpath.replace("\\", "/") - broken_hrefs.append(f"{rel}: href '{href}' -> {target_md}") - -if broken_hrefs: - for b in broken_hrefs[:20]: - fail(f"Broken href: {b}") - if len(broken_hrefs) > 20: - fail(f"...and {len(broken_hrefs) - 20} more broken hrefs") -else: - ok("All internal Card hrefs resolve to real files") - - -# ── 4. No old repo URLs ─────────────────────────────────────────────────────── -print("\n[4] Repo URL consistency") -old_patterns = ["Hawksight-AI/semantica", "semantica-dev/semantica"] -old_url_hits = [] -for fpath in glob.glob(DOCS_DIR + "/**/*.md", recursive=True) + [ - os.path.join(DOCS_DIR, "docs.json") -]: - with open(fpath, encoding="utf-8") as f: - content = f.read() - for pat in old_patterns: - if pat in content: - old_url_hits.append(f"{fpath}: contains '{pat}'") - -if old_url_hits: - for h in old_url_hits: - fail(h) -else: - ok("No old repo URLs found (Hawksight-AI, semantica-dev)") - - -# ── 5. All reference .md files have frontmatter ─────────────────────────────── -print("\n[5] Reference page frontmatter") -ref_pages = list(glob.glob(DOCS_DIR + "/reference/*.md")) -no_frontmatter = [] -for fpath in ref_pages: - with open(fpath, encoding="utf-8") as f: - content = f.read() - if not content.startswith("---"): - no_frontmatter.append(os.path.basename(fpath)) - -if no_frontmatter: - for f in no_frontmatter: - fail(f"Missing frontmatter: {f}") -else: - ok(f"All {len(ref_pages)} reference pages have frontmatter") - - -# ── 6. Code examples: no non-existent class names (exact word match) ────────── -print("\n[6] Known-wrong class names") -# (symbol, file, exclude_pattern) — exclude_pattern avoids substring false positives -banned = [ - ("BaseIngestor", "docs/architecture.md", None), - ("BaseExtractor", "docs/architecture.md", None), - ("BasePlugin", "docs/architecture.md", None), - (r"PluginRegistry\.register\(", "docs/architecture.md", r"register_plugin"), - ("start_explorer", "docs/reference/explorer.md", None), - (r"graph\.save\(", "docs/reference/explorer.md", None), - (r"\bDataNormalizer\b", "docs/reference/normalize.md", None), - (r"\bEntityResolver\b", "docs/reference/deduplication.md", None), - # ReasoningEngine: only flag as exact word, not as part of TemporalReasoningEngine - (r"(? 10: - fail(f"...and {len(py39_hits) - 10} more") -else: - ok("No Python 3.9+ lowercase generic type hints in doc code blocks") - - -# ── 8. Module table covers all 27 modules ───────────────────────────────────── -print("\n[8] Module table coverage in index.md") -expected_modules = [ - "semantica.ingest", "semantica.parse", "semantica.split", "semantica.normalize", - "semantica.semantic_extract", "semantica.kg", "semantica.ontology", "semantica.reasoning", - "semantica.embeddings", "semantica.vector_store", "semantica.graph_store", "semantica.triplet_store", - "semantica.context", "semantica.provenance", "semantica.change_management", - "semantica.deduplication", "semantica.conflicts", "semantica.export", "semantica.visualization", - "semantica.pipeline", "semantica.seed", "semantica.llms", "semantica.mcp_server", - "semantica.explorer", "semantica.evals", "semantica.utils", "semantica.core", -] -index_path = os.path.join(DOCS_DIR, "index.md") -with open(index_path, encoding="utf-8") as f: - index_content = f.read() - -missing_modules = [m for m in expected_modules if m not in index_content] -if missing_modules: - for m in missing_modules: - fail(f"Module missing from index table: {m}") -else: - ok(f"All {len(expected_modules)} modules present in index.md table") - - -# ── Summary ────────────────────────────────────────────────────────────────── -print(f"\n{'='*60}") -print(f"Results: {len(results['pass'])} passed, {len(results['fail'])} failed") -if results["fail"]: - print("STATUS: FAIL") - raise SystemExit(1) -else: - print("STATUS: ALL CHECKS PASSED") + print("All checks passed")