From c7174e9852ae155236d4046d2651adf24fc82728 Mon Sep 17 00:00:00 2001 From: Saurabh Meena <127095776+SaurabhScripts@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:12:30 +0530 Subject: [PATCH 001/129] fix(context): reject Markdown import symlinks --- docs/reference/context.md | 3 +- semantica/context/agent_memory.py | 60 ++++++++++++--- tests/context/test_agent_memory_markdown.py | 81 +++++++++++++++++++++ 3 files changed, 132 insertions(+), 12 deletions(-) diff --git a/docs/reference/context.md b/docs/reference/context.md index 18950e9e..94bcd115 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -626,7 +626,8 @@ files is idempotent. Memory-local `entities` and `relationships` are preserved a provenance but are not applied to `ContextGraph` by Markdown import. Use a dedicated export directory: matching files are overwritten, but unrelated or stale Markdown files are not deleted automatically. Export refuses to overwrite symbolic links and -uses atomic file replacement. Timestamp offsets are preserved in Markdown and +uses atomic file replacement; import also refuses symbolic-link files and directories. +Timestamp offsets are preserved in Markdown and normalized to UTC only for comparisons, so aware and local-naive records can be queried together safely. Vector-store writes are deferred until the in-memory import commits; adapter synchronization remains best-effort and logs failures. diff --git a/semantica/context/agent_memory.py b/semantica/context/agent_memory.py index 2f2b8995..93c8b568 100644 --- a/semantica/context/agent_memory.py +++ b/semantica/context/agent_memory.py @@ -59,9 +59,11 @@ License: MIT """ import copy +import errno import hashlib import os import re +import stat import tempfile from collections import deque from dataclasses import dataclass, field @@ -1865,7 +1867,8 @@ class AgentMemory: if "\n" not in data and "\r" not in data: candidate = Path(data) try: - candidate_exists = candidate.exists() + candidate_is_symlink = candidate.is_symlink() + candidate_exists = candidate_is_symlink or candidate.exists() except OSError as exc: error_message = ( "Failed to inspect possible Markdown import " @@ -1907,29 +1910,64 @@ class AgentMemory: return memories def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]: + if path.is_symlink(): + raise ValueError(f"Refusing to import Markdown symbolic link: {path}") + if not path.exists(): raise FileNotFoundError(f"Markdown import path does not exist: {path}") if path.is_dir(): - file_paths = sorted( - ( - file_path - for file_path in path.iterdir() - if file_path.is_file() - and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS - ), - key=lambda file_path: (file_path.name.casefold(), file_path.name), - ) + file_paths = [] + for file_path in path.iterdir(): + if file_path.suffix.lower() not in self._MARKDOWN_EXTENSIONS: + continue + if file_path.is_symlink(): + raise ValueError( + f"Refusing to import Markdown symbolic link: {file_path}" + ) + if file_path.is_file(): + file_paths.append(file_path) + file_paths.sort(key=lambda item: (item.name.casefold(), item.name)) elif path.is_file(): file_paths = [path] else: raise ValueError(f"Markdown import path is not a file or directory: {path}") return [ - (str(file_path), file_path.read_text(encoding="utf-8")) + (str(file_path), self._read_markdown_file(file_path)) for file_path in file_paths ] + @staticmethod + def _read_markdown_file(file_path: Path) -> str: + """Read a regular Markdown file without following a raced symlink.""" + if file_path.is_symlink(): + raise ValueError(f"Refusing to import Markdown symbolic link: {file_path}") + + flags = os.O_RDONLY + nofollow_flag = getattr(os, "O_NOFOLLOW", 0) + flags |= nofollow_flag + try: + file_descriptor = os.open(file_path, flags) + except OSError as exc: + if nofollow_flag and exc.errno == errno.ELOOP: + raise ValueError( + f"Refusing to import Markdown symbolic link: {file_path}" + ) from exc + raise + + try: + if not stat.S_ISREG(os.fstat(file_descriptor).st_mode): + raise ValueError( + f"Markdown import path is not a regular file: {file_path}" + ) + with os.fdopen(file_descriptor, mode="r", encoding="utf-8") as source: + file_descriptor = -1 + return source.read() + finally: + if file_descriptor >= 0: + os.close(file_descriptor) + def _markdown_to_memory_dict( self, document: str, source: str = "markdown document" ) -> Dict[str, Any]: diff --git a/tests/context/test_agent_memory_markdown.py b/tests/context/test_agent_memory_markdown.py index fb9158ed..d403504c 100644 --- a/tests/context/test_agent_memory_markdown.py +++ b/tests/context/test_agent_memory_markdown.py @@ -1,6 +1,8 @@ import errno +import os from copy import deepcopy from datetime import datetime, timedelta, timezone +from pathlib import Path from unittest.mock import MagicMock, patch import pytest @@ -698,6 +700,85 @@ def test_markdown_string_path_inspection_errors_are_actionable(): assert exc_info.value.__cause__ is original_error +@pytest.mark.parametrize("use_string_path", [False, True]) +def test_markdown_import_rejects_symlinked_file(tmp_path, use_string_path): + outside = tmp_path / "outside.md" + outside.write_text( + markdown_document(required_frontmatter(), "Do not import"), + encoding="utf-8", + ) + source = tmp_path / "memory.md" + source.symlink_to(outside) + payload = str(source) if use_string_path else source + memory = AgentMemory() + + with pytest.raises(ValueError, match="symbolic link"): + memory.import_data(payload, format="markdown") + + assert memory.count() == 0 + + +@pytest.mark.parametrize("use_string_path", [False, True]) +def test_markdown_import_rejects_broken_symlink(tmp_path, use_string_path): + source = tmp_path / "missing-memory.md" + source.symlink_to(tmp_path / "missing-target.md") + payload = str(source) if use_string_path else source + + with pytest.raises(ValueError, match="symbolic link"): + AgentMemory().import_data(payload, format="markdown") + + +@pytest.mark.parametrize("use_string_path", [False, True]) +def test_markdown_import_rejects_symlinked_directory(tmp_path, use_string_path): + outside = tmp_path / "outside" + outside.mkdir() + (outside / "memory.md").write_text( + markdown_document(required_frontmatter(), "Do not import"), + encoding="utf-8", + ) + source = tmp_path / "memory-export" + source.symlink_to(outside, target_is_directory=True) + payload = str(source) if use_string_path else source + memory = AgentMemory() + + with pytest.raises(ValueError, match="symbolic link"): + memory.import_data(payload, format="markdown") + + assert memory.count() == 0 + + +def test_markdown_import_rejects_symlinked_file_in_directory(tmp_path): + outside = tmp_path / "outside.md" + outside.write_text( + markdown_document(required_frontmatter(), "Do not import"), + encoding="utf-8", + ) + source = tmp_path / "memory-export" + source.mkdir() + (source / "memory.md").symlink_to(outside) + memory = AgentMemory() + + with pytest.raises(ValueError, match="symbolic link"): + memory.import_data(source, format="markdown") + + assert memory.count() == 0 + + +@pytest.mark.skipif(not hasattr(os, "O_NOFOLLOW"), reason="requires O_NOFOLLOW") +def test_markdown_import_does_not_follow_symlink_raced_before_open(tmp_path): + outside = tmp_path / "outside.md" + outside.write_text( + markdown_document(required_frontmatter(), "Do not import"), + encoding="utf-8", + ) + source = tmp_path / "memory.md" + source.symlink_to(outside) + + with patch.object(Path, "is_symlink", return_value=False): + with pytest.raises(ValueError, match="symbolic link"): + AgentMemory._read_markdown_file(source) + + def test_legacy_dict_import_behavior_is_unchanged(): memory = AgentMemory() data = { From c77ce9394ab1007c2a445b37b2fa2699e1603220 Mon Sep 17 00:00:00 2001 From: Saurabh Meena <127095776+SaurabhScripts@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:21:52 +0530 Subject: [PATCH 002/129] feat(context): add ContextGraph Markdown round-trip --- docs/guides/context-graphs.md | 23 + docs/reference/context.md | 4 +- semantica/context/context_graph.py | 734 ++++++++++++++++++- tests/context/test_context_graph_markdown.py | 482 ++++++++++++ 4 files changed, 1231 insertions(+), 12 deletions(-) create mode 100644 tests/context/test_context_graph_markdown.py diff --git a/docs/guides/context-graphs.md b/docs/guides/context-graphs.md index 8521db7e..b23a3b85 100644 --- a/docs/guides/context-graphs.md +++ b/docs/guides/context-graphs.md @@ -436,6 +436,29 @@ d = graph.to_dict() # d["statistics"] → {"node_count": int, "edge_count": int} ``` +For a human-editable, version-control-friendly representation, save a Markdown +directory instead: + +```python +graph.save_to_file("context_graph/", format="markdown") + +restored = ContextGraph(advanced_analytics=True) +restored.load_from_file("context_graph/", format="markdown") +``` + +The directory contains a versioned `graph.md` manifest for graph identity, +relationships, and cross-graph link descriptors, plus one file per node under +`nodes/`. A node's content is its Markdown body; its ID, type, properties, +metadata, and temporal validity are YAML frontmatter. Node, edge, family, graph, +and cross-graph link IDs are preserved across round trips. + +Markdown loading uses replacement semantics, like `from_dict()`: it parses and +validates the complete directory before replacing the current graph. Invalid YAML, +duplicate IDs, dangling edge endpoints, unsupported versions, and unsafe symbolic +links fail without partially mutating the graph. Re-exporting to an existing managed +directory atomically replaces it, removing stale node files. To avoid accidental data +loss, a non-empty directory without the ContextGraph manifest is never replaced. + If the graph had cross-graph links created with `link_graph()`, call `resolve_links()` after loading to restore live navigation — object references cannot be serialized, so they must be reconnected manually: ```python diff --git a/docs/reference/context.md b/docs/reference/context.md index 18950e9e..4551eefa 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -435,8 +435,8 @@ print("Nodes: {}, Edges: {}".format(stats["node_count"], stats["edge_count"])) | `query(query, skip, limit)` | `List[Dict]` | Full-text search over node content | | `stats()` | `Dict` | Node/edge counts, type breakdowns, graph density | | `density()` | `float` | Graph density score | -| `save_to_file(path)` | `None` | Persist graph to JSON | -| `load_from_file(path)` | `None` | Load graph from JSON | +| `save_to_file(path, format="json")` | `None` | Persist graph as JSON or a Markdown directory | +| `load_from_file(path, format="json")` | `None` | Replace graph state from JSON or a Markdown directory | | `build_from_conversations(conversations, link_entities)` | `Dict` | Build graph from conversation data | | `link_graph(other_graph, source_node_id, target_node_id, link_type)` | `str` | Create cross-graph navigation link; returns `link_id` | | `navigate_to(link_id)` | `Tuple` | Follow a cross-graph link to `(target_graph, target_node_id)` | diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 06419759..4adefc77 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -106,20 +106,68 @@ Production Use Cases: """ from collections import defaultdict, deque +import copy from dataclasses import dataclass, field -from datetime import datetime, timezone +from datetime import date, datetime, timezone +import errno +import hashlib import json +import os +from pathlib import Path +import re +import shutil +import stat +import tempfile import threading import itertools from typing import Any, Dict, List, Optional, Set, Tuple, Union import uuid +import yaml + from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from ..utils.helpers import classify_path_distance from ..utils.skos import is_skos_hierarchy_edge, validate_skos_hierarchy from .entity_linker import EntityLinker + +class _UniqueKeySafeLoader(yaml.SafeLoader): + """Safe YAML loader that rejects ambiguous duplicate mapping keys.""" + + +def _construct_unique_mapping( + loader: _UniqueKeySafeLoader, node: yaml.MappingNode, deep: bool = False +) -> Dict[Any, Any]: + loader.flatten_mapping(node) + mapping = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + try: + duplicate = key in mapping + except TypeError as exc: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + "found an unhashable key", + key_node.start_mark, + ) from exc + if duplicate: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +_UniqueKeySafeLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_unique_mapping +) + + # Optional imports for advanced features try: from ..kg import ( @@ -424,6 +472,12 @@ class ContextGraph: Perfect for building intelligent AI agents that can learn from decisions! """ + _MARKDOWN_FORMAT = "semantica-context-graph" + _MARKDOWN_VERSION = 1 + _MARKDOWN_MANIFEST = "graph.md" + _MARKDOWN_NODES_DIRECTORY = "nodes" + _MARKDOWN_EXTENSIONS = frozenset({".md", ".markdown"}) + def __init__(self, config: Optional[Dict[str, Any]] = None, **kwargs): """ Initialize context graph with optional advanced features. @@ -977,14 +1031,21 @@ class ContextGraph: ) ) - def save_to_file(self, path: str) -> None: + def save_to_file( + self, path: Union[str, Path], format: str = "json" + ) -> None: """ - Save context graph to file (JSON format). + Save the context graph in JSON or Markdown format. Args: - path: File path to save to + path: JSON file path or Markdown export directory + format: Persistence format (``json`` or ``markdown``) """ - import json + normalized_format = self._normalize_persistence_format(format) + if normalized_format == "markdown": + self._save_markdown_directory(Path(path)) + self.logger.info(f"Saved context graph Markdown to {path}") + return with self._lock: @@ -1011,15 +1072,21 @@ class ContextGraph: self.logger.info(f"Saved context graph to {path}") - def load_from_file(self, path: str) -> None: + def load_from_file( + self, path: Union[str, Path], format: str = "json" + ) -> None: """ - Load context graph from file (JSON format). + Load the context graph from JSON or Markdown. Args: - path: File path to load from + path: JSON file path or Markdown export directory + format: Persistence format (``json`` or ``markdown``) """ - import json - import os + normalized_format = self._normalize_persistence_format(format) + if normalized_format == "markdown": + self._load_markdown_directory(Path(path)) + self.logger.info(f"Loaded context graph Markdown from {path}") + return if not os.path.exists(path): self.logger.warning(f"File not found: {path}") @@ -1079,6 +1146,653 @@ class ContextGraph: self.logger.info(f"Loaded context graph from {path}") + @staticmethod + def _normalize_persistence_format(format: str) -> str: + if not isinstance(format, str) or not format.strip(): + raise ValueError("Context graph persistence format must be a string.") + normalized = format.strip().lower() + if normalized not in {"json", "markdown"}: + raise ValueError( + f"Unsupported context graph persistence format: {format!r}. " + "Expected 'json' or 'markdown'." + ) + return normalized + + def _save_markdown_directory(self, destination: Path) -> None: + if not destination.name: + raise ValueError("Markdown export destination cannot be a filesystem root.") + if destination.is_symlink(): + raise ValueError( + f"Refusing to replace Markdown symbolic link: {destination}" + ) + if destination.exists() and not destination.is_dir(): + raise ValueError( + f"Markdown export destination is not a directory: {destination}" + ) + if destination.exists() and any(destination.iterdir()): + manifest = destination / self._MARKDOWN_MANIFEST + try: + document = self._read_markdown_file(manifest) + frontmatter, _ = self._parse_markdown_document( + document, str(manifest) + ) + is_managed = ( + frontmatter.get("format") == self._MARKDOWN_FORMAT + and not isinstance(frontmatter.get("version"), bool) + and frontmatter.get("version") == self._MARKDOWN_VERSION + ) + except (FileNotFoundError, ValueError): + is_managed = False + if not is_managed: + raise ValueError( + "Refusing to replace a non-empty directory that is not a " + f"managed ContextGraph export: {destination}" + ) + + manifest_document, node_documents = self._markdown_documents() + destination.parent.mkdir(parents=True, exist_ok=True) + staging_path = Path( + tempfile.mkdtemp( + dir=str(destination.parent), prefix=f".{destination.name}.staging-" + ) + ) + backup_path: Optional[Path] = None + try: + nodes_path = staging_path / self._MARKDOWN_NODES_DIRECTORY + nodes_path.mkdir() + self._write_staged_markdown( + staging_path / self._MARKDOWN_MANIFEST, manifest_document + ) + for filename, document in node_documents: + self._write_staged_markdown(nodes_path / filename, document) + + if destination.exists(): + backup_path = destination.parent / ( + f".{destination.name}.backup-{uuid.uuid4().hex}" + ) + os.replace(destination, backup_path) + try: + os.replace(staging_path, destination) + staging_path = None + except BaseException: + if backup_path is not None and not destination.exists(): + os.replace(backup_path, destination) + backup_path = None + raise + + if backup_path is not None: + shutil.rmtree(backup_path) + backup_path = None + finally: + if staging_path is not None: + shutil.rmtree(staging_path, ignore_errors=True) + if backup_path is not None and backup_path.exists(): + self.logger.warning( + "ContextGraph export left backup directory %s", backup_path + ) + + @staticmethod + def _write_staged_markdown(path: Path, document: str) -> None: + with path.open("x", encoding="utf-8") as output: + output.write(document) + output.flush() + os.fsync(output.fileno()) + + def _markdown_documents(self) -> Tuple[str, List[Tuple[str, str]]]: + with self._lock: + nodes = [ + { + "id": node.node_id, + "type": node.node_type, + "properties": copy.deepcopy(node.properties), + "metadata": copy.deepcopy(node.metadata), + "valid_from": node.valid_from, + "valid_until": node.valid_until, + "content": node.content, + } + for node in self.nodes.values() + ] + edges = [ + { + "id": edge.edge_id, + "family_id": edge.family_id or edge.edge_id, + "source": edge.source_id, + "target": edge.target_id, + "type": edge.edge_type, + "weight": edge.weight, + "metadata": copy.deepcopy(edge.metadata), + "valid_from": edge.valid_from, + "valid_until": edge.valid_until, + } + for edge in self.edges + ] + graph_id = self.graph_id + links_by_id = { + link_id: copy.deepcopy(link) + for link_id, link in self._unresolved_links.items() + } + for link_id, ( + other_graph, + source_node_id, + target_node_id, + ) in self._linked_graphs.items(): + links_by_id[link_id] = { + "link_id": link_id, + "source_node_id": source_node_id, + "target_node_id": target_node_id, + "other_graph_id": other_graph.graph_id, + } + + edges.sort( + key=lambda edge: ( + str(edge["id"]), + str(edge["source"]), + str(edge["target"]), + str(edge["type"]), + ) + ) + links = sorted( + links_by_id.values(), key=lambda link: str(link.get("link_id", "")) + ) + manifest = { + "format": self._MARKDOWN_FORMAT, + "version": self._MARKDOWN_VERSION, + "graph_id": graph_id, + "edges": edges, + "links": links, + } + manifest_body = ( + "# Context Graph\n\n" + "Graph relationships are stored in frontmatter. Node content is in " + "the `nodes` directory.\n" + ) + manifest_document = self._render_markdown_document( + manifest, manifest_body, "graph manifest" + ) + + node_documents = [] + filenames = set() + for node in sorted(nodes, key=lambda item: str(item["id"])): + filename = self._node_markdown_filename(node["id"]) + normalized_filename = filename.casefold() + if normalized_filename in filenames: + raise ValueError( + f"Cannot export ContextGraph: duplicate filename {filename!r}." + ) + filenames.add(normalized_filename) + content = node.pop("content") + node_documents.append( + ( + filename, + self._render_markdown_document( + node, content, f"node {node['id']!r}" + ), + ) + ) + return manifest_document, node_documents + + @staticmethod + def _node_markdown_filename(node_id: Any) -> str: + if not isinstance(node_id, str) or not node_id.strip(): + raise ValueError("Cannot export a ContextGraph node without a string ID.") + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", node_id) + slug = re.sub(r"-+", "-", slug).strip("._-")[:80].rstrip("._-") + slug = slug or "node" + digest = hashlib.sha256(node_id.encode("utf-8")).hexdigest()[:12] + return f"{slug}--{digest}.md" + + @classmethod + def _render_markdown_document( + cls, frontmatter: Dict[str, Any], body: str, source: str + ) -> str: + if not isinstance(body, str): + raise ValueError(f"Cannot export {source}: Markdown body must be a string.") + canonical = cls._canonical_markdown_value(frontmatter, source) + try: + yaml_text = yaml.safe_dump( + canonical, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + ) + except yaml.YAMLError as exc: + raise ValueError( + f"Cannot export {source}: metadata is not YAML serializable." + ) from exc + return f"---\n{yaml_text}---\n\n{body}" + + @classmethod + def _canonical_markdown_value( + cls, value: Any, source: str, ancestors: Optional[Set[int]] = None + ) -> Any: + ancestors = set() if ancestors is None else ancestors + if isinstance(value, dict): + if any(not isinstance(key, str) for key in value): + raise ValueError( + f"Invalid Markdown metadata in {source}: " + "mapping keys must be strings." + ) + identity = id(value) + if identity in ancestors: + raise ValueError( + f"Invalid Markdown metadata in {source}: " + "values cannot contain cycles." + ) + ancestors.add(identity) + try: + return { + key: cls._canonical_markdown_value( + value[key], source, ancestors + ) + for key in sorted(value) + } + finally: + ancestors.remove(identity) + if isinstance(value, list): + identity = id(value) + if identity in ancestors: + raise ValueError( + f"Invalid Markdown metadata in {source}: " + "values cannot contain cycles." + ) + ancestors.add(identity) + try: + return [ + cls._canonical_markdown_value(item, source, ancestors) + for item in value + ] + finally: + ancestors.remove(identity) + if isinstance(value, tuple) or isinstance(value, set): + raise ValueError( + f"Invalid Markdown metadata in {source}: " + "tuples and sets are not supported." + ) + return value + + def _load_markdown_directory(self, source: Path) -> None: + manifest_document, node_documents = self._read_markdown_directory(source) + manifest, _ = self._parse_markdown_document( + manifest_document, str(source / self._MARKDOWN_MANIFEST) + ) + graph_id, edges, links = self._parse_markdown_manifest(manifest, source) + + nodes_by_id: Dict[str, ContextNode] = {} + for node_source, document in node_documents: + frontmatter, body = self._parse_markdown_document(document, node_source) + node = self._parse_markdown_node(frontmatter, body, node_source) + if node.node_id in nodes_by_id: + raise ValueError( + f"Duplicate Markdown node ID {node.node_id!r} in {node_source}." + ) + nodes_by_id[node.node_id] = node + + missing_endpoints = sorted( + { + endpoint + for edge in edges + for endpoint in (edge.source_id, edge.target_id) + if endpoint not in nodes_by_id + } + ) + if missing_endpoints: + missing = ", ".join(repr(endpoint) for endpoint in missing_endpoints) + raise ValueError( + f"Invalid ContextGraph Markdown: edge endpoint(s) {missing} " + "do not have node files." + ) + + hierarchy_edges = [ + { + "source": edge.source_id, + "target": edge.target_id, + "type": edge.edge_type, + } + for edge in edges + if is_skos_hierarchy_edge(edge.to_dict()) + ] + if hierarchy_edges: + validate_skos_hierarchy(hierarchy_edges, []) + + unresolved_links = {} + for link in links: + link_id = link["link_id"] + if link_id in unresolved_links: + raise ValueError( + f"Duplicate cross-graph link ID {link_id!r} in graph manifest." + ) + if link["source_node_id"] not in nodes_by_id: + raise ValueError( + f"Cross-graph link {link_id!r} references missing source node " + f"{link['source_node_id']!r}." + ) + unresolved_links[link_id] = link + + adjacency: Dict[str, List[ContextEdge]] = defaultdict(list) + node_type_index: Dict[str, Set[str]] = defaultdict(set) + edge_type_index: Dict[str, List[ContextEdge]] = defaultdict(list) + for node in nodes_by_id.values(): + node_type_index[node.node_type].add(node.node_id) + for edge in edges: + adjacency[edge.source_id].append(edge) + edge_type_index[edge.edge_type].append(edge) + + with self._lock: + self.graph_id = graph_id + self.nodes.clear() + self.nodes.update(nodes_by_id) + self.edges.clear() + self.edges.extend(edges) + self._adjacency.clear() + self._adjacency.update(adjacency) + self.node_type_index.clear() + self.node_type_index.update(node_type_index) + self.edge_type_index.clear() + self.edge_type_index.update(edge_type_index) + self._linked_graphs.clear() + self._unresolved_links.clear() + self._unresolved_links.update(unresolved_links) + self._analytics_cache.clear() + + if self.mutation_callback and not self._suspend_mutation_callback: + try: + self.mutation_callback( + "RELOAD_GRAPH", + graph_id, + {"node_count": len(nodes_by_id), "edge_count": len(edges)}, + ) + except Exception as exc: + self.logger.warning( + "Audit trail callback failed for Markdown graph load: %s", exc + ) + + def _read_markdown_directory( + self, source: Path + ) -> Tuple[str, List[Tuple[str, str]]]: + if source.is_symlink(): + raise ValueError(f"Refusing to import Markdown symbolic link: {source}") + if not source.exists(): + raise FileNotFoundError( + f"ContextGraph Markdown import path does not exist: {source}" + ) + if not source.is_dir(): + raise ValueError( + f"ContextGraph Markdown import path is not a directory: {source}" + ) + + manifest_path = source / self._MARKDOWN_MANIFEST + manifest_document = self._read_markdown_file(manifest_path) + nodes_path = source / self._MARKDOWN_NODES_DIRECTORY + if nodes_path.is_symlink(): + raise ValueError(f"Refusing to import Markdown symbolic link: {nodes_path}") + if not nodes_path.is_dir(): + raise ValueError( + f"ContextGraph Markdown nodes directory is missing: {nodes_path}" + ) + + node_paths = [] + for path in nodes_path.iterdir(): + if path.is_symlink(): + raise ValueError(f"Refusing to import Markdown symbolic link: {path}") + if path.suffix.lower() not in self._MARKDOWN_EXTENSIONS: + continue + if not path.is_file(): + raise ValueError(f"Markdown node path is not a regular file: {path}") + node_paths.append(path) + node_paths.sort(key=lambda path: (path.name.casefold(), path.name)) + return manifest_document, [ + (str(path), self._read_markdown_file(path)) for path in node_paths + ] + + @staticmethod + def _read_markdown_file(path: Path) -> str: + if path.is_symlink(): + raise ValueError(f"Refusing to import Markdown symbolic link: {path}") + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as exc: + if exc.errno == errno.ELOOP or path.is_symlink(): + raise ValueError( + f"Refusing to import Markdown symbolic link: {path}" + ) from exc + if exc.errno == errno.ENOENT: + raise FileNotFoundError(f"Markdown file is missing: {path}") from exc + raise OSError( + exc.errno, + f"Failed to read Markdown file {path}: {exc.strerror or str(exc)}", + exc.filename or str(path), + ) from exc + + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError(f"Markdown path is not a regular file: {path}") + with os.fdopen(descriptor, "r", encoding="utf-8") as input_file: + descriptor = None + return input_file.read() + finally: + if descriptor is not None: + os.close(descriptor) + + @staticmethod + def _parse_markdown_document( + document: str, source: str + ) -> Tuple[Dict[str, Any], str]: + lines = document.splitlines(keepends=True) + if not lines or lines[0].rstrip("\r\n") != "---": + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + "document must start with '---'." + ) + closing_index = next( + ( + index + for index, line in enumerate(lines[1:], start=1) + if line.rstrip("\r\n") == "---" + ), + None, + ) + if closing_index is None: + raise ValueError( + f"Invalid Markdown frontmatter in {source}: missing closing '---'." + ) + try: + loaded = yaml.load( + "".join(lines[1:closing_index]), Loader=_UniqueKeySafeLoader + ) + except yaml.YAMLError as exc: + raise ValueError( + f"Invalid Markdown frontmatter in {source}: {exc}" + ) from exc + frontmatter = {} if loaded is None else loaded + if not isinstance(frontmatter, dict): + raise ValueError( + f"Invalid Markdown frontmatter in {source}: expected a YAML mapping." + ) + if any(not isinstance(key, str) for key in frontmatter): + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + "field names must be strings." + ) + + body = "".join(lines[closing_index + 1 :]) + if body.startswith("\r\n"): + body = body[2:] + elif body.startswith("\n"): + body = body[1:] + return frontmatter, body + + def _parse_markdown_manifest( + self, manifest: Dict[str, Any], source: Path + ) -> Tuple[str, List[ContextEdge], List[Dict[str, str]]]: + manifest_source = str(source / self._MARKDOWN_MANIFEST) + if manifest.get("format") != self._MARKDOWN_FORMAT: + raise ValueError( + f"Invalid ContextGraph Markdown manifest in {manifest_source}: " + f"'format' must be {self._MARKDOWN_FORMAT!r}." + ) + version = manifest.get("version") + if isinstance(version, bool) or version != self._MARKDOWN_VERSION: + raise ValueError( + f"Unsupported ContextGraph Markdown version {version!r} in " + f"{manifest_source}; expected {self._MARKDOWN_VERSION}." + ) + graph_id = self._required_markdown_string( + manifest.get("graph_id"), "graph_id", manifest_source + ) + + raw_edges = manifest.get("edges", []) + if not isinstance(raw_edges, list): + raise ValueError( + f"Invalid ContextGraph Markdown manifest in {manifest_source}: " + "'edges' must be a list." + ) + edges = [] + edge_ids = set() + for index, raw_edge in enumerate(raw_edges): + edge_source = f"{manifest_source} edge[{index}]" + edge = self._parse_markdown_edge(raw_edge, edge_source) + if edge.edge_id in edge_ids: + raise ValueError( + f"Duplicate Markdown edge ID {edge.edge_id!r} in {edge_source}." + ) + edge_ids.add(edge.edge_id) + edges.append(edge) + + raw_links = manifest.get("links", []) + if not isinstance(raw_links, list): + raise ValueError( + f"Invalid ContextGraph Markdown manifest in {manifest_source}: " + "'links' must be a list." + ) + links = [ + self._parse_markdown_link(link, f"{manifest_source} link[{index}]") + for index, link in enumerate(raw_links) + ] + return graph_id, edges, links + + def _parse_markdown_node( + self, frontmatter: Dict[str, Any], body: str, source: str + ) -> ContextNode: + node_id = self._required_markdown_string(frontmatter.get("id"), "id", source) + node_type = self._required_markdown_string( + frontmatter.get("type"), "type", source + ) + properties = self._markdown_mapping( + frontmatter.get("properties", {}), "properties", source + ) + metadata = self._markdown_mapping( + frontmatter.get("metadata", {}), "metadata", source + ) + return ContextNode( + node_id=node_id, + node_type=node_type, + content=body, + properties=properties, + metadata=metadata, + valid_from=self._markdown_temporal_value( + frontmatter.get("valid_from"), "valid_from", source + ), + valid_until=self._markdown_temporal_value( + frontmatter.get("valid_until"), "valid_until", source + ), + ) + + def _parse_markdown_edge(self, raw_edge: Any, source: str) -> ContextEdge: + if not isinstance(raw_edge, dict): + raise ValueError(f"Invalid Markdown edge in {source}: expected a mapping.") + edge_id = self._required_markdown_string(raw_edge.get("id"), "id", source) + family_value = raw_edge.get("family_id", raw_edge.get("familyId", edge_id)) + family_id = self._required_markdown_string(family_value, "family_id", source) + source_id = self._required_markdown_string( + raw_edge.get("source"), "source", source + ) + target_id = self._required_markdown_string( + raw_edge.get("target"), "target", source + ) + edge_type = self._required_markdown_string(raw_edge.get("type"), "type", source) + weight = raw_edge.get("weight", 1.0) + if isinstance(weight, bool) or not isinstance(weight, (int, float)): + raise ValueError( + f"Invalid Markdown edge in {source}: 'weight' must be a number." + ) + metadata = self._markdown_mapping( + raw_edge.get("metadata", {}), "metadata", source + ) + return ContextEdge( + edge_id=edge_id, + family_id=family_id, + source_id=source_id, + target_id=target_id, + edge_type=edge_type, + weight=float(weight), + metadata=metadata, + valid_from=self._markdown_temporal_value( + raw_edge.get("valid_from"), "valid_from", source + ), + valid_until=self._markdown_temporal_value( + raw_edge.get("valid_until"), "valid_until", source + ), + ) + + def _parse_markdown_link(self, raw_link: Any, source: str) -> Dict[str, str]: + if not isinstance(raw_link, dict): + raise ValueError( + f"Invalid cross-graph link in {source}: expected a mapping." + ) + return { + field_name: self._required_markdown_string( + raw_link.get(field_name), field_name, source + ) + for field_name in ( + "link_id", + "source_node_id", + "target_node_id", + "other_graph_id", + ) + } + + @staticmethod + def _required_markdown_string(value: Any, field_name: str, source: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + f"'{field_name}' must be a non-empty string." + ) + return value + + @classmethod + def _markdown_mapping( + cls, value: Any, field_name: str, source: str + ) -> Dict[str, Any]: + if value is None: + return {} + if not isinstance(value, dict): + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + f"'{field_name}' must be a mapping." + ) + canonical = cls._canonical_markdown_value(value, source) + return dict(canonical) + + @staticmethod + def _markdown_temporal_value( + value: Any, field_name: str, source: str + ) -> Optional[str]: + if value is None: + return None + if isinstance(value, (date, datetime)): + return value.isoformat() + if isinstance(value, str) and value.strip(): + return value + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + f"'{field_name}' must be an ISO-8601 string." + ) + + def find_node(self, node_id: str) -> Optional[Dict[str, Any]]: """Find a node by ID.""" with self._lock: diff --git a/tests/context/test_context_graph_markdown.py b/tests/context/test_context_graph_markdown.py new file mode 100644 index 00000000..a5e33210 --- /dev/null +++ b/tests/context/test_context_graph_markdown.py @@ -0,0 +1,482 @@ +from pathlib import Path + +import pytest +import yaml + +import semantica.context.context_graph as context_graph_module +from semantica.context.context_graph import ContextEdge, ContextGraph, ContextNode + + +def _read_markdown(path: Path): + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + closing_index = next( + index + for index, line in enumerate(lines[1:], start=1) + if line.rstrip("\r\n") == "---" + ) + frontmatter = yaml.safe_load("".join(lines[1:closing_index])) or {} + body = "".join(lines[closing_index + 1 :]) + if body.startswith("\r\n"): + body = body[2:] + elif body.startswith("\n"): + body = body[1:] + return frontmatter, body + + +def _write_markdown(path: Path, frontmatter, body: str) -> None: + yaml_text = yaml.safe_dump( + frontmatter, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + ) + path.write_text(f"---\n{yaml_text}---\n\n{body}", encoding="utf-8") + + +def _node_file(export_path: Path, node_id: str) -> Path: + for path in (export_path / "nodes").glob("*.md"): + frontmatter, _ = _read_markdown(path) + if frontmatter.get("id") == node_id: + return path + raise AssertionError(f"No Markdown file found for node {node_id!r}") + + +def _normalized_state(graph: ContextGraph): + links = {link_id: dict(link) for link_id, link in graph._unresolved_links.items()} + for link_id, ( + other_graph, + source_node_id, + target_node_id, + ) in graph._linked_graphs.items(): + links[link_id] = { + "link_id": link_id, + "source_node_id": source_node_id, + "target_node_id": target_node_id, + "other_graph_id": other_graph.graph_id, + } + return { + "graph_id": graph.graph_id, + "nodes": { + node_id: { + "type": node.node_type, + "content": node.content, + "properties": node.properties, + "metadata": node.metadata, + "valid_from": node.valid_from, + "valid_until": node.valid_until, + } + for node_id, node in graph.nodes.items() + }, + "edges": sorted( + ( + { + "id": edge.edge_id, + "family_id": edge.family_id, + "source": edge.source_id, + "target": edge.target_id, + "type": edge.edge_type, + "weight": edge.weight, + "metadata": edge.metadata, + "valid_from": edge.valid_from, + "valid_until": edge.valid_until, + } + for edge in graph.edges + ), + key=lambda edge: edge["id"], + ), + "links": links, + } + + +def _sample_graph(): + graph = ContextGraph(advanced_analytics=False) + graph.graph_id = "graph-primary" + graph._add_internal_node( + ContextNode( + node_id="policy/\u6771\u4eac", + node_type="Policy", + content="# Retention\n\nKeep evidence.\n---\n", + properties={"priority": 2, "nested": {"owner": "governance"}}, + metadata={"source": "manual", "tags": ["retention", "legal"]}, + valid_from="2026-01-01T00:00:00+00:00", + valid_until="2027-01-01T00:00:00+00:00", + ) + ) + graph._add_internal_node( + ContextNode( + node_id="evidence-1", + node_type="Evidence", + content="Original source", + properties={"checksum": "abc123"}, + metadata={"classification": "internal"}, + ) + ) + graph._add_internal_edge( + ContextEdge( + edge_id="edge-supports", + family_id="family-supports", + source_id="evidence-1", + target_id="policy/\u6771\u4eac", + edge_type="SUPPORTS", + weight=0.75, + metadata={"confidence": 0.9}, + valid_from="2026-02-01", + valid_until="2026-12-31", + ) + ) + + other = ContextGraph(advanced_analytics=False) + other.graph_id = "graph-secondary" + other.add_node("source-page", "Page", "Source page") + link_id = graph.link_graph(other, "policy/\u6771\u4eac", "source-page") + return graph, other, link_id + + +def _directory_contents(path: Path): + return { + str(file_path.relative_to(path)): file_path.read_bytes() + for file_path in path.rglob("*") + if file_path.is_file() + } + + +def test_markdown_round_trip_preserves_complete_graph_state(tmp_path): + graph, other, link_id = _sample_graph() + export_path = tmp_path / "context-graph" + + graph.save_to_file(export_path, format="markdown") + + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(export_path, format="markdown") + + assert _normalized_state(restored) == _normalized_state(graph) + assert restored.resolve_links({other.graph_id: other}) == 1 + linked_graph, entry_node = restored.navigate_to(link_id) + assert linked_graph is other + assert entry_node == "source-page" + + +def test_markdown_manual_node_and_edge_edits_are_imported(tmp_path): + graph, _, _ = _sample_graph() + export_path = tmp_path / "context-graph" + graph.save_to_file(export_path, format="markdown") + + node_path = _node_file(export_path, "policy/\u6771\u4eac") + node_frontmatter, _ = _read_markdown(node_path) + node_frontmatter["metadata"]["reviewed"] = True + _write_markdown(node_path, node_frontmatter, "# Updated policy\n") + + manifest_path = export_path / "graph.md" + manifest, manifest_body = _read_markdown(manifest_path) + manifest["edges"][0]["type"] = "VERIFIES" + manifest["edges"][0]["weight"] = 1.0 + manifest["edges"][0]["metadata"]["reviewed_by"] = "human" + _write_markdown(manifest_path, manifest, manifest_body) + + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(export_path, format="markdown") + + node = restored.nodes["policy/\u6771\u4eac"] + assert node.content == "# Updated policy\n" + assert node.metadata["reviewed"] is True + edge = restored.edges[0] + assert edge.edge_type == "VERIFIES" + assert edge.weight == 1.0 + assert edge.metadata["reviewed_by"] == "human" + + +def test_markdown_export_is_deterministic_and_removes_stale_nodes(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.graph_id = "deterministic-graph" + graph.add_node("kept", "Note", "Keep") + graph.add_node("removed", "Note", "Remove") + graph.add_edge("kept", "removed", "REFERENCES") + first = tmp_path / "first" + second = tmp_path / "second" + + graph.save_to_file(first, format="markdown") + graph.save_to_file(second, format="markdown") + assert _directory_contents(first) == _directory_contents(second) + + stale_path = _node_file(first, "removed") + graph.nodes.pop("removed") + graph.edges.clear() + graph.save_to_file(first, format="markdown") + + assert not stale_path.exists() + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(first, format="markdown") + assert set(restored.nodes) == {"kept"} + assert restored.edges == [] + + +def test_markdown_empty_graph_round_trip(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.graph_id = "empty-graph" + export_path = tmp_path / "empty" + + graph.save_to_file(export_path, format="markdown") + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(export_path, format="markdown") + + assert restored.graph_id == "empty-graph" + assert restored.nodes == {} + assert restored.edges == [] + + +@pytest.mark.parametrize( + "corruption, expected_error", + [ + ("unsupported-version", "Unsupported ContextGraph Markdown version"), + ("duplicate-edge", "Duplicate Markdown edge ID"), + ("dangling-edge", "do not have node files"), + ("duplicate-node", "Duplicate Markdown node ID"), + ("cyclic-skos", "SKOS hierarchy contains a cycle"), + ], +) +def test_invalid_markdown_does_not_mutate_existing_graph( + tmp_path, corruption, expected_error +): + source, _, _ = _sample_graph() + export_path = tmp_path / corruption + source.save_to_file(export_path, format="markdown") + + manifest_path = export_path / "graph.md" + manifest, manifest_body = _read_markdown(manifest_path) + if corruption == "unsupported-version": + manifest["version"] = 2 + elif corruption == "duplicate-edge": + manifest["edges"].append(dict(manifest["edges"][0])) + elif corruption == "dangling-edge": + manifest["edges"][0]["target"] = "missing-node" + elif corruption == "duplicate-node": + original = _node_file(export_path, "evidence-1") + (original.parent / "duplicate.md").write_bytes(original.read_bytes()) + elif corruption == "cyclic-skos": + manifest["edges"] = [ + { + "id": "broader-1", + "family_id": "broader-1", + "source": "evidence-1", + "target": "policy/\u6771\u4eac", + "type": "skos:broader", + "weight": 1.0, + "metadata": {}, + }, + { + "id": "broader-2", + "family_id": "broader-2", + "source": "policy/\u6771\u4eac", + "target": "evidence-1", + "type": "skos:broader", + "weight": 1.0, + "metadata": {}, + }, + ] + _write_markdown(manifest_path, manifest, manifest_body) + + target = ContextGraph(advanced_analytics=False) + target.add_node("sentinel", "Existing", "Do not replace") + before = _normalized_state(target) + + with pytest.raises(ValueError, match=expected_error): + target.load_from_file(export_path, format="markdown") + + assert _normalized_state(target) == before + + +def test_duplicate_yaml_keys_are_rejected(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + export_path = tmp_path / "graph" + graph.save_to_file(export_path, format="markdown") + node_path = _node_file(export_path, "node-1") + document = node_path.read_text(encoding="utf-8") + node_path.write_text(document.replace("id: node-1", "id: node-1\nid: duplicate")) + + with pytest.raises(ValueError, match="duplicate key 'id'"): + ContextGraph(advanced_analytics=False).load_from_file( + export_path, format="markdown" + ) + + +def test_markdown_export_refuses_unmanaged_nonempty_directory(tmp_path): + destination = tmp_path / "existing" + destination.mkdir() + marker = destination / "keep.txt" + marker.write_text("keep", encoding="utf-8") + + with pytest.raises(ValueError, match="not a managed ContextGraph export"): + ContextGraph(advanced_analytics=False).save_to_file( + destination, format="markdown" + ) + + assert marker.read_text(encoding="utf-8") == "keep" + + +def test_markdown_export_rejects_unrelated_graph_markdown_file(tmp_path): + destination = tmp_path / "existing" + destination.mkdir() + unrelated = destination / "graph.md" + _write_markdown(unrelated, {"title": "Unrelated notes"}, "Keep me") + + with pytest.raises(ValueError, match="not a managed ContextGraph export"): + ContextGraph(advanced_analytics=False).save_to_file( + destination, format="markdown" + ) + + assert unrelated.exists() + + +def test_markdown_export_preserves_manifest_inspection_errors(tmp_path, monkeypatch): + graph = ContextGraph(advanced_analytics=False) + destination = tmp_path / "existing" + graph.save_to_file(destination, format="markdown") + real_reader = ContextGraph._read_markdown_file + + def fail_manifest_read(path): + if path == destination / "graph.md": + raise PermissionError("permission denied") + return real_reader(path) + + monkeypatch.setattr( + ContextGraph, "_read_markdown_file", staticmethod(fail_manifest_read) + ) + + with pytest.raises(PermissionError, match="permission denied"): + graph.save_to_file(destination, format="markdown") + + +def test_markdown_export_restores_previous_directory_when_publish_fails( + tmp_path, monkeypatch +): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("original", "Note", "Original") + destination = tmp_path / "graph" + graph.save_to_file(destination, format="markdown") + original_contents = _directory_contents(destination) + graph.add_node("new", "Note", "New") + + real_replace = context_graph_module.os.replace + + def fail_staged_publish(source, target): + if ".staging-" in Path(source).name and Path(target) == destination: + raise OSError("simulated publish failure") + return real_replace(source, target) + + monkeypatch.setattr(context_graph_module.os, "replace", fail_staged_publish) + + with pytest.raises(OSError, match="simulated publish failure"): + graph.save_to_file(destination, format="markdown") + + assert _directory_contents(destination) == original_contents + assert not list(tmp_path.glob(".graph.staging-*")) + assert not list(tmp_path.glob(".graph.backup-*")) + + +def test_markdown_load_rebuilds_indexes_and_emits_one_reload_event(tmp_path): + source, _, _ = _sample_graph() + destination = tmp_path / "graph" + source.save_to_file(destination, format="markdown") + events = [] + target = ContextGraph( + advanced_analytics=False, + mutation_callback=lambda *event: events.append(event), + ) + + target.load_from_file(destination, format="markdown") + + assert target.node_type_index["Policy"] == {"policy/\u6771\u4eac"} + assert target.edge_type_index["SUPPORTS"][0].edge_id == "edge-supports" + assert target._adjacency["evidence-1"][0].target_id == "policy/\u6771\u4eac" + assert events == [ + ( + "RELOAD_GRAPH", + "graph-primary", + {"node_count": len(target.nodes), "edge_count": len(target.edges)}, + ) + ] + + +def test_markdown_export_rejects_recursive_metadata(tmp_path): + recursive = {} + recursive["self"] = recursive + graph = ContextGraph(advanced_analytics=False) + graph._add_internal_node(ContextNode("node-1", "Note", "Body", metadata=recursive)) + + with pytest.raises(ValueError, match="values cannot contain cycles"): + graph.save_to_file(tmp_path / "graph", format="markdown") + + +def test_markdown_import_and_export_reject_symlinks(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + real_export = tmp_path / "real" + graph.save_to_file(real_export, format="markdown") + + directory_link = tmp_path / "directory-link" + try: + directory_link.symlink_to(real_export, target_is_directory=True) + except (NotImplementedError, OSError): + pytest.skip("Symbolic links are not available on this platform") + + with pytest.raises(ValueError, match="symbolic link"): + ContextGraph(advanced_analytics=False).load_from_file( + directory_link, format="markdown" + ) + with pytest.raises(ValueError, match="symbolic link"): + graph.save_to_file(directory_link, format="markdown") + + manifest_path = real_export / "graph.md" + manifest_target = tmp_path / "manifest.md" + manifest_target.write_bytes(manifest_path.read_bytes()) + manifest_path.unlink() + manifest_path.symlink_to(manifest_target) + with pytest.raises(ValueError, match="symbolic link"): + ContextGraph(advanced_analytics=False).load_from_file( + real_export, format="markdown" + ) + + +@pytest.mark.skipif( + not hasattr(context_graph_module.os, "O_NOFOLLOW"), + reason="O_NOFOLLOW is unavailable on this platform", +) +def test_markdown_import_nofollow_check_closes_symlink_race(tmp_path, monkeypatch): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + export_path = tmp_path / "graph" + graph.save_to_file(export_path, format="markdown") + node_path = _node_file(export_path, "node-1") + target = tmp_path / "target.md" + target.write_bytes(node_path.read_bytes()) + node_path.unlink() + node_path.symlink_to(target) + + real_is_symlink = Path.is_symlink + + def miss_precheck(path): + if path == node_path: + return False + return real_is_symlink(path) + + monkeypatch.setattr(Path, "is_symlink", miss_precheck) + + with pytest.raises(ValueError, match="symbolic link"): + ContextGraph(advanced_analytics=False).load_from_file( + export_path, format="markdown" + ) + + +def test_json_remains_default_and_unknown_format_is_rejected(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("node-1", "Note", "Body") + json_path = tmp_path / "graph.json" + graph.save_to_file(json_path) + + restored = ContextGraph(advanced_analytics=False) + restored.load_from_file(json_path) + assert "node-1" in restored.nodes + + with pytest.raises(ValueError, match="Unsupported context graph"): + graph.save_to_file(tmp_path / "graph", format="html") From dec05b907d1db444f6047c9321690229083fc059 Mon Sep 17 00:00:00 2001 From: Saurabh Meena <127095776+SaurabhScripts@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:36:15 +0530 Subject: [PATCH 003/129] fix(context): validate Markdown graph persistence --- semantica/context/context_graph.py | 31 ++++++++++++- tests/context/test_context_graph_markdown.py | 47 ++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 4adefc77..9d4e73cf 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -1291,6 +1291,24 @@ class ContextGraph: str(edge["type"]), ) ) + seen_edge_ids: Set[str] = set() + duplicate_edge_ids: Set[str] = set() + for edge in edges: + edge_id = edge["id"] + if not isinstance(edge_id, str) or not edge_id.strip(): + raise ValueError( + "Cannot export ContextGraph: every edge must have a string ID." + ) + if edge_id in seen_edge_ids: + duplicate_edge_ids.add(edge_id) + seen_edge_ids.add(edge_id) + if duplicate_edge_ids: + duplicates = ", ".join( + repr(edge_id) for edge_id in sorted(duplicate_edge_ids) + ) + raise ValueError( + f"Cannot export ContextGraph: duplicate edge ID(s): {duplicates}." + ) links = sorted( links_by_id.values(), key=lambda link: str(link.get("link_id", "")) ) @@ -1783,13 +1801,22 @@ class ContextGraph: ) -> Optional[str]: if value is None: return None - if isinstance(value, (date, datetime)): + if isinstance(value, datetime): + return _normalize_temporal_input(value) + if isinstance(value, date): return value.isoformat() if isinstance(value, str) and value.strip(): + try: + _normalize_temporal_input(value) + except ValueError as exc: + raise ValueError( + f"Invalid Markdown frontmatter in {source}: " + f"'{field_name}' must be a valid ISO-8601 string." + ) from exc return value raise ValueError( f"Invalid Markdown frontmatter in {source}: " - f"'{field_name}' must be an ISO-8601 string." + f"'{field_name}' must be a valid ISO-8601 string." ) diff --git a/tests/context/test_context_graph_markdown.py b/tests/context/test_context_graph_markdown.py index a5e33210..b2cd0c24 100644 --- a/tests/context/test_context_graph_markdown.py +++ b/tests/context/test_context_graph_markdown.py @@ -224,6 +224,21 @@ def test_markdown_empty_graph_round_trip(tmp_path): assert restored.edges == [] +def test_markdown_export_rejects_duplicate_edge_ids_before_writing(tmp_path): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("source", "Note", "Source") + graph.add_node("target", "Note", "Target") + graph.add_edge("source", "target", "REFERENCES") + graph.add_edge("source", "target", "REFERENCES") + export_path = tmp_path / "graph" + + assert graph.edges[0].edge_id == graph.edges[1].edge_id + with pytest.raises(ValueError, match=r"duplicate edge ID.*[0-9a-f-]+"): + graph.save_to_file(export_path, format="markdown") + + assert not export_path.exists() + + @pytest.mark.parametrize( "corruption, expected_error", [ @@ -285,6 +300,38 @@ def test_invalid_markdown_does_not_mutate_existing_graph( assert _normalized_state(target) == before +@pytest.mark.parametrize( + ("location", "field_name"), + [("node", "valid_from"), ("edge", "valid_until")], +) +def test_invalid_markdown_temporal_value_does_not_mutate_existing_graph( + tmp_path, location, field_name +): + source, _, _ = _sample_graph() + export_path = tmp_path / location + source.save_to_file(export_path, format="markdown") + + if location == "node": + document_path = _node_file(export_path, "policy/\u6771\u4eac") + else: + document_path = export_path / "graph.md" + frontmatter, body = _read_markdown(document_path) + if location == "node": + frontmatter[field_name] = "not-a-date" + else: + frontmatter["edges"][0][field_name] = "not-a-date" + _write_markdown(document_path, frontmatter, body) + + target = ContextGraph(advanced_analytics=False) + target.add_node("sentinel", "Existing", "Do not replace") + before = _normalized_state(target) + + with pytest.raises(ValueError, match=rf"'{field_name}'.*valid ISO-8601"): + target.load_from_file(export_path, format="markdown") + + assert _normalized_state(target) == before + + def test_duplicate_yaml_keys_are_rejected(tmp_path): graph = ContextGraph(advanced_analytics=False) graph.add_node("node-1", "Note", "Body") From 0ca7b8d48961cffee51d072d18b303d7f68aa6dd Mon Sep 17 00:00:00 2001 From: ArmanGrewal007 Date: Mon, 10 Aug 2026 17:35:37 +0530 Subject: [PATCH 004/129] fix(methods): improve error handling in vector similarity calculations --- semantica/semantic_extract/methods.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/semantica/semantic_extract/methods.py b/semantica/semantic_extract/methods.py index 72898deb..06854619 100644 --- a/semantica/semantic_extract/methods.py +++ b/semantica/semantic_extract/methods.py @@ -247,6 +247,8 @@ def find_best_match_index(text: str, candidates: List[str]) -> Tuple[int, float] Find the best matching candidate index and score. Uses hybrid similarity approach: Exact -> Synonym -> Substring -> Embeddings -> Vector -> Fuzzy. Optimized for batch processing to avoid redundant embedding calculations. + Embedding/vector similarity stages are best-effort; if they fail, matching falls back + to remaining strategies instead of raising. Returns: Tuple[int, float]: (best_candidate_index, best_score). Index is -1 if no candidates. @@ -381,8 +383,10 @@ def find_best_match_index(text: str, candidates: List[str]) -> Tuple[int, float] if score > vector_score: vector_score = score vector_idx = i - except Exception: - pass + except Exception as e: + logger.debug(f"Vector similarity calculation failed: {e}") + vector_score = 0.0 + vector_idx = -1 if vector_score > best_score: best_score = vector_score From 6148975e8334df2e201a363cfdbe8e414b5e7a0a Mon Sep 17 00:00:00 2001 From: ArmanGrewal007 Date: Mon, 10 Aug 2026 18:36:37 +0530 Subject: [PATCH 005/129] fix(methods): enhance error logging for vector similarity calculations --- semantica/semantic_extract/methods.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/semantica/semantic_extract/methods.py b/semantica/semantic_extract/methods.py index 06854619..09d98b22 100644 --- a/semantica/semantic_extract/methods.py +++ b/semantica/semantic_extract/methods.py @@ -372,10 +372,12 @@ def find_best_match_index(text: str, candidates: List[str]) -> Tuple[int, float] vector_idx = -1 if nlp and nlp.vocab.vectors.shape[0] > 0: + failing_candidate_idx = -1 try: doc = nlp(text) if doc.vector_norm: for i, candidate in enumerate(candidates): + failing_candidate_idx = i if not candidate: continue cand_doc = nlp(candidate) if cand_doc.vector_norm: @@ -383,8 +385,12 @@ def find_best_match_index(text: str, candidates: List[str]) -> Tuple[int, float] if score > vector_score: vector_score = score vector_idx = i - except Exception as e: - logger.debug(f"Vector similarity calculation failed: {e}") + except Exception: + logger.debug( + "Vector similarity calculation failed at candidate index %s; continuing with fallback scoring.", + failing_candidate_idx if failing_candidate_idx >= 0 else "N/A", + exc_info=True, + ) vector_score = 0.0 vector_idx = -1 From 20781e8a9e44e9bed5b74efe542d24c04ed0e500 Mon Sep 17 00:00:00 2001 From: yulinlina Date: Mon, 10 Aug 2026 17:50:28 +0000 Subject: [PATCH 006/129] Add graph storage backend compatibility matrix (addresses #888) --- docs/storage-backends.md | 128 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 docs/storage-backends.md diff --git a/docs/storage-backends.md b/docs/storage-backends.md new file mode 100644 index 00000000..689002fb --- /dev/null +++ b/docs/storage-backends.md @@ -0,0 +1,128 @@ +# Graph storage backends and feature matrix + +Semantica separates graph modeling from physical storage. LPG backends are accessed through `graph_store` adapters; RDF backends are accessed through `triplet_store` adapters. + +This page is intentionally conservative: it distinguishes between an adapter existing, a feature being generally available with that model, and a backend needing user-supplied wiring. + +## Status labels + +- `built-in`: adapter implementation exists in Semantica core. +- `tested`: covered by automated integration fixtures or tests. +- `example-only`: usable example exists, but support is not asserted by integration tests. +- `interface/BYO`: interface or integration point exists; bring your own backend wiring. + +## Adapter inventory + +| Backend | Model | Adapter | Status | Reference | +| --- | --- | --- | --- | --- | +| Neo4j | LPG | `semantica.graph_store.Neo4jGraphStore` | built-in | `cookbook/introduction/09_Graph_Store.ipynb` | +| Amazon Neptune | LPG | `semantica.graph_store.NeptuneGraphStore` | built-in | `cookbook/introduction/21_Amazon_Neptune_Store.ipynb` | +| Apache AGE | LPG | `semantica.graph_store.AgeGraphStore` | built-in | `docs/graph_stores/apache_age.md` | +| RDF4J | RDF | `semantica.triplet_store.RDF4JStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` | +| Apache Jena | RDF | `semantica.triplet_store.JenaStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` | +| Blazegraph | RDF | `semantica.triplet_store.BlazegraphStore` | built-in | `cookbook/introduction/20_Triplet_Store.ipynb` | +| Anzo | RDF | `semantica.triplet_store.AnzoStore` | interface/BYO | `cookbook/introduction/20_Triplet_Store.ipynb` | + +## Feature matrix + +`Yes` means the capability is expected to work with the adapter and graph model. `Partial` means the capability works with model-specific constraints. `BYO` means the user must supply or validate wiring for the backend. + +| Backend | Model | Ingestion | Context graph construction | Reasoning/analytics | Provenance | Known limitations | +| --- | --- | --- | --- | --- | --- | --- | +| Neo4j | LPG | Yes | Yes | Yes | Partial | Provenance and context metadata are stored as node and edge properties; relationship properties and stable node identifiers are required. | +| Amazon Neptune | LPG | Yes | Yes | Partial | Partial | Use the property-graph endpoint; AWS auth, VPC, and endpoint configuration can affect local tests. Provenance depends on node/edge properties. | +| Apache AGE | LPG | Yes | Yes | Partial | Partial | Runs through PostgreSQL/AGE; Cypher compatibility and property handling can differ from standalone LPG engines. | +| RDF4J | RDF | Yes | Partial | Partial | Partial | Context separation relies on named graphs; triple-level provenance may require reification or graph-level metadata. | +| Apache Jena | RDF | Yes | Partial | Partial | Partial | Named graphs are needed for context separation; backend configuration and transaction behavior matter. | +| Blazegraph | RDF | Yes | Partial | Partial | Partial | Use quads/named graphs for context; IRI stability and graph naming matter for provenance. | +| Anzo | RDF | BYO | BYO | BYO | BYO | Anzo deployments are environment-specific; validate repository/graph naming, named-graph support, and provenance mapping. | + +## RDF and LPG differences + +- LPG backends store context and provenance as graph elements and properties. If a backend does not support relationship properties, some provenance patterns may be degraded. +- RDF backends rely on IRIs, named graphs, and optional reification. Context graphs and provenance are easiest to preserve when the store supports named graphs/quads. +- Ingestion works across both models, but the physical representation differs: LPG stores nodes/edges directly, while RDF stores subject-predicate-object statements. +- Reasoning and analytics should be validated against the adapter's query capabilities, especially for path traversal, property filters, and named-graph queries. + +## Minimal connection examples + +Prefer the referenced notebook cells for a working setup. The examples below show the intended adapter entrypoints, not a universal connection DSL. + +### Neo4j + +```python +from semantica.graph_store import Neo4jGraphStore + +store = Neo4jGraphStore( + uri='bolt://localhost:7687', + username='neo4j', + password='password' +) +``` + +### Amazon Neptune + +```python +from semantica.graph_store import NeptuneGraphStore + +store = NeptuneGraphStore( + host='your-neptune-endpoint', + port=8182 +) +``` + +### Apache AGE + +```python +from semantica.graph_store import AgeGraphStore + +store = AgeGraphStore( + dsn='postgresql://user:password@localhost:5432/semantica', + graph='semantica' +) +``` + +### RDF4J + +```python +from semantica.triplet_store import RDF4JStore + +store = RDF4JStore( + url='http://localhost:8080/rdf4j-server', + repository='semantica' +) +``` + +### Apache Jena + +```python +from semantica.triplet_store import JenaStore + +store = JenaStore( + url='http://localhost:3030', + dataset='semantica' +) +``` + +### Blazegraph + +```python +from semantica.triplet_store import BlazegraphStore + +store = BlazegraphStore( + url='http://localhost:9999/blazegraph/sparql' +) +``` + +### Anzo + +```python +from semantica.triplet_store import AnzoStore + +store = AnzoStore( + url='http://anzo-host:10000', + repository='semantica' +) +``` + +Replace hostnames, ports, repositories, graphs, and credentials with values from your environment. For regulated or self-hosted deployments, keep credentials in environment variables or secret storage rather than source code. From 1b9bb4c345b017b97b57ef30ce8fe8361ea65dac Mon Sep 17 00:00:00 2001 From: Sameer6305 Date: Sat, 8 Aug 2026 15:44:26 +0530 Subject: [PATCH 007/129] test(context): harden Markdown import symlink coverage --- semantica/context/agent_memory.py | 8 ++++++ tests/context/test_agent_memory_markdown.py | 29 ++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/semantica/context/agent_memory.py b/semantica/context/agent_memory.py index 93c8b568..7fef3b38 100644 --- a/semantica/context/agent_memory.py +++ b/semantica/context/agent_memory.py @@ -1944,6 +1944,14 @@ class AgentMemory: if file_path.is_symlink(): raise ValueError(f"Refusing to import Markdown symbolic link: {file_path}") + # Defend the open itself against a symlink introduced after the + # is_symlink() check above (TOCTOU). O_NOFOLLOW is used where the + # platform supports it, causing os.open() to fail with ELOOP if the + # target became a symlink in the meantime. On platforms without + # O_NOFOLLOW (e.g. Windows), os.open() follows symlinks and there is + # no kernel-level way to close this race; the preceding + # is_symlink() check is the only protection there, so the strength + # of the final-open race protection differs by platform. flags = os.O_RDONLY nofollow_flag = getattr(os, "O_NOFOLLOW", 0) flags |= nofollow_flag diff --git a/tests/context/test_agent_memory_markdown.py b/tests/context/test_agent_memory_markdown.py index d403504c..807e5858 100644 --- a/tests/context/test_agent_memory_markdown.py +++ b/tests/context/test_agent_memory_markdown.py @@ -55,6 +55,25 @@ def markdown_document(frontmatter, body=""): return f"---\n{yaml_text}---\n\n{body}" +def _require_symlink_support(tmp_path): + """Skip the test if this environment cannot create symbolic links. + + Symlink creation can be unavailable even on POSIX (e.g. restricted + containers) and commonly requires elevated privilege or Developer Mode + on Windows. Probe actual capability instead of assuming based on + platform, so these tests still run wherever symlinks genuinely work. + """ + probe_target = tmp_path / ".symlink_probe_target" + probe_link = tmp_path / ".symlink_probe_link" + probe_target.write_text("", encoding="utf-8") + try: + probe_link.symlink_to(probe_target) + except OSError as exc: + pytest.skip(f"environment cannot create symbolic links: {exc}") + probe_link.unlink() + probe_target.unlink() + + def required_frontmatter(memory_id="mem_test", **overrides): frontmatter = { "id": memory_id, @@ -702,6 +721,7 @@ def test_markdown_string_path_inspection_errors_are_actionable(): @pytest.mark.parametrize("use_string_path", [False, True]) def test_markdown_import_rejects_symlinked_file(tmp_path, use_string_path): + _require_symlink_support(tmp_path) outside = tmp_path / "outside.md" outside.write_text( markdown_document(required_frontmatter(), "Do not import"), @@ -720,16 +740,21 @@ def test_markdown_import_rejects_symlinked_file(tmp_path, use_string_path): @pytest.mark.parametrize("use_string_path", [False, True]) def test_markdown_import_rejects_broken_symlink(tmp_path, use_string_path): + _require_symlink_support(tmp_path) source = tmp_path / "missing-memory.md" source.symlink_to(tmp_path / "missing-target.md") payload = str(source) if use_string_path else source + memory = AgentMemory() with pytest.raises(ValueError, match="symbolic link"): - AgentMemory().import_data(payload, format="markdown") + memory.import_data(payload, format="markdown") + + assert memory.count() == 0 @pytest.mark.parametrize("use_string_path", [False, True]) def test_markdown_import_rejects_symlinked_directory(tmp_path, use_string_path): + _require_symlink_support(tmp_path) outside = tmp_path / "outside" outside.mkdir() (outside / "memory.md").write_text( @@ -748,6 +773,7 @@ def test_markdown_import_rejects_symlinked_directory(tmp_path, use_string_path): def test_markdown_import_rejects_symlinked_file_in_directory(tmp_path): + _require_symlink_support(tmp_path) outside = tmp_path / "outside.md" outside.write_text( markdown_document(required_frontmatter(), "Do not import"), @@ -766,6 +792,7 @@ def test_markdown_import_rejects_symlinked_file_in_directory(tmp_path): @pytest.mark.skipif(not hasattr(os, "O_NOFOLLOW"), reason="requires O_NOFOLLOW") def test_markdown_import_does_not_follow_symlink_raced_before_open(tmp_path): + _require_symlink_support(tmp_path) outside = tmp_path / "outside.md" outside.write_text( markdown_document(required_frontmatter(), "Do not import"), From a3d8064f3d397f6be2c45c60f2711a128bc0dbeb Mon Sep 17 00:00:00 2001 From: Saurabh Meena <127095776+SaurabhScripts@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:37:31 +0530 Subject: [PATCH 008/129] docs: add Markdown import hardening changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 244b45fc..9a8a8170 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Markdown import followed symbolic links even though Markdown export already refused to overwrite them** (#851, follow-up to #765, #786) by @SaurabhScripts + - `AgentMemory._read_markdown_path()` now rejects a symlink file, a broken symlink, or a symlinked directory supplied directly as an import path, and rejects any symlinked Markdown entry discovered while walking an import directory - before any parsing happens, so an import can no longer read a different file or directory than the path presented to the caller + - New `_read_markdown_file()` re-checks `is_symlink()` immediately before opening (closing the window between directory-scan validation and the actual read), opens with `O_NOFOLLOW` on platforms that support it, and verifies the resulting descriptor is a regular file via `fstat`/`S_ISREG` before reading, so a symlink swapped in after validation is still rejected rather than silently followed + - Documented the import restriction in `docs/reference/context.md`; added 8 tests to `tests/context/test_agent_memory_markdown.py` covering file/directory/broken-symlink rejection for both `str` and `Path` inputs, plus a simulated-race test proving the `O_NOFOLLOW` open still catches a symlink when the pre-open `is_symlink()` check is bypassed + - Disclosed limitation: `O_NOFOLLOW` isn't available on Windows, so the final open there relies solely on the pre-open `is_symlink()` check rather than a kernel-enforced guarantee against a race + - Any additional review follow-up commits land in this same PR/entry rather than as a separate changelog item + - **`DecisionEmbeddingPipeline.find_similar_decisions()` crashed with `AttributeError` for any `VectorStore` backend other than `inmemory`** (#842, closes #839) by @Sameer6305 - `_get_candidate_embeddings()` iterated `VectorStore.vectors`/`VectorStore.metadata` directly, internal dicts only populated for `backend="inmemory"`; every persistent backend (FAISS, Pinecone, Qdrant, Milvus, ...) raised `AttributeError`. It now fetches candidates via the backend-agnostic `VectorStore.search_vectors()`, reading metadata via a `res.get("metadata") or res.get("payload")` fallback for backends that key it differently - Backends such as FAISS don't return the raw vector for each hit; `find_similar_decisions()` and `_find_semantic_similar()` now fall back to the search-provided score (normalized from `distance` when present) as the semantic similarity for those candidates instead of computing cosine similarity against a zero placeholder vector From 55f7eba389c26714000f133e7c9dcab0d8ad5c99 Mon Sep 17 00:00:00 2001 From: Saurabh Meena <127095776+SaurabhScripts@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:37:31 +0530 Subject: [PATCH 009/129] fix(context): preserve Markdown publish errors --- semantica/context/context_graph.py | 26 +++++++++++++-- tests/context/test_context_graph_markdown.py | 35 ++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 9d4e73cf..dea62d66 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -1214,10 +1214,30 @@ class ContextGraph: try: os.replace(staging_path, destination) staging_path = None - except BaseException: + except BaseException as publish_error: if backup_path is not None and not destination.exists(): - os.replace(backup_path, destination) - backup_path = None + try: + os.replace(backup_path, destination) + except BaseException as restore_error: + self.logger.error( + "Failed to restore previous ContextGraph Markdown " + "export from %s after publish failure; preserving " + "the original publish error", + backup_path, + exc_info=( + type(restore_error), + restore_error, + restore_error.__traceback__, + ), + ) + add_note = getattr(publish_error, "add_note", None) + if add_note is not None: + add_note( + "Restoring the previous ContextGraph Markdown " + f"export also failed: {restore_error}" + ) + else: + backup_path = None raise if backup_path is not None: diff --git a/tests/context/test_context_graph_markdown.py b/tests/context/test_context_graph_markdown.py index b2cd0c24..18e54e51 100644 --- a/tests/context/test_context_graph_markdown.py +++ b/tests/context/test_context_graph_markdown.py @@ -421,6 +421,41 @@ def test_markdown_export_restores_previous_directory_when_publish_fails( assert not list(tmp_path.glob(".graph.backup-*")) +def test_markdown_export_preserves_publish_error_when_restore_fails( + tmp_path, monkeypatch, caplog +): + graph = ContextGraph(advanced_analytics=False) + graph.add_node("original", "Note", "Original") + destination = tmp_path / "graph" + graph.save_to_file(destination, format="markdown") + original_contents = _directory_contents(destination) + graph.add_node("new", "Note", "New") + + real_replace = context_graph_module.os.replace + + def fail_publish_and_restore(source, target): + source_path = Path(source) + if ".staging-" in source_path.name and Path(target) == destination: + raise OSError("simulated publish failure") + if ".backup-" in source_path.name and Path(target) == destination: + raise PermissionError("simulated restore failure") + return real_replace(source, target) + + monkeypatch.setattr(context_graph_module.os, "replace", fail_publish_and_restore) + caplog.set_level("ERROR") + + with pytest.raises(OSError, match="simulated publish failure"): + graph.save_to_file(destination, format="markdown") + + assert "preserving the original publish error" in caplog.text + assert "simulated restore failure" in caplog.text + assert not destination.exists() + backup_paths = list(tmp_path.glob(".graph.backup-*")) + assert len(backup_paths) == 1 + assert _directory_contents(backup_paths[0]) == original_contents + assert not list(tmp_path.glob(".graph.staging-*")) + + def test_markdown_load_rebuilds_indexes_and_emits_one_reload_event(tmp_path): source, _, _ = _sample_graph() destination = tmp_path / "graph" From f737f7267592fe1dcfe410e13281ec1813dcbe37 Mon Sep 17 00:00:00 2001 From: devansh121sinha Date: Tue, 11 Aug 2026 01:05:53 +0530 Subject: [PATCH 010/129] test(conflicts): add coverage for 4 resolution strategies and 3 conflict types --- tests/conflicts/test_conflicts.py | 193 +++++++++++++++++++++++++++++- 1 file changed, 192 insertions(+), 1 deletion(-) diff --git a/tests/conflicts/test_conflicts.py b/tests/conflicts/test_conflicts.py index 34dad4d2..1e69c2e6 100644 --- a/tests/conflicts/test_conflicts.py +++ b/tests/conflicts/test_conflicts.py @@ -238,6 +238,197 @@ class TestConflictsModule(unittest.TestCase): checklist = generator.export_investigation_checklist(guide, format="text") self.assertIn("INVESTIGATION GUIDE: c1", checklist) + def test_conflict_resolver_credibility_weighted(self): + """Test credibility-weighted resolution strategy (#865).""" + resolver = ConflictResolver() + + # Boost the credibility of "doc_trusted" so its value should win + # even though it only has one vote, vs. two lower-credibility votes. + resolver.source_tracker.source_credibility["doc_trusted"] = 1.0 + resolver.source_tracker.source_credibility["doc_flaky"] = 0.1 + + conflict = Conflict( + conflict_id="c_cred", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 30, 40], + sources=[ + {"document": "doc_flaky", "confidence": 0.9}, + {"document": "doc_flaky", "confidence": 0.9}, + {"document": "doc_trusted", "confidence": 0.9}, + ], + ) + + result = resolver.resolve_conflict(conflict, strategy="credibility_weighted") + self.assertTrue(result.resolved) + self.assertEqual(result.resolved_value, 40) + self.assertEqual(result.resolution_strategy, "credibility_weighted") + self.assertGreater(result.confidence, 0.0) + + def test_conflict_resolver_first_seen(self): + """Test first-seen resolution strategy (#865).""" + resolver = ConflictResolver() + + conflict = Conflict( + conflict_id="c_first", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[ + {"document": "doc1", "confidence": 0.9}, + {"document": "doc2", "confidence": 0.9}, + ], + ) + + result = resolver.resolve_conflict(conflict, strategy="first_seen") + self.assertTrue(result.resolved) + self.assertEqual(result.resolved_value, 30) # first value in the list + self.assertEqual(result.resolution_strategy, "first_seen") + self.assertEqual(result.sources_used, ["doc1"]) + + def test_conflict_resolver_manual_review(self): + """Test manual-review resolution strategy flags without resolving (#865).""" + resolver = ConflictResolver() + + conflict = Conflict( + conflict_id="c_manual", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[{"document": "doc1"}, {"document": "doc2"}], + severity="high", + ) + + result = resolver.resolve_conflict(conflict, strategy="manual_review") + self.assertFalse(result.resolved) + self.assertEqual(result.resolution_strategy, "manual_review") + self.assertTrue(result.metadata.get("requires_manual_review")) + self.assertEqual(result.metadata.get("severity"), "high") + + def test_conflict_resolver_expert_review(self): + """Test expert-review resolution strategy flags without resolving (#865).""" + resolver = ConflictResolver() + + conflict = Conflict( + conflict_id="c_expert", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[{"document": "doc1"}, {"document": "doc2"}], + severity="critical", + ) + + result = resolver.resolve_conflict(conflict, strategy="expert_review") + self.assertFalse(result.resolved) + self.assertEqual(result.resolution_strategy, "expert_review") + self.assertTrue(result.metadata.get("requires_expert_review")) + self.assertEqual(result.metadata.get("severity"), "critical") + + def test_conflict_detector_relationship_conflicts(self): + """Test relationship conflict detection (#865).""" + detector = ConflictDetector() + + relationships = [ + { + "id": "r1", + "source_id": "e1", + "target_id": "e2", + "type": "works_at", + "source": "doc1", + }, + { + "id": "r1", + "source_id": "e1", + "target_id": "e2", + "type": "founded", + "source": "doc2", + }, + ] + + conflicts = detector.detect_relationship_conflicts(relationships) + self.assertEqual(len(conflicts), 1) + conflict = conflicts[0] + self.assertEqual(conflict.conflict_type, ConflictType.RELATIONSHIP_CONFLICT) + self.assertEqual(conflict.relationship_id, "r1") + self.assertEqual(conflict.property_name, "type") + self.assertIn("works_at", conflict.conflicting_values) + self.assertIn("founded", conflict.conflicting_values) + + def test_conflict_detector_relationship_conflicts_no_conflict(self): + """Relationships with a single occurrence should not raise conflicts (#865).""" + detector = ConflictDetector() + + relationships = [ + {"id": "r1", "source_id": "e1", "target_id": "e2", "type": "works_at"}, + ] + + conflicts = detector.detect_relationship_conflicts(relationships) + self.assertEqual(len(conflicts), 0) + + def test_conflict_detector_temporal_conflicts(self): + """Test temporal conflict detection (#865).""" + detector = ConflictDetector() + + entities = [ + {"id": "e1", "founded": "1998", "source": "doc1", "confidence": 0.9}, + {"id": "e1", "founded": "2004", "source": "doc2", "confidence": 0.8}, + ] + + conflicts = detector.detect_temporal_conflicts(entities) + self.assertEqual(len(conflicts), 1) + conflict = conflicts[0] + self.assertEqual(conflict.conflict_type, ConflictType.TEMPORAL_CONFLICT) + self.assertEqual(conflict.entity_id, "e1") + self.assertEqual(conflict.property_name, "founded") + self.assertIn("1998", conflict.conflicting_values) + self.assertIn("2004", conflict.conflicting_values) + + def test_conflict_detector_temporal_conflicts_no_conflict(self): + """Matching temporal values across sources should not raise conflicts (#865).""" + detector = ConflictDetector() + + entities = [ + {"id": "e1", "founded": "1998", "source": "doc1"}, + {"id": "e1", "founded": "1998", "source": "doc2"}, + ] + + conflicts = detector.detect_temporal_conflicts(entities) + self.assertEqual(len(conflicts), 0) + + def test_conflict_detector_logical_conflicts(self): + """Test logical conflict detection for incompatible entity types (#865).""" + detector = ConflictDetector() + + entities = [ + {"id": "e1", "type": "Person", "source": "doc1"}, + {"id": "e1", "type": "Organization", "source": "doc2"}, + ] + + conflicts = detector.detect_logical_conflicts(entities) + self.assertEqual(len(conflicts), 1) + conflict = conflicts[0] + self.assertEqual(conflict.conflict_type, ConflictType.LOGICAL_CONFLICT) + self.assertEqual(conflict.entity_id, "e1") + self.assertEqual(conflict.severity, "critical") + self.assertIn("Person", conflict.conflicting_values) + self.assertIn("Organization", conflict.conflicting_values) + + def test_conflict_detector_logical_conflicts_compatible_types(self): + """Compatible/unrelated types should not raise logical conflicts (#865).""" + detector = ConflictDetector() + + entities = [ + {"id": "e1", "type": "Person", "source": "doc1"}, + {"id": "e1", "type": "Employee", "source": "doc2"}, + ] + + conflicts = detector.detect_logical_conflicts(entities) + self.assertEqual(len(conflicts), 0) + if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file From 92dc3304f8ad8e9506433aaebb251270923df570 Mon Sep 17 00:00:00 2001 From: devansh121sinha Date: Tue, 11 Aug 2026 01:32:43 +0530 Subject: [PATCH 011/129] =?UTF-8?q?test(conflicts):=20address=20Qodo=20rev?= =?UTF-8?q?iew=20=E2=80=94=20add=20analyzer=20tests,=20use=20validated=20s?= =?UTF-8?q?etter,=20fix=20newline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/conflicts/test_conflicts.py | 104 +++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/tests/conflicts/test_conflicts.py b/tests/conflicts/test_conflicts.py index 1e69c2e6..cf2e5e11 100644 --- a/tests/conflicts/test_conflicts.py +++ b/tests/conflicts/test_conflicts.py @@ -244,8 +244,10 @@ class TestConflictsModule(unittest.TestCase): # Boost the credibility of "doc_trusted" so its value should win # even though it only has one vote, vs. two lower-credibility votes. - resolver.source_tracker.source_credibility["doc_trusted"] = 1.0 - resolver.source_tracker.source_credibility["doc_flaky"] = 0.1 + # Use the validated setter rather than mutating the internal dict + # directly, so the test stays coupled to the public API surface. + resolver.source_tracker.set_source_credibility("doc_trusted", 1.0) + resolver.source_tracker.set_source_credibility("doc_flaky", 0.1) conflict = Conflict( conflict_id="c_cred", @@ -429,6 +431,104 @@ class TestConflictsModule(unittest.TestCase): conflicts = detector.detect_logical_conflicts(entities) self.assertEqual(len(conflicts), 0) + def test_conflict_analyzer_by_source_breakdown(self): + """Test the by_source breakdown of analyze_conflicts (#902).""" + analyzer = ConflictAnalyzer() + + conflicts = [ + Conflict( + conflict_id="c1", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[{"document": "doc1"}, {"document": "doc2"}], + severity="medium", + ), + Conflict( + conflict_id="c2", + conflict_type=ConflictType.TYPE_CONFLICT, + entity_id="e2", + property_name="type", + conflicting_values=["Person", "Org"], + sources=[{"document": "doc1"}, {"document": "doc3"}], + severity="critical", + ), + ] + + analysis = analyzer.analyze_conflicts(conflicts) + + self.assertIn("by_source", analysis) + by_source = analysis["by_source"] + + # doc1 appears in both conflicts, doc2 and doc3 in one each. + self.assertEqual(by_source["counts"]["doc1"], 2) + self.assertEqual(by_source["counts"]["doc2"], 1) + self.assertEqual(by_source["counts"]["doc3"], 1) + + top_sources = { + s["source"]: s["conflict_count"] for s in by_source["top_sources"] + } + self.assertEqual(top_sources["doc1"], 2) + + self.assertIn("doc1", by_source["details"]) + doc1_entries = by_source["details"]["doc1"] + self.assertEqual(len(doc1_entries), 2) + self.assertEqual({e["conflict_id"] for e in doc1_entries}, {"c1", "c2"}) + + def test_conflict_analyzer_analyze_trends(self): + """Test analyze_trends over deterministic, time-ordered data (#902).""" + analyzer = ConflictAnalyzer() + + def make_conflict(conflict_id, timestamp): + return Conflict( + conflict_id=conflict_id, + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[{"document": "doc1", "metadata": {"timestamp": timestamp}}], + ) + + # January: 1 conflict. February: 3 conflicts (>10% increase -> "increasing"). + conflicts = [ + make_conflict("c1", "2023-01-05T00:00:00"), + make_conflict("c2", "2023-02-01T00:00:00"), + make_conflict("c3", "2023-02-10T00:00:00"), + make_conflict("c4", "2023-02-20T00:00:00"), + ] + + trends = analyzer.analyze_trends(conflicts) + + self.assertEqual(len(trends), 2) + self.assertEqual(trends[0]["period"], "2023-01") + self.assertEqual(trends[0]["conflict_count"], 1) + self.assertEqual(trends[1]["period"], "2023-02") + self.assertEqual(trends[1]["conflict_count"], 3) + self.assertEqual(trends[1]["trend"], "increasing") + self.assertEqual(trends[1]["trend_direction"], "up") + + def test_conflict_analyzer_analyze_trends_insufficient_data(self): + """Single-period data should report insufficient_data, not crash (#902).""" + analyzer = ConflictAnalyzer() + + conflict = Conflict( + conflict_id="c1", + conflict_type=ConflictType.VALUE_CONFLICT, + entity_id="e1", + property_name="age", + conflicting_values=[30, 32], + sources=[ + {"document": "doc1", "metadata": {"timestamp": "2023-01-05T00:00:00"}} + ], + ) + + trends = analyzer.analyze_trends([conflict]) + + self.assertEqual(len(trends), 1) + self.assertEqual(trends[0]["trend"], "insufficient_data") + self.assertEqual(trends[0]["conflict_count"], 1) + if __name__ == "__main__": unittest.main() \ No newline at end of file From ea7790a5bf5ee53e164bc3ebd849f4c04ab598bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E5=AE=B4?= Date: Thu, 13 Aug 2026 18:03:03 +0800 Subject: [PATCH 012/129] fix(docker): pin runtime to python:3.13-slim gensim (core dependency) has no prebuilt cp314 wheel, and the slim base image lacks gcc to build from source, so 'pip install .[explorer]' fails on python:3.14-slim. Pin to python:3.13-slim (still satisfies requires-python>=3.8) until gensim ships a cp314 wheel. Co-Authored-By: Claude --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0cb1f418..a462509e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN npm ci COPY explorer/ ./ RUN mkdir -p /app/semantica && npm run build -FROM python:3.14-slim AS runtime +FROM python:3.13-slim AS runtime ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ From b2d54a668343978912b7e47d8f0d0a09369e015d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E5=AE=B4?= Date: Thu, 13 Aug 2026 18:03:03 +0800 Subject: [PATCH 013/129] fix(context): CJK decision similarity and rebuild decision indexes on load Add a character-bigram overlap-coefficient fallback to _calculate_decision_content_similarity so CJK scenarios (no whitespace tokenization) can match recorded decisions; the previous whitespace Jaccard was always 0 for CJK. Rebuild _decisions/_decision_index/_entity_index/_temporal_index from persisted decision nodes at the end of load_from_file, otherwise find_precedents_by_scenario and decision_count break after a reload since save_to_file does not serialize the internal decision indexes. Co-Authored-By: Claude --- semantica/context/context_graph.py | 75 ++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 06419759..e034ce66 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -1077,6 +1077,44 @@ class ContextGraph: if link_id: self._unresolved_links[link_id] = link_meta + # Rebuild decision indexes from persisted decision nodes so that + # find_precedents_by_scenario / decision counts work after a reload + decision_nodes = [ + n for n in self.nodes.values() + if (getattr(n, "node_type", None) or "").lower() == "decision" + ] + if decision_nodes: + if not hasattr(self, "_decisions"): + self._decisions = {} + self._decision_index = defaultdict(set) + self._entity_index = defaultdict(set) + self._temporal_index = [] + for node in decision_nodes: + meta = dict(getattr(node, "metadata", {}) or {}) + meta.update(getattr(node, "properties", {}) or {}) + decision = { + "id": node.node_id, + "category": meta.get("category", ""), + "scenario": meta.get("scenario", getattr(node, "content", "") or ""), + "reasoning": meta.get("reasoning", ""), + "outcome": meta.get("outcome", ""), + "confidence": meta.get("confidence", 0.0), + "entities": meta.get("entities", []), + "decision_maker": meta.get("decision_maker"), + "timestamp": meta.get("timestamp", 0.0), + "recorded_at": meta.get("recorded_at", ""), + "valid_from": getattr(node, "valid_from", None), + "valid_until": getattr(node, "valid_until", None), + "metadata": {}, + } + self._decisions[node.node_id] = decision + if decision["category"]: + self._decision_index[decision["category"]].add(node.node_id) + for entity in decision["entities"]: + self._entity_index[entity].add(node.node_id) + self._temporal_index.append((node.node_id, decision["timestamp"])) + self._temporal_index.sort(key=lambda x: x[1], reverse=True) + self.logger.info(f"Loaded context graph from {path}") def find_node(self, node_id: str) -> Optional[Dict[str, Any]]: @@ -3051,19 +3089,38 @@ class ContextGraph: return False return True + @staticmethod + def _char_bigrams(text: str) -> set: + """Character bigrams over whitespace-stripped text (CJK fallback).""" + chars = "".join(text.lower().split()) + return {chars[i:i + 2] for i in range(len(chars) - 1)} + def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float: - """Calculate content similarity between scenario and decision.""" + """Calculate content similarity between scenario and decision. + + Combines word-level Jaccard (works for space-separated languages) + with character-bigram signals (fallback for CJK text without spaces). + For the bigram side we use the overlap coefficient |A∩B| / min(|A|,|B|) + instead of Jaccard, so that a short query against a long decision + document is not penalised for length mismatch. + """ try: - # Simple word-based similarity - scenario_words = set(scenario.lower().split()) decision_text = f"{decision['scenario']} {decision['reasoning']} {' '.join(decision['entities'])}" + + # Word-based similarity + scenario_words = set(scenario.lower().split()) decision_words = set(decision_text.lower().split()) - - intersection = scenario_words.intersection(decision_words) - union = scenario_words.union(decision_words) - - return len(intersection) / len(union) if union else 0.0 - + word_union = scenario_words | decision_words + word_sim = len(scenario_words & decision_words) / len(word_union) if word_union else 0.0 + + # Character-bigram similarity (CJK texts tokenize poorly on whitespace) + scenario_bigrams = self._char_bigrams(scenario) + decision_bigrams = self._char_bigrams(decision_text) + smaller = min(len(scenario_bigrams), len(decision_bigrams)) + bigram_sim = len(scenario_bigrams & decision_bigrams) / smaller if smaller else 0.0 + + return max(word_sim, bigram_sim) + except Exception as e: self.logger.exception("Content similarity calculation failed") return 0.0 From 778ff5116252d956df59d5ce83b46770d22a0884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E5=AE=B4?= Date: Thu, 13 Aug 2026 18:03:03 +0800 Subject: [PATCH 014/129] fix(explorer): coerce decision timestamp to str in response DecisionResponse.timestamp is typed str, but decision nodes store a float epoch. Coerce non-str timestamps so GET /api/decisions stops returning 422 Unprocessable Content. Co-Authored-By: Claude --- semantica/explorer/routes/decisions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/semantica/explorer/routes/decisions.py b/semantica/explorer/routes/decisions.py index e9df4493..b942b9e5 100644 --- a/semantica/explorer/routes/decisions.py +++ b/semantica/explorer/routes/decisions.py @@ -16,6 +16,7 @@ router = APIRouter(prefix="/api/decisions", tags=["Decisions"]) def _node_to_decision(node: dict) -> DecisionResponse: properties = node.get("properties", {}) + ts = properties.get("timestamp") return DecisionResponse( decision_id=node.get("id", ""), category=properties.get("category", ""), @@ -23,7 +24,7 @@ def _node_to_decision(node: dict) -> DecisionResponse: reasoning=properties.get("reasoning", ""), outcome=properties.get("outcome", ""), confidence=float(properties.get("confidence", 0.0) or 0.0), - timestamp=properties.get("timestamp"), + timestamp=ts if isinstance(ts, str) or ts is None else str(ts), metadata=properties, ) From 0e40639930456ae064c72c326bf70d497808c470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E5=AE=B4?= Date: Thu, 13 Aug 2026 18:04:08 +0800 Subject: [PATCH 015/129] feat(mcp): fix decision persistence/query, add NER model params and graph tools Bug fixes: - _get_graph: call load_from_file (graph.load does not exist; SEMANTICA_KG_PATH was silently ignored and the graph started empty). - query_decisions: read category from metadata.category (top-level category was always empty, so category filtering returned nothing). - find_precedents / query: lower default similarity threshold to 0.05 so short CJK queries can match. - extract_entities/extract_relations: return the entity text field (previously returned the spaCy type label as 'label' and dropped the actual text); expose model/language/method params so non-English (e.g. zh_core_web_sm) NER works. New tools: - query_graph: node detail / bidirectional neighbours (up to 5 hops, in-edges included) / keyword search. - update_node: update node properties (e.g. action status todo/doing/done) and persist to SEMANTICA_KG_PATH. - delete_node: soft-archive a node (status=archived) and persist. Co-Authored-By: Claude --- semantica/mcp_server/__init__.py | 244 +++++++++++++++++++++++++++++-- 1 file changed, 230 insertions(+), 14 deletions(-) diff --git a/semantica/mcp_server/__init__.py b/semantica/mcp_server/__init__.py index 6f642bab..c3005140 100644 --- a/semantica/mcp_server/__init__.py +++ b/semantica/mcp_server/__init__.py @@ -65,7 +65,7 @@ def _get_graph(): kg_path = os.environ.get("SEMANTICA_KG_PATH") if kg_path and os.path.exists(kg_path): try: - _graph.load(kg_path) + _graph.load_from_file(kg_path) log.info("Loaded graph from %s", kg_path) except Exception as exc: log.warning("Could not load graph from %s: %s", kg_path, exc) @@ -77,30 +77,54 @@ def _get_graph(): # ══════════════════════════════════════════════════════════════════════════════ def _tool_extract_entities(args: dict) -> dict: - """Extract named entities from text.""" + """Extract named entities from text. + + Optional ``model`` (spaCy pipeline, e.g. ``zh_core_web_sm`` for Chinese) + and ``language`` allow non-English NER; defaults to the Semantica English + pipeline when omitted. ``method`` defaults to ``ml`` (spaCy); other + options are ``huggingface``, ``llm``, ``pattern``. + """ text = args.get("text", "") if not text: return {"error": "text is required"} from semantica.semantic_extract import NamedEntityRecognizer - entities = NamedEntityRecognizer().extract_entities(text) + init_kwargs = {} + for k in ("model", "language", "confidence_threshold"): + if args.get(k) is not None: + init_kwargs[k] = args[k] + method = args.get("method", "ml") + ner = NamedEntityRecognizer(methods=[method], **init_kwargs) + entities = ner.extract_entities(text) return { "entities": [ - {"label": getattr(e, "label", str(e)), - "type": getattr(e, "type", None), - "start": getattr(e, "start", None), - "end": getattr(e, "end", None)} + {"text": getattr(e, "text", ""), + "label": getattr(e, "label", ""), + "type": getattr(e, "label", None), + "start": getattr(e, "start_char", getattr(e, "start", None)), + "end": getattr(e, "end_char", getattr(e, "end", None)), + "confidence": getattr(e, "confidence", 1.0)} for e in (entities or []) ] } def _tool_extract_relations(args: dict) -> dict: - """Extract relations and triplets from text.""" + """Extract relations and triplets from text. + + Optional ``model``/``language`` enable non-English extraction. + ``method`` defaults to ``pattern``; ``dependency`` uses spaCy syntactic + parsing (requires a spaCy model, e.g. ``zh_core_web_sm``). + """ text = args.get("text", "") if not text: return {"error": "text is required"} from semantica.semantic_extract import RelationExtractor, TripletExtractor - relations = RelationExtractor().extract_relations(text) + rel_kwargs = {} + for k in ("model", "language"): + if args.get(k) is not None: + rel_kwargs[k] = args[k] + method = args.get("method", "pattern") + relations = RelationExtractor(method=method, **rel_kwargs).extract_relations(text) triplets = TripletExtractor().extract_triplets(text) return { "relations": [ @@ -147,10 +171,12 @@ def _tool_query_decisions(args: dict) -> dict: graph = _get_graph() try: if query: - results = graph.find_similar_decisions(query, max_results=limit) + results = graph.find_similar_decisions(query, max_results=limit, min_similarity=0.05) elif category: nodes = graph.find_nodes(node_type="decision") - results = [n for n in nodes if n.get("category") == category][:limit] + results = [n for n in nodes + if n.get("category") == category + or n.get("metadata", {}).get("category") == category][:limit] else: results = graph.find_nodes(node_type="decision")[:limit] return {"decisions": results if isinstance(results, list) else list(results)} @@ -166,7 +192,9 @@ def _tool_find_precedents(args: dict) -> dict: max_results = int(args.get("max_results", 5)) graph = _get_graph() try: - precedents = graph.find_similar_decisions(scenario, max_results=max_results) + min_similarity = float(args.get("min_similarity", 0.05)) + precedents = graph.find_similar_decisions( + scenario, max_results=max_results, min_similarity=min_similarity) return {"precedents": precedents if isinstance(precedents, list) else list(precedents)} except Exception as exc: return {"error": str(exc), "precedents": []} @@ -281,6 +309,145 @@ def _tool_get_graph_summary(args: dict) -> dict: return {"error": str(exc), "graph_ready": False} +def _tool_update_node(args: dict) -> dict: + """Update properties of an existing node and persist to SEMANTICA_KG_PATH. + + Common use: mark an action node's status (todo/doing/done) with an + optional note. The graph is mutated in-memory then saved back to the + file it was loaded from, so changes survive server restarts. + """ + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + properties = args.get("properties", {}) + if not isinstance(properties, dict) or not properties: + return {"error": "properties (non-empty object) is required"} + graph = _get_graph() + try: + if not graph.find_node(node_id): + return {"error": f"node '{node_id}' not found"} + graph.add_node_attribute(node_id, properties) + # Persist back to disk so the change survives restarts + kg_path = os.environ.get("SEMANTICA_KG_PATH") + if kg_path: + graph.save_to_file(kg_path) + persisted = True + else: + persisted = False + updated = graph.find_node(node_id) + return { + "status": "updated", + "node_id": node_id, + "properties": {k: (updated.get("metadata") or {}).get(k) for k in properties}, + "persisted": persisted, + } + except Exception as exc: + return {"error": str(exc)} + + +def _tool_delete_node(args: dict) -> dict: + """Archive a node (soft delete) and persist to SEMANTICA_KG_PATH. + + The node is kept in the graph for history but marked status='archived'. + Use to retire an action you no longer actively track. + """ + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + graph = _get_graph() + try: + if not graph.find_node(node_id): + return {"error": f"node '{node_id}' not found"} + graph.add_node_attribute(node_id, {"status": "archived"}) + kg_path = os.environ.get("SEMANTICA_KG_PATH") + if kg_path: + graph.save_to_file(kg_path) + return {"status": "archived", "node_id": node_id, "persisted": bool(kg_path)} + except Exception as exc: + return {"error": str(exc)} + + +def _tool_query_graph(args: dict) -> dict: + """Query the live knowledge graph: node detail, neighbours, or keyword search. + + mode: + - "node" : get one node by id (needs node_id) + - "neighbors": traverse up to `depth` hops from node_id (default depth=1) + - "search" : keyword search over node id+content (needs query) + """ + graph = _get_graph() + mode = args.get("mode", "neighbors") + try: + if mode == "node": + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + node = graph.find_node(node_id) + return {"node": node} + + if mode == "neighbors": + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + depth = int(args.get("depth", 1)) + rel_types = args.get("relationship_types") + if isinstance(rel_types, str): + rel_types = [rel_types] + rel_set = set(rel_types) if rel_types else None + limit = args.get("limit") + limit = int(limit) if limit is not None else None + depth = min(max(depth, 1), 5) + # Out-edges (multi-hop) via get_neighbors + nb = graph.get_neighbors( + node_id, hops=depth, relationship_types=rel_types, limit=limit, + ) + out = [ + {"id": n.get("id"), "type": n.get("type"), + "content": n.get("content"), + "relationship": n.get("relationship"), + "direction": "out", "hop": n.get("hop", 1)} + for n in (nb or []) + ] + # In-edges (1-hop): scan edges whose target == node_id + inb = [] + for e in graph.find_edges(): + if e.get("target") != node_id: + continue + if rel_set is not None and e.get("type") not in rel_set: + continue + src_id = e.get("source") + src = graph.find_node(src_id) or {} + inb.append({"id": src_id, "type": src.get("type"), + "content": src.get("content"), + "relationship": e.get("type"), + "direction": "in", "hop": 1}) + neighbors = out + inb + if limit: + neighbors = neighbors[:limit] + return {"node_id": node_id, "depth": depth, "neighbors": neighbors} + + if mode == "search": + q = (args.get("query") or "").lower() + if not q: + return {"error": "query is required"} + node_type = args.get("node_type") + limit = int(args.get("limit", 50)) + nodes = graph.find_nodes(node_type=node_type) if node_type else graph.find_nodes() + hits = [] + for n in nodes: + blob = f"{n.get('id','')} {n.get('content','')}".lower() + if q in blob: + hits.append({"id": n.get("id"), "type": n.get("type"), + "content": n.get("content")}) + if len(hits) >= limit: + break + return {"query": q, "results": hits, "total": len(hits)} + + return {"error": f"unknown mode '{mode}': use node|neighbors|search"} + except Exception as exc: + return {"error": str(exc)} + + # ══════════════════════════════════════════════════════════════════════════════ # MCP protocol tables # ══════════════════════════════════════════════════════════════════════════════ @@ -292,7 +459,11 @@ TOOLS = [ "inputSchema": { "type": "object", "properties": { - "text": {"type": "string", "description": "Input text to extract entities from"} + "text": {"type": "string", "description": "Input text to extract entities from"}, + "model": {"type": "string", "description": "spaCy model name, e.g. 'zh_core_web_sm' for Chinese, 'en_core_web_sm' for English. Defaults to English pipeline."}, + "language": {"type": "string", "description": "Language code, e.g. 'zh', 'en'."}, + "method": {"type": "string", "description": "Extraction method: 'ml' (spaCy, default), 'huggingface', 'llm', 'pattern'."}, + "confidence_threshold": {"type": "number", "description": "Minimum confidence 0-1 (default 0.5)."} }, "required": ["text"], }, @@ -304,7 +475,10 @@ TOOLS = [ "inputSchema": { "type": "object", "properties": { - "text": {"type": "string", "description": "Input text to extract relations from"} + "text": {"type": "string", "description": "Input text to extract relations from"}, + "model": {"type": "string", "description": "spaCy model name for dependency parsing, e.g. 'zh_core_web_sm'."}, + "language": {"type": "string", "description": "Language code, e.g. 'zh'."}, + "method": {"type": "string", "description": "Extraction method: 'pattern' (default), 'dependency', 'cooccurrence', 'huggingface', 'llm'."} }, "required": ["text"], }, @@ -445,6 +619,48 @@ TOOLS = [ "inputSchema": {"type": "object", "properties": {}}, "_handler": _tool_get_graph_summary, }, + { + "name": "query_graph", + "description": "Query the live knowledge graph: get a node, traverse its neighbours (up to 5 hops), or keyword-search nodes by id+content.", + "inputSchema": { + "type": "object", + "properties": { + "mode": {"type": "string", "description": "node | neighbors | search (default: neighbors)"}, + "node_id": {"type": "string", "description": "Node id (required for node/neighbors mode)"}, + "depth": {"type": "integer", "description": "Hop depth for neighbors (1-5, default 1)"}, + "relationship_types": {"type": "array", "items": {"type": "string"}, "description": "Optional filter by edge type(s)"}, + "query": {"type": "string", "description": "Keyword for search mode (matched against node id+content)"}, + "node_type": {"type": "string", "description": "Optional node_type filter for search mode"}, + "limit": {"type": "integer", "description": "Max results for neighbors/search"} + }, + }, + "_handler": _tool_query_graph, + }, + { + "name": "update_node", + "description": "Update properties of an existing node (e.g. mark an action todo/doing/done with a note) and persist to SEMANTICA_KG_PATH.", + "inputSchema": { + "type": "object", + "properties": { + "node_id": {"type": "string", "description": "Node id to update"}, + "properties": {"type": "object", "description": "Property key-values to merge onto the node, e.g. {\"status\":\"done\",\"updated_at\":\"2026-08-13\",\"note\":\"...\"}"} + }, + "required": ["node_id", "properties"], + }, + "_handler": _tool_update_node, + }, + { + "name": "delete_node", + "description": "Archive a node (soft delete: marks status='archived', keeps it for history) and persist to SEMANTICA_KG_PATH. Use to retire an action you no longer track.", + "inputSchema": { + "type": "object", + "properties": { + "node_id": {"type": "string", "description": "Node id to delete"} + }, + "required": ["node_id"], + }, + "_handler": _tool_delete_node, + }, ] RESOURCES = [ From 21edb700b29845012fce8f73e110d4378fce0756 Mon Sep 17 00:00:00 2001 From: LeonSGP43 Date: Sat, 15 Aug 2026 11:56:17 +0800 Subject: [PATCH 016/129] docs(cookbook): add provenance tracking tutorial (PROV-O lineage, invalidation, checksums) Add cookbook/introduction/22_Provenance_Tracking.ipynb covering the provenance module end to end: - tracking entities/relationships with audit-grade source details (DOI + location + verbatim quote + confidence) - lineage walks (get_lineage / trace_lineage) - revision history and multi-source audits - prov:Invalidation (correct-without-delete) and storage statistics - tamper-evidence via chained SHA-256 checksums All API calls verified against semantica/provenance/manager.py. Signed-off-by: LeonSGP43 --- .../introduction/22_Provenance_Tracking.ipynb | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 cookbook/introduction/22_Provenance_Tracking.ipynb diff --git a/cookbook/introduction/22_Provenance_Tracking.ipynb b/cookbook/introduction/22_Provenance_Tracking.ipynb new file mode 100644 index 00000000..ce285f3d --- /dev/null +++ b/cookbook/introduction/22_Provenance_Tracking.ipynb @@ -0,0 +1,243 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Provenance Tracking (W3C PROV-O)\n", + "\n", + "## Overview\n", + "\n", + "In high-stakes domains — healthcare, legal, finance, research — a Knowledge Graph is only as trustworthy as its ability to answer **\"where did this fact come from?\"**. Semantica's `provenance` module provides audit-grade, W3C PROV-O-aligned tracking for every entity, relationship and chunk that flows through your pipeline.\n", + "\n", + "In this cookbook you will learn how to:\n", + "\n", + "- Track entities and relationships with **source details** (DOI, page, verbatim quote, confidence)\n", + "- Walk the full **lineage** of a fact (document → chunk → entity → KG)\n", + "- Audit **revision history** and **all sources** behind an entity\n", + "- **Invalidate** a fact without deleting it (prov:Invalidation) — corrections stay provable\n", + "- Verify **tamper-evidence** with chained SHA-256 checksums\n", + "\n", + "**The Scenario:** a research team ingests findings from two scientific papers (with DOIs) into a Knowledge Graph. A regulator later asks: *\"Which paper, which figure, and which exact sentence supports the claim that fish biomass increased by 463%? And was that fact ever corrected?\"*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -q semantica" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from semantica.provenance import (\n", + " ProvenanceManager,\n", + " compute_checksum,\n", + " verify_checksum,\n", + ")\n", + "\n", + "# In-memory storage for this demo; pass storage_path=\"provenance.db\"\n", + "# (or a config with provenance.storage_path) for a persistent SQLite backend.\n", + "prov = ProvenanceManager()\n", + "print(\"ProvenanceManager ready (in-memory storage)\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Track Entities with Audit-Grade Source Details\n", + "\n", + "Every fact we ingest carries its evidence with it: the **source identifier** (a DOI here), the **location** inside the source (a figure), the **verbatim quote**, and the extractor's **confidence**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Finding from paper #1\n", + "entry_biomass = prov.track_entity(\n", + " entity_id=\"claim_biomass_increase\",\n", + " source=\"DOI:10.1371/journal.pone.0023601\",\n", + " confidence=0.92,\n", + " source_location=\"Figure 2\",\n", + " source_quote=\"Total fish biomass increased by 463% ...\",\n", + ")\n", + "\n", + "# Supporting entity from paper #2\n", + "entry_reserve = prov.track_entity(\n", + " entity_id=\"marine_reserve_1\",\n", + " source=\"DOI:10.1126/science.1088121\",\n", + " confidence=0.88,\n", + " source_location=\"Table 1\",\n", + " source_quote=\"... no-take marine reserve at Cabo Pulmo ...\",\n", + ")\n", + "\n", + "print(\"Tracked:\", entry_biomass.entity_id, \"|\", entry_reserve.entity_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Track the Relationship Between Facts\n", + "\n", + "Facts rarely stand alone. The claim about biomass increase is *about* the marine reserve — that relationship is a first-class provenance-tracked object too." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rel = prov.track_relationship(\n", + " relationship_id=\"rel_biomass_about_reserve\",\n", + " source=\"DOI:10.1371/journal.pone.0023601\",\n", + " metadata={\"type\": \"measured_at\"},\n", + ")\n", + "\n", + "print(\"Relationship tracked:\", rel.entity_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Walk the Lineage\n", + "\n", + "`get_lineage` reconstructs everything known about a fact; `trace_lineage` returns the ordered chain of `ProvenanceEntry` records — every version, every activity, every agent that touched it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lineage = prov.get_lineage(\"claim_biomass_increase\")\n", + "print(json.dumps(lineage, indent=2, default=str)[:800])\n", + "\n", + "print(\"\\n--- ordered chain ---\")\n", + "for e in prov.trace_lineage(\"claim_biomass_increase\"):\n", + " print(f\"{e.entity_id} | v{getattr(e, 'version', '?')} | {e.activity_id}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Audit Sources and Revision History\n", + "\n", + "When the regulator asks *\"has this fact ever been corrected?\"*, `revision_history` answers with the full version chain, and `get_all_sources` lists every source document that ever supported the entity." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "revisions = prov.revision_history(\"claim_biomass_increase\")\n", + "print(f\"{len(revisions)} revision(s) on record\")\n", + "\n", + "for s in prov.get_all_sources(\"claim_biomass_increase\"):\n", + " print(\"source:\", s)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Invalidate — Correct Without Deleting\n", + "\n", + "Suppose paper #1 is retracted in part. An audit trail must **not** silently delete the fact: `invalidate` archives the pre-invalidation state and appends a fresh `prov:Invalidation` entry naming **who** retracted it and **why**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "invalidated = prov.invalidate(\n", + " entity_id=\"claim_biomass_increase\",\n", + " agent_id=\"reviewer_dr_chen\",\n", + " reason=\"Partial retraction: Figure 2 statistics corrected by publisher (see erratum).\",\n", + ")\n", + "print(\"Invalidated:\", invalidated.entity_id, \"| invalidated flag:\", getattr(invalidated, \"invalidated\", True))\n", + "\n", + "stats = prov.get_statistics()\n", + "print(\"\\nStorage statistics:\", json.dumps(stats, indent=2, default=str))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Verify Tamper-Evidence\n", + "\n", + "Each entry carries a deterministic SHA-256 checksum chained to the previous entry. Recompute and compare to detect any after-the-fact corruption of the provenance record." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# entry_biomass was returned by track_entity in Step 1\n", + "ok = verify_checksum(entry_biomass)\n", + "print(\"Checksum verified:\", ok)\n", + "\n", + "print(\"Computed:\", compute_checksum(entry_biomass)[:16], \"...\")\n", + "print(\"Stored: \", entry_biomass.checksum[:16] if getattr(entry_biomass, 'checksum', None) else \"(see entry fields)\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Need | Call |\n", + "|---|---|\n", + "| Record a fact's evidence | `prov.track_entity(entity_id, source, confidence=..., source_location=..., source_quote=...)` |\n", + "| Record a relationship | `prov.track_relationship(relationship_id, source, metadata=...)` |\n", + "| Full lineage of a fact | `prov.get_lineage(entity_id)` / `prov.trace_lineage(entity_id)` |\n", + "| \"Was it ever corrected?\" | `prov.revision_history(entity_id)` |\n", + "| \"Which sources support it?\" | `prov.get_all_sources(entity_id)` |\n", + "| Retract without deleting | `prov.invalidate(entity_id, agent_id, reason=...)` |\n", + "| Tamper check | `verify_checksum(entry)` |\n", + "\n", + "### Where to go next\n", + "\n", + "- **Conflict Detection and Resolution** (notebook 17) — what happens when two sources disagree.\n", + "- **Your First Knowledge Graph** (notebook 08) — plug `provenance=True` into extractors so tracking happens automatically during ingestion.\n", + "- The module docstring (`help(semantica.provenance)`) documents opt-in integration with `kg`, `split` and `conflicts` trackers." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From d42af280e857f2aa0e6fab9de92ed6a26e71f04f Mon Sep 17 00:00:00 2001 From: Varun Sahni Date: Sat, 15 Aug 2026 11:50:58 +0530 Subject: [PATCH 017/129] fix: prevent fallback recursion and write proper Parquet in embed generate command Fixes #994: 1. Prevent self-recursion in methods.py: generate_embeddings, embed_text, calculate_similarity, and pool_embeddings all registered themselves as custom methods, causing infinite self-calls when dispatch invoked them without explicitly passing method parameter. Fix: check custom_method is not the function itself before recursing. 2. Fix embed generate --output corrupt output: the CLI wrote json.dumps(result, default=str) which produced plaintext repr of numpy arrays (e.g. '[1.49e-01 4.85e-02 ...]') instead of proper Parquet. Fix: detect .parquet extension (case-insensitive), convert numpy array to pandas DataFrame with dim_* columns and id index, use to_parquet(). Non-parquet extensions fall back to JSON with clear ImportError message. 3. Add pyarrow>=14.0.0 to core dependencies (previously only in ingest-parquet/ingest-arrow optional extras). The documented quick-start flow of embed generate --output ... requires pyarrow out of the box. (Note: pandas>=1.3.0 is already a core dependency; pyarrow is the missing piece.) Note: .github/workflows/* files are excluded from this PR as they require a token with workflow scope. Upstream workflows are unchanged. --- .github/workflows/benchmark.yml | 51 ----- .github/workflows/ci.yml | 83 -------- .github/workflows/codeql.yml | 97 --------- .github/workflows/defender-for-devops.yml | 88 -------- .github/workflows/docs.yml | 68 ------ .github/workflows/release.yml | 73 ------- .github/workflows/security-scan.yml | 239 ---------------------- .github/workflows/security.yml | 42 ---- .github/workflows/verify-action-pins.yml | 28 --- pyproject.toml | 3 +- semantica/cli.py | 23 ++- semantica/embeddings/methods.py | 16 +- 12 files changed, 32 insertions(+), 779 deletions(-) delete mode 100644 .github/workflows/benchmark.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/defender-for-devops.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/security-scan.yml delete mode 100644 .github/workflows/security.yml delete mode 100644 .github/workflows/verify-action-pins.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 0157ee24..00000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Semantica Performance Suite - -on: - workflow_dispatch: - -permissions: - contents: read - -jobs: - performance-test: - name: Benchmark Runner (Ubuntu/Python 3.12) - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - - name: Set up Python 3.11 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.11" - cache: 'pip' - - - name: Install Dependencies - env: - - BENCHMARK_REAL_LIBS: "1" - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -r benchmarks/requirements.txt - python -m spacy download en_core_web_sm - pip install rdflib neo4j faiss-cpu torch pyarrow pdfplumber python-pptx openpyxl lxml python-docx beautifulsoup4 chardet langdetect - - - name: Execute Benchmarks (Real Mode) - env: - BENCHMARK_REAL_LIBS: "1" - run: | - python benchmarks/benchmarks_runner.py - # Optional: Compare to baseline (requires previous run artifact) - # pytest-benchmark --storage file://benchmarks/results --benchmark-compare - - - name: Upload Benchmark Results - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: benchmark-report-${{ github.run_id }} - path: benchmarks/results - retention-days: 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 4ea31ff8..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: CI - -permissions: - contents: read - -on: - push: - branches: [main] - paths-ignore: - - 'docs/**' - - 'docs_check.py' - - '**/*.md' - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - - 'docs_check.py' - - '**/*.md' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: explorer/package-lock.json - - name: Install Explorer frontend dependencies - working-directory: explorer - run: npm ci - - name: Test Explorer frontend - working-directory: explorer - run: | - npm run test:graph-store - npm run test:graph-workspace - npm run test:plugin-registry - - name: Build Explorer frontend - working-directory: explorer - run: npm run build - - name: Install pinned Python dependencies - run: | - pip install -r requirements-ci.txt - - name: Verify requirements-ci.txt is up to date - run: | - pip install uv==0.12.1 - # Re-resolve with the committed file as a constraint: upstream package - # releases must NOT fail CI (deps only change when pyproject.toml - # changes intentionally). Compare only version lines (pkg==ver), - # ignoring the -c constraint comments and the `\` line continuations - # that --generate-hashes emits. - uv pip compile pyproject.toml --python-version 3.11 --extra all \ - --constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt - diff \ - <(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \ - <(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt) - - run: pip install build - # wheel is build-time only (not in requirements-ci.txt) — install the - # same pinned version [build-system] declares so --no-isolation works. - - run: pip install wheel==0.48.0 - - name: Build package (no isolation — pinned deps) - run: python -m build --no-isolation - - name: Verify Explorer frontend is packaged - run: | - python - <<'PY' - import zipfile - from pathlib import Path - - wheels = list(Path("dist").glob("*.whl")) - assert wheels, "No wheel was built" - - with zipfile.ZipFile(wheels[0]) as wheel: - names = set(wheel.namelist()) - - assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel" - assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel" - - print("Explorer frontend is packaged") - PY diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 0c15e84c..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: CodeQL - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - - cron: '30 1 * * 1' # Every Monday 7 AM IST - -permissions: - contents: read - security-events: write - actions: read - -jobs: - analyze: - name: Analyze Python - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - # The CodeQL bundle download (github/codeql-action/init's "Setup CodeQL - # tools" step) streams a ~1GB tarball from GitHub's release CDN and - # does not retry on a transient connection reset (ECONNRESET) itself - # (github/codeql-action, unresolved as of v4 / CLI 2.26.1: the HTTP - # error is retryable but isn't retried internally). Since a `uses:` - # step can't be wrapped by a shell-level retry action, attempt init - # up to 3 times; each retry is a fresh download attempt with no - # meaningful state carried over from a failed attempt. - - name: Initialize CodeQL (attempt 1) - id: codeql-init-1 - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - continue-on-error: true - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Initialize CodeQL (attempt 2) - id: codeql-init-2 - if: steps.codeql-init-1.outcome == 'failure' - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - continue-on-error: true - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Initialize CodeQL (attempt 3) - id: codeql-init-3 - if: steps.codeql-init-2.outcome == 'failure' - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Autobuild - uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - category: "/language:python" - upload: false - id: codeql - - - name: Upload SARIF (Advanced Setup only) - # Uploads results only when Default Setup is not active. - # If Default Setup is still enabled, this step skips gracefully - # instead of failing the workflow with HTTP 409. - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - sarif_file: ${{ steps.codeql.outputs.sarif-output }} - category: "/language:python" - wait-for-processing: true - continue-on-error: true - - # NOTE: Auto-dismissal by rule-id is intentionally removed. - # Dismissing every alert that matches a rule ID would silently suppress - # future real vulnerabilities of the same type. The alerts below were - # individually triaged and dismissed manually in the security-enhancement - # PR (alerts #12–#18). New alerts must be reviewed and dismissed by hand, - # or will auto-close when the underlying code no longer triggers them. - # - # If you need to dismiss a specific known-safe alert, pin its alert NUMBER - # here and remove it once CodeQL stops reporting it naturally. Example: - # - # PINNED_ALERT_NUMBERS=(12 13 14 15 16 17 18) - # for NUM in "${PINNED_ALERT_NUMBERS[@]}"; do - # gh api repos/$REPO/code-scanning/alerts/$NUM \ - # -X PATCH -f state=dismissed -f dismissed_reason="false positive" \ - # -f dismissed_comment="" - # done diff --git a/.github/workflows/defender-for-devops.yml b/.github/workflows/defender-for-devops.yml deleted file mode 100644 index becb7d64..00000000 --- a/.github/workflows/defender-for-devops.yml +++ /dev/null @@ -1,88 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. -# -# Microsoft Security DevOps (MSDO) is a command line application which integrates static analysis tools into the development cycle. -# MSDO installs, configures and runs the latest versions of static analysis tools -# (including, but not limited to, SDL/security and compliance tools). -# -# The Microsoft Security DevOps action is currently in beta and runs on the windows-latest queue, -# as well as Windows self hosted agents. ubuntu-latest support coming soon. -# -# For more information about the action , check out https://github.com/microsoft/security-devops-action -# -# Please note this workflow do not integrate your GitHub Org with Microsoft Defender For DevOps. You have to create an integration -# and provide permission before this can report data back to azure. -# Read the official documentation here : https://learn.microsoft.com/en-us/azure/defender-for-cloud/quickstart-onboard-github - -name: "Microsoft Defender For Devops" - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: '43 17 * * 6' - -permissions: - contents: read - security-events: write - -jobs: - MSDO: - # currently only windows-latest is supported - runs-on: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 - with: - dotnet-version: | - 5.0.x - 6.0.x - - name: Run Microsoft Security DevOps - uses: microsoft/security-devops-action@08976cb623803b1b36d7112d4ff9f59eae704de0 # v1.12.0 - id: msdo - with: - # checkov is intentionally excluded from this MSDO step. - # MSDO 0.215.0's guardian.cmd wrapper treats checkov's exit code 1 - # (emitted whenever any violation is found, even below the active severity - # threshold) as a fatal "tool error" and breaks the build even when - # "Active results: 0" and "Found no breaking results." The .checkov.yaml - # soft-fail setting is never read by the guardian wrapper. - # IaC security scanning continues below in this same MSDO job identity. - # That preserves the existing GitHub code-scanning configuration while - # avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper. - tools: eslint,templateanalyzer,terrascan - - name: Upload results to Security tab - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - sarif_file: ${{ steps.msdo.outputs.sarifFile }} - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.12" - - - name: Install Checkov - run: python -m pip install checkov==3.3.1 - - - name: Run Checkov - shell: pwsh - env: - PYTHONUTF8: "1" - run: | - New-Item -ItemType Directory -Force reports | Out-Null - checkov --directory . --framework kubernetes helm dockerfile github_actions secrets bicep arm --soft-fail --output sarif --output-file-path reports/checkov.sarif - if (-not (Test-Path reports/checkov.sarif)) { - $sarif = Get-ChildItem -Path reports -Recurse -Filter *.sarif | Select-Object -First 1 - if ($null -eq $sarif) { throw "Checkov did not produce a SARIF file" } - Copy-Item $sarif.FullName reports/checkov.sarif - } - - - name: Upload Checkov results to Security tab - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - if: always() - with: - sarif_file: reports/checkov.sarif diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index ca149d7e..00000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Build and Deploy Documentation - -on: - push: - branches: [main] - paths: - - 'docs/**' - - 'docs_check.py' - - 'CHANGELOG.md' - - 'RELEASE.md' - pull_request: - branches: [main] - paths: - - 'docs/**' - - 'docs_check.py' - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - validate: - name: Validate Documentation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - - run: python docs_check.py - - deploy: - name: Build and Deploy to GitHub Pages - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - needs: validate - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - - - name: Export static site - run: | - cd docs - npx mintlify export --output ../export.zip - cd .. - unzip -q export.zip -d site - - - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 - - - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 - with: - path: ./site - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index bbc8770c..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Release - -on: - push: - tags: ['v*'] - -permissions: - contents: read - -jobs: - release: - runs-on: ubuntu-latest - environment: pypi - concurrency: - group: release-${{ github.ref }} - cancel-in-progress: false - permissions: - contents: write # for the GitHub Release - id-token: write # for PyPI Trusted Publishing (OIDC) and attestation signing - attestations: write # for SLSA build provenance - # If you add another job to this workflow, give it its own explicit - # `permissions:` block rather than relying on the workflow-level default - # above (contents: read) - do not widen the workflow-level default. - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: explorer/package-lock.json - - name: Build Explorer frontend - working-directory: explorer - run: | - npm ci - npm run build - # Install the pinned dependency set (with hashes) so the sdist/wheel - # build runs against the same versions CI tests against. - - name: Install pinned build dependencies - run: pip install -r requirements-ci.txt - - run: pip install build - # wheel is build-time only (not in requirements-ci.txt) — install the - # same pinned version [build-system] declares so --no-isolation works. - - run: pip install wheel==0.48.0 - - name: Build package (no isolation — pinned deps) - run: python -m build --no-isolation - - name: Verify Explorer frontend is packaged - run: | - python - <<'PY' - import zipfile - from pathlib import Path - - wheels = list(Path("dist").glob("*.whl")) - assert wheels, "No wheel was built" - - with zipfile.ZipFile(wheels[0]) as wheel: - names = set(wheel.namelist()) - - assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel" - assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel" - - print("Explorer frontend is packaged") - PY - - name: Attest build provenance - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4 - with: - subject-path: 'dist/*' - - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3 - with: - files: dist/* - - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml deleted file mode 100644 index 5b4461af..00000000 --- a/.github/workflows/security-scan.yml +++ /dev/null @@ -1,239 +0,0 @@ -name: Security Scan - -on: - schedule: - - cron: '30 1 * * 1,4' # Mon/Thu 7 AM IST - push: - branches: [main] - paths-ignore: - - 'docs/**' - - 'mkdocs.yml' - - 'requirements-docs.txt' - - '**/*.md' - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - - 'mkdocs.yml' - - 'requirements-docs.txt' - - '**/*.md' - -permissions: - contents: read - -jobs: - security-scan: - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - actions: read - # Needed for the "Comment PR with Security Results" step below. Safe on - # pull_request (not pull_request_target): GitHub always forces a - # read-only token for PRs from forks regardless of this permission. - pull-requests: write - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Install the pinned dependency set FIRST so Safety scans Semantica's - # exact CI/release dependency tree (requirements-ci.txt is generated - # from pyproject.toml extras, so this covers the project's real deps). - pip install -r requirements-ci.txt - # Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq - # first lets the pinned requirements overwrite their transitive deps - # (e.g. rich), which breaks the safety CLI at runtime. - pip install safety bandit semgrep jq - - - name: Run Safety Check (Package Vulnerabilities) - run: | - # NOTE: Safety 3.x repurposed --output to select a console format - # (json/text/screen/...), not a file path. Writing JSON to a file - # now requires --save-json; the previous `--output safety-report.json` - # usage was silently invalid and never produced a report. - safety check --save-json safety-report.json || true - - # Guard 1: fail loudly if Safety exited before writing a report at all - # (network error, API auth failure, tool crash). Without this check a - # missing or empty file causes jq to fall back to "0", making a broken - # scanner indistinguishable from a clean scan. - if [ ! -s safety-report.json ]; then - echo "::error::Safety scan produced no report (safety-report.json is missing or empty). Treating as failure — check for network errors, API auth failures, or Safety crashes in the logs above." - exit 1 - fi - - echo "Checking for package vulnerabilities..." - - # No || echo "0" fallback: if jq fails (malformed JSON, missing key, - # vulnerabilities:null) VULNS will be empty or "null" so guard 2 below - # catches it rather than silently treating the broken report as zero. - VULNS=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null) - - # Guard 2: ensure VULNS is a non-negative integer before the -gt - # comparison. "null" (missing/null key) or "" (jq parse failure) would - # cause bash's -gt to throw an arithmetic error and fall through to the - # success branch — the same silent-pass bug as a missing file. - if ! [[ "$VULNS" =~ ^[0-9]+$ ]]; then - echo "::error::Safety report exists but 'vulnerabilities' is missing or non-numeric (got: '${VULNS}'). The report may be malformed or Safety may have written an error-only JSON. Treating as failure." - exit 1 - fi - - if [ "$VULNS" -gt 0 ]; then - echo "❌ Security vulnerabilities found: $VULNS" - echo "CI will fail to prevent merging of vulnerable dependencies" - echo "" - echo "Vulnerability details:" - jq -r '.vulnerabilities[] | "- \(.package_name)==\(.analyzed_version): \(.vulnerability_id) (\(.CVE // "no CVE assigned"))"' safety-report.json || true - exit 1 - else - echo "✅ No security vulnerabilities found" - fi - - - name: Run Bandit (Code Security Linter) - run: | - bandit -r semantica/ -f json -o bandit-report.json || true - echo "Checking for HIGH severity security issues..." - - # Count HIGH severity issues - HIGH_ISSUES=$(bandit -r semantica/ -f json -ll 2>/dev/null | jq -r '.results[]? | select(.issue_severity == "HIGH") | .test_name' 2>/dev/null | wc -l || echo "0") - - if [ "$HIGH_ISSUES" -gt 0 ]; then - echo "❌ HIGH severity security issues found: $HIGH_ISSUES" - echo "CI will fail to prevent merging of high-risk code" - echo "" - echo "High severity issues:" - bandit -r semantica/ -ll | grep "Severity: High" -A 5 -B 1 || true - exit 1 - else - echo "✅ No HIGH severity security issues found" - fi - - - name: Run Semgrep (Static Analysis) - run: | - echo "Running Semgrep static analysis..." - semgrep --config=auto --json --output=semgrep-report.json semantica/ || true - - # Run security-focused rules - echo "Checking for security patterns..." - SECURITY_ISSUES=$(semgrep --config=p/security --json semantica/ 2>/dev/null | jq '.results | length' 2>/dev/null || echo "0") - - if [ "$SECURITY_ISSUES" -gt 0 ]; then - echo "⚠️ Security patterns found: $SECURITY_ISSUES" - echo "Review these findings for potential improvements" - semgrep --config=p/security semantica/ || true - else - echo "✅ No security patterns found" - fi - - - name: Upload Security Reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: security-reports - retention-days: 14 - path: | - safety-report.json - bandit-report.json - semgrep-report.json - - - name: Comment PR with Security Results - if: github.event_name == 'pull_request' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 - with: - script: | - const fs = require('fs'); - - // Renders one tool's findings as a section. `items` is already - // the list of pre-formatted "- `thing` in `where`" strings; this - // just handles the found/not-found/report-missing framing and - // collapses long lists into a
block so the comment - // doesn't turn into a wall of text. - function renderSection(title, reportPath, parse) { - let data; - try { - data = JSON.parse(fs.readFileSync(reportPath, 'utf8')); - } catch (e) { - return [ - `### ${title}`, - `⚠️ No report found at \`${reportPath}\` — the scan may have failed before producing output. Check the job logs.`, - ].join('\n'); - } - - const items = parse(data); - if (items.length === 0) { - return [`### ${title}`, `✅ No findings.`].join('\n'); - } - - const lines = [`### ${title}`, `Found **${items.length}**.`, '']; - const shown = items.slice(0, 15); - if (items.length > 15) { - lines.push('
', 'Show all findings', ''); - lines.push(...items); - lines.push('', '
'); - } else { - lines.push(...shown); - } - return lines.join('\n'); - } - - const safetySection = renderSection( - 'Safety — dependency vulnerabilities', - 'safety-report.json', - (data) => (data.vulnerabilities || []).map( - (v) => `- \`${v.package_name}==${v.analyzed_version}\`: ${v.vulnerability_id}` + - (v.CVE ? ` (${v.CVE})` : '') + ` — ${v.advisory || 'no advisory text'}` - ) - ); - - const banditSection = renderSection( - 'Bandit — HIGH-severity code issues', - 'bandit-report.json', - (data) => (data.results || []) - .filter((issue) => issue.issue_severity === 'HIGH') - .map((issue) => `- \`${issue.test_name}\` in \`${issue.filename}:${issue.line_number}\``) - ); - - const semgrepSection = renderSection( - 'Semgrep — static analysis patterns', - 'semgrep-report.json', - (data) => (data.results || []).map( - (issue) => `- \`${issue.check_id}\` in \`${issue.path}:${issue.start?.line ?? '?'}\`` - ) - ); - - const comment = [ - '# 🔒 Security Scan Results', - '', - safetySection, - '', - banditSection, - '', - semgrepSection, - '', - '---', - '', - '*This security scan runs automatically on source-code PRs and bi-weekly (skipped for doc/markdown-only changes).*', - '', - '📊 **Security Policy**: CI fails on Safety vulnerabilities and Bandit HIGH-severity findings. Semgrep findings above are informational and do not block merge.', - ].join('\n'); - - try { - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment, - }); - console.log('✅ Security comment posted successfully'); - } catch (error) { - console.log('⚠️ Could not post security comment:', error.message); - console.log('📋 Security scan results saved to artifacts'); - } diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml deleted file mode 100644 index 412e7eaa..00000000 --- a/.github/workflows/security.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Security - -on: - schedule: - - cron: '0 0 * * 1' - workflow_dispatch: - pull_request: - branches: [main] - paths: - - 'pyproject.toml' - - 'requirements-ci.txt' - - '.github/workflows/security.yml' - -permissions: - contents: read - -jobs: - audit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - # Upgrade first: actions/setup-python's baked-in setuptools has been - # behind known-vulnerable floors before (e.g. PYSEC-2026-3447 / - # setuptools 75.1.0), so don't trust the preinstalled one. - - run: python -m pip install --upgrade pip setuptools - # Audit the pinned dependency set (requirements-ci.txt is compiled from - # pyproject.toml with --extra all — the same coverage as the [all] - # extra, minus the Linux-only gpu set — so this keeps scan parity with - # CI/release builds without a time-dependent resolution). This is the - # fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or - # python-multipart installed to look at. - - run: pip install -r requirements-ci.txt - # PR runs gate on findings, since they're scoped to actual - # pyproject.toml changes under review. The schedule/workflow_dispatch - # runs stay non-blocking until a full pass over pre-existing findings - # across the whole [all] tree has been done. - - run: pip install pip-audit - - run: pip-audit -r requirements-ci.txt - continue-on-error: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/verify-action-pins.yml b/.github/workflows/verify-action-pins.yml deleted file mode 100644 index 6e7dada9..00000000 --- a/.github/workflows/verify-action-pins.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Verify Action Pins - -on: - pull_request: - paths: - - '.github/workflows/**' - - '.github/scripts/verify-action-pins.sh' - push: - branches: [main] - paths: - - '.github/workflows/**' - - '.github/scripts/verify-action-pins.sh' - schedule: - - cron: '0 3 * * 1' # weekly, in case an upstream tag is deliberately moved - workflow_dispatch: - -permissions: - contents: read - -jobs: - verify: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Verify pinned action SHAs match their tag comments - env: - GH_TOKEN: ${{ github.token }} - run: bash .github/scripts/verify-action-pins.sh diff --git a/pyproject.toml b/pyproject.toml index 03949d4e..738e68ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,8 @@ dependencies = [ "loguru>=0.7.3", "structlog>=22.1.0", "gensim>=4.4.0", - "httpx<0.29.0" + "httpx<0.29.0", + "pyarrow>=14.0.0" ] [project.urls] diff --git a/semantica/cli.py b/semantica/cli.py index 7b944dc6..b5e14ebd 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -1708,7 +1708,28 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str, except ImportError as exc: raise click.ClickException(f"Embeddings module not available: {exc}") from exc if output: - Path(output).write_text(json.dumps(result, default=str), encoding="utf-8") + output_path = Path(output) + try: + import numpy as np + import pandas as pd + if output_path.suffix.lower() == ".parquet": + arr = np.asarray(result) + if arr.ndim == 1: + arr = arr[np.newaxis, :] + # schema: one column per embedding plus an id column + columns = [f"dim_{i}" for i in range(arr.shape[1])] + df = pd.DataFrame(arr, columns=columns) + df.index.name = "id" + df.to_parquet(output_path, index=True) + else: + output_path.write_text( + json.dumps(result, default=str), encoding="utf-8" + ) + except ImportError as exc: + raise click.ClickException( + f"Missing dependency for --output: {exc}. " + f"Install pyarrow/pandas with: pip install semantica[ingest-parquet]" + ) from exc _ok(cli_ctx, f"Wrote {output}") elif _is_json(cli_ctx, local_json): _jecho(result if isinstance(result, dict) else {"status": "ok"}) diff --git a/semantica/embeddings/methods.py b/semantica/embeddings/methods.py index 30c47279..d2e8848e 100644 --- a/semantica/embeddings/methods.py +++ b/semantica/embeddings/methods.py @@ -116,9 +116,9 @@ def generate_embeddings( >>> emb = generate_embeddings("Hello world", method="default") >>> embs = generate_embeddings(["text1", "text2"], method="text") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("generation", method) - if custom_method: + if custom_method and custom_method is not generate_embeddings: try: return custom_method(data, data_type=data_type, **kwargs) except Exception as e: @@ -164,9 +164,9 @@ def embed_text( >>> emb = embed_text("Hello world", method="sentence_transformers") >>> embs = embed_text(["text1", "text2"], method="sentence_transformers") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("text", method) - if custom_method: + if custom_method and custom_method is not embed_text: try: return custom_method(text, **kwargs) except Exception as e: @@ -224,9 +224,9 @@ def calculate_similarity( >>> similarity = calculate_similarity(emb1, emb2, method="cosine") >>> print(f"Similarity: {similarity:.3f}") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("similarity", method) - if custom_method: + if custom_method and custom_method is not calculate_similarity: try: return custom_method(embedding1, embedding2, **kwargs) except Exception as e: @@ -271,9 +271,9 @@ def pool_embeddings( >>> pooled = pool_embeddings(embeddings, method="mean") >>> attention_pooled = pool_embeddings(embeddings, method="attention") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("pooling", method) - if custom_method: + if custom_method and custom_method is not pool_embeddings: try: return custom_method(embeddings, **kwargs) except Exception as e: From 616f5ca9b9a6c5f6f3b7bb2ee878021f3d831416 Mon Sep 17 00:00:00 2001 From: Varun Sahni Date: Sat, 15 Aug 2026 14:04:15 +0530 Subject: [PATCH 018/129] fix: use list[float] vector column in embed generate Parquet output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qodo finding: embed generate wrote scalar dim_* columns, but embed index only detects embeddings when a column's values are list/np.ndarray. This broke the generate→index pipeline with 'No vector column found'. Fix: write a single 'embedding' column where each value is a list[float], matching what embed index's isinstance(df[c].iloc[0], (list, np.ndarray)) check expects. Row indices serve as ids (embed index will pass ids=None to create_index, which is acceptable — vectors index correctly regardless). Also addressed from Qodo review: - .parquet suffix check is now case-insensitive (.lower()) - pandas already a core dependency (bot was wrong) - pyarrow dependency remains added --- semantica/cli.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/semantica/cli.py b/semantica/cli.py index b5e14ebd..f55d8939 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -1716,11 +1716,17 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str, arr = np.asarray(result) if arr.ndim == 1: arr = arr[np.newaxis, :] - # schema: one column per embedding plus an id column - columns = [f"dim_{i}" for i in range(arr.shape[1])] - df = pd.DataFrame(arr, columns=columns) + # Schema: single 'embedding' column (list[float] per row). + # embed index detects vector columns via + # isinstance(df[c].iloc[0], (list, np.ndarray)). + # Row indices serve as ids: embed index will see ids=None + # but vectors will index correctly regardless. + df = pd.DataFrame({ + "embedding": [list(row) for row in arr], + }) df.index.name = "id" - df.to_parquet(output_path, index=True) + df.index = [str(i) for i in range(len(arr))] + df.to_parquet(output_path, index=False) else: output_path.write_text( json.dumps(result, default=str), encoding="utf-8" From aee6e5ad9cc9ffc30a8a0c22e639bbb0e89914f0 Mon Sep 17 00:00:00 2001 From: LeonSGP43 Date: Sun, 16 Aug 2026 11:52:43 +0800 Subject: [PATCH 019/129] docs(cookbook): address review - use sequence_id in lineage walk, demonstrate verify_chain in tamper-evidence step Signed-off-by: LeonSGP43 --- cookbook/introduction/22_Provenance_Tracking.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cookbook/introduction/22_Provenance_Tracking.ipynb b/cookbook/introduction/22_Provenance_Tracking.ipynb index ce285f3d..7df9ab1d 100644 --- a/cookbook/introduction/22_Provenance_Tracking.ipynb +++ b/cookbook/introduction/22_Provenance_Tracking.ipynb @@ -129,7 +129,7 @@ "\n", "print(\"\\n--- ordered chain ---\")\n", "for e in prov.trace_lineage(\"claim_biomass_increase\"):\n", - " print(f\"{e.entity_id} | v{getattr(e, 'version', '?')} | {e.activity_id}\")" + " print(f\"{e.entity_id} | seq#{e.sequence_id} | {e.activity_id}\")" ] }, { @@ -200,7 +200,9 @@ "print(\"Checksum verified:\", ok)\n", "\n", "print(\"Computed:\", compute_checksum(entry_biomass)[:16], \"...\")\n", - "print(\"Stored: \", entry_biomass.checksum[:16] if getattr(entry_biomass, 'checksum', None) else \"(see entry fields)\")" + "print(\"Stored: \", entry_biomass.checksum[:16] if getattr(entry_biomass, 'checksum', None) else \"(see entry fields)\")\n", + "chain = prov.verify_chain()\n", + "print(\"Chain verification:\", json.dumps(chain, default=str)[:200])\n" ] }, { From 4b6cc095850e4ed692d600c8ac088faf6de7f07a Mon Sep 17 00:00:00 2001 From: Varun Sahni Date: Sun, 16 Aug 2026 15:49:39 +0530 Subject: [PATCH 020/129] fix: write JSON/JSONL output as real lists, reject unsupported formats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-Parquet branch still used json.dumps(result, default=str), which stringifies numpy arrays to their repr() — the same corrupt-output bug #994 reports, just for .json/.jsonl extensions instead of .parquet. embed index reads .json/.jsonl via pd.read_json(lines=...) and detects a vector column by isinstance(val, (list, np.ndarray)); a repr() string fails that check, so generate→index still breaks for JSON outputs. - .json/.jsonl now use pandas to_json(orient='records') with real lists - Unsupported extensions (.txt, .csv, etc.) now raise ClickException instead of silently writing JSON text, matching embed index behavior - Error message corrected: pyarrow is now a core dep, not an extra --- semantica/cli.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/semantica/cli.py b/semantica/cli.py index f55d8939..37a3a978 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -1709,32 +1709,41 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str, raise click.ClickException(f"Embeddings module not available: {exc}") from exc if output: output_path = Path(output) + suffix = output_path.suffix.lower() try: import numpy as np import pandas as pd - if output_path.suffix.lower() == ".parquet": - arr = np.asarray(result) - if arr.ndim == 1: - arr = arr[np.newaxis, :] + arr = np.asarray(result) + if arr.ndim == 1: + arr = arr[np.newaxis, :] + if arr.ndim != 2: + raise click.ClickException( + f"embed generate --output expects a 1-D or 2-D array, " + f"got {arr.ndim}-D (shape {arr.shape})" + ) + rows = [list(row) for row in arr] + if suffix == ".parquet": # Schema: single 'embedding' column (list[float] per row). # embed index detects vector columns via # isinstance(df[c].iloc[0], (list, np.ndarray)). - # Row indices serve as ids: embed index will see ids=None - # but vectors will index correctly regardless. - df = pd.DataFrame({ - "embedding": [list(row) for row in arr], - }) - df.index.name = "id" - df.index = [str(i) for i in range(len(arr))] + df = pd.DataFrame({"embedding": rows}) df.to_parquet(output_path, index=False) + elif suffix in (".json", ".jsonl"): + df = pd.DataFrame({"embedding": rows}) + df.to_json( + output_path, + orient="records", + lines=(suffix == ".jsonl"), + ) else: - output_path.write_text( - json.dumps(result, default=str), encoding="utf-8" + raise click.ClickException( + f"Unsupported output format '{suffix}'. " + "Use .parquet, .json, or .jsonl" ) except ImportError as exc: raise click.ClickException( f"Missing dependency for --output: {exc}. " - f"Install pyarrow/pandas with: pip install semantica[ingest-parquet]" + "Install pyarrow with: pip install pyarrow" ) from exc _ok(cli_ctx, f"Wrote {output}") elif _is_json(cli_ctx, local_json): From 4a451f410d1d94c7fead07c1188faaf9908bd34f Mon Sep 17 00:00:00 2001 From: OctoBored <212877535+OctoBored@users.noreply.github.com> Date: Mon, 17 Aug 2026 08:16:29 +0000 Subject: [PATCH 021/129] docs: fix broken star history chart in README The star history chart was broken due to GitHub stargazer API restrictions, so it could no longer be rendered. Update the README to point to a working alternative that uses a different data source requiring no API token. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8b89fd5f..8e729b2d 100644 --- a/README.md +++ b/README.md @@ -1566,11 +1566,11 @@ On-premises deployment · Private cloud · Custom domain implementations · SLA- ## Star History - + - - - Star History Chart + + + Star History Chart From 0f308b2078af6dac71322f4943dc12b2af4fc249 Mon Sep 17 00:00:00 2001 From: Sakshi Jain Date: Tue, 18 Aug 2026 12:00:37 +0530 Subject: [PATCH 022/129] feat(explorer): add markdown content preview and source view --- explorer/package-lock.json | 1474 ++++++++++++++++- explorer/package.json | 6 +- .../GraphWorkspace/GraphInspectorPanel.tsx | 14 + .../GraphWorkspace/MarkdownContentViewer.tsx | 337 ++++ explorer/tests/markdownContentViewer.test.ts | 70 + 5 files changed, 1895 insertions(+), 6 deletions(-) create mode 100644 explorer/src/workspaces/GraphWorkspace/MarkdownContentViewer.tsx create mode 100644 explorer/tests/markdownContentViewer.test.ts diff --git a/explorer/package-lock.json b/explorer/package-lock.json index 98e9bc17..f8ffecff 100644 --- a/explorer/package-lock.json +++ b/explorer/package-lock.json @@ -24,6 +24,8 @@ "react-arborist": "^3.4.3", "react-dom": "^19.2.4", "react-dropzone": "^15.0.0", + "react-markdown": "^10.1.0", + "remark-gfm": "^4.0.1", "sigma": "^3.0.2", "vis-data": "^8.0.3", "vis-timeline": "^8.5.0" @@ -1565,6 +1567,15 @@ "@types/d3-selection": "*" } }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/esrecurse": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", @@ -1576,9 +1587,17 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "dev": true, "license": "MIT" }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, "node_modules/@types/hammerjs": { "version": "2.0.46", "resolved": "https://registry.npmjs.org/@types/hammerjs/-/hammerjs-2.0.46.tgz", @@ -1586,6 +1605,15 @@ "license": "MIT", "peer": true }, + "node_modules/@types/hast": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", + "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -1593,6 +1621,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "24.12.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", @@ -1607,7 +1650,6 @@ "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", - "devOptional": true, "license": "MIT", "dependencies": { "csstype": "^3.2.2" @@ -1631,6 +1673,12 @@ "optional": true, "peer": true }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.58.2", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.2.tgz", @@ -1874,6 +1922,12 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.3.tgz", + "integrity": "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==", + "license": "ISC" + }, "node_modules/@vitejs/plugin-react": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", @@ -1992,6 +2046,16 @@ "@babel/types": "^7.26.0" } }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -2083,12 +2147,72 @@ ], "license": "CC-BY-4.0" }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/classcat": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.5.tgz", "integrity": "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==", "license": "MIT" }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -2139,7 +2263,6 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "devOptional": true, "license": "MIT" }, "node_modules/d3-color": { @@ -2251,7 +2374,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2265,6 +2387,19 @@ } } }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -2272,6 +2407,28 @@ "dev": true, "license": "MIT" }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/dnd-core": { "version": "14.0.1", "resolved": "https://registry.npmjs.org/dnd-core/-/dnd-core-14.0.1.tgz", @@ -2549,6 +2706,16 @@ "node": ">=4.0" } }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -2568,6 +2735,12 @@ "node": ">=0.8.x" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -2819,6 +2992,46 @@ "graphology-types": ">=0.23.0" } }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", + "integrity": "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-js": "^1.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hermes-estree": { "version": "0.25.1", "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", @@ -2845,6 +3058,16 @@ "react-is": "^16.7.0" } }, + "node_modules/html-url-attributes": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", + "integrity": "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -2865,6 +3088,46 @@ "node": ">=0.8.19" } }, + "node_modules/inline-style-parser": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.7.tgz", + "integrity": "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==", + "license": "MIT" + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -2888,6 +3151,28 @@ "node": ">=0.10.0" } }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -2995,6 +3280,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -3026,6 +3321,16 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/marked": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", @@ -3039,12 +3344,857 @@ "node": ">= 18" } }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", + "integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", + "integrity": "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/memoize-one": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==", "license": "MIT" }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", + "integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, "node_modules/minimatch": { "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", @@ -3095,7 +4245,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/nanoid": { @@ -3205,6 +4354,31 @@ "mnemonist": "^0.39.2" } }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -3349,6 +4523,16 @@ "@egjs/hammerjs": "^2.0.17" } }, + "node_modules/property-information": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.2.0.tgz", + "integrity": "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -3459,6 +4643,33 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "license": "MIT" }, + "node_modules/react-markdown": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-10.1.0.tgz", + "integrity": "sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "html-url-attributes": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "unified": "^11.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=18", + "react": ">=18" + } + }, "node_modules/react-refresh": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", @@ -3492,6 +4703,72 @@ "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", "license": "MIT" }, + "node_modules/remark-gfm": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", + "integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.2.tgz", + "integrity": "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/rollup": { "version": "4.60.2", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz", @@ -3596,12 +4873,54 @@ "node": ">=0.10.0" } }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/state-local": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/state-local/-/state-local-1.0.7.tgz", "integrity": "sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==", "license": "MIT" }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/style-to-js": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.21.tgz", + "integrity": "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ==", + "license": "MIT", + "dependencies": { + "style-to-object": "1.0.14" + } + }, + "node_modules/style-to-object": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.14.tgz", + "integrity": "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw==", + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.7" + } + }, "node_modules/tinyglobby": { "version": "0.2.16", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", @@ -3619,6 +4938,26 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/ts-api-utils": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", @@ -3715,6 +5054,93 @@ "devOptional": true, "license": "MIT" }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", + "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/update-browserslist-db": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", @@ -3779,6 +5205,34 @@ "uuid": "dist-node/bin/uuid" } }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/vis-data": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/vis-data/-/vis-data-8.0.3.tgz", @@ -4020,6 +5474,16 @@ "optional": true } } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } } } } diff --git a/explorer/package.json b/explorer/package.json index 162f2bc0..0c6f913a 100644 --- a/explorer/package.json +++ b/explorer/package.json @@ -8,8 +8,10 @@ "build": "tsc -b && vite build", "lint": "eslint .", "preview": "vite preview", + "test": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts", "test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs", - "test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts", + "test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts", + "test:markdown-viewer": "node --import tsx --test tests/markdownContentViewer.test.ts", "test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs" }, "dependencies": { @@ -29,6 +31,8 @@ "react-arborist": "^3.4.3", "react-dom": "^19.2.4", "react-dropzone": "^15.0.0", + "react-markdown": "^10.1.0", + "remark-gfm": "^4.0.1", "sigma": "^3.0.2", "vis-data": "^8.0.3", "vis-timeline": "^8.5.0" diff --git a/explorer/src/workspaces/GraphWorkspace/GraphInspectorPanel.tsx b/explorer/src/workspaces/GraphWorkspace/GraphInspectorPanel.tsx index d2720756..3cae43f6 100644 --- a/explorer/src/workspaces/GraphWorkspace/GraphInspectorPanel.tsx +++ b/explorer/src/workspaces/GraphWorkspace/GraphInspectorPanel.tsx @@ -3,6 +3,7 @@ import { Loader2 } from "lucide-react"; import { graph } from "../../store/graphStore"; import { GRAPH_THEME, withAlpha } from "./graphTheme"; import type { GraphSelectedNodeKind } from "./types"; +import { MarkdownContentViewer } from "./MarkdownContentViewer"; export type LinkPrediction = { target: string; @@ -364,6 +365,11 @@ export function GraphInspectorPanel({ ([key]) => !["x","y","valid_from","valid_until","content","source","source_url","pmid","pmids","evidence","provenance","confidence"].includes(key), ); + const nodeContent = (typeof attributes?.content === "string" && attributes.content) + ? attributes.content + : (typeof properties.content === "string" && properties.content) + ? properties.content + : ""; return (