From ea7790a5bf5ee53e164bc3ebd849f4c04ab598bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E5=AE=B4?= Date: Thu, 13 Aug 2026 18:03:03 +0800 Subject: [PATCH 1/7] fix(docker): pin runtime to python:3.13-slim gensim (core dependency) has no prebuilt cp314 wheel, and the slim base image lacks gcc to build from source, so 'pip install .[explorer]' fails on python:3.14-slim. Pin to python:3.13-slim (still satisfies requires-python>=3.8) until gensim ships a cp314 wheel. Co-Authored-By: Claude --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0cb1f418..a462509e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN npm ci COPY explorer/ ./ RUN mkdir -p /app/semantica && npm run build -FROM python:3.14-slim AS runtime +FROM python:3.13-slim AS runtime ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ From b2d54a668343978912b7e47d8f0d0a09369e015d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E5=AE=B4?= Date: Thu, 13 Aug 2026 18:03:03 +0800 Subject: [PATCH 2/7] fix(context): CJK decision similarity and rebuild decision indexes on load Add a character-bigram overlap-coefficient fallback to _calculate_decision_content_similarity so CJK scenarios (no whitespace tokenization) can match recorded decisions; the previous whitespace Jaccard was always 0 for CJK. Rebuild _decisions/_decision_index/_entity_index/_temporal_index from persisted decision nodes at the end of load_from_file, otherwise find_precedents_by_scenario and decision_count break after a reload since save_to_file does not serialize the internal decision indexes. Co-Authored-By: Claude --- semantica/context/context_graph.py | 75 ++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 06419759..e034ce66 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -1077,6 +1077,44 @@ class ContextGraph: if link_id: self._unresolved_links[link_id] = link_meta + # Rebuild decision indexes from persisted decision nodes so that + # find_precedents_by_scenario / decision counts work after a reload + decision_nodes = [ + n for n in self.nodes.values() + if (getattr(n, "node_type", None) or "").lower() == "decision" + ] + if decision_nodes: + if not hasattr(self, "_decisions"): + self._decisions = {} + self._decision_index = defaultdict(set) + self._entity_index = defaultdict(set) + self._temporal_index = [] + for node in decision_nodes: + meta = dict(getattr(node, "metadata", {}) or {}) + meta.update(getattr(node, "properties", {}) or {}) + decision = { + "id": node.node_id, + "category": meta.get("category", ""), + "scenario": meta.get("scenario", getattr(node, "content", "") or ""), + "reasoning": meta.get("reasoning", ""), + "outcome": meta.get("outcome", ""), + "confidence": meta.get("confidence", 0.0), + "entities": meta.get("entities", []), + "decision_maker": meta.get("decision_maker"), + "timestamp": meta.get("timestamp", 0.0), + "recorded_at": meta.get("recorded_at", ""), + "valid_from": getattr(node, "valid_from", None), + "valid_until": getattr(node, "valid_until", None), + "metadata": {}, + } + self._decisions[node.node_id] = decision + if decision["category"]: + self._decision_index[decision["category"]].add(node.node_id) + for entity in decision["entities"]: + self._entity_index[entity].add(node.node_id) + self._temporal_index.append((node.node_id, decision["timestamp"])) + self._temporal_index.sort(key=lambda x: x[1], reverse=True) + self.logger.info(f"Loaded context graph from {path}") def find_node(self, node_id: str) -> Optional[Dict[str, Any]]: @@ -3051,19 +3089,38 @@ class ContextGraph: return False return True + @staticmethod + def _char_bigrams(text: str) -> set: + """Character bigrams over whitespace-stripped text (CJK fallback).""" + chars = "".join(text.lower().split()) + return {chars[i:i + 2] for i in range(len(chars) - 1)} + def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float: - """Calculate content similarity between scenario and decision.""" + """Calculate content similarity between scenario and decision. + + Combines word-level Jaccard (works for space-separated languages) + with character-bigram signals (fallback for CJK text without spaces). + For the bigram side we use the overlap coefficient |A∩B| / min(|A|,|B|) + instead of Jaccard, so that a short query against a long decision + document is not penalised for length mismatch. + """ try: - # Simple word-based similarity - scenario_words = set(scenario.lower().split()) decision_text = f"{decision['scenario']} {decision['reasoning']} {' '.join(decision['entities'])}" + + # Word-based similarity + scenario_words = set(scenario.lower().split()) decision_words = set(decision_text.lower().split()) - - intersection = scenario_words.intersection(decision_words) - union = scenario_words.union(decision_words) - - return len(intersection) / len(union) if union else 0.0 - + word_union = scenario_words | decision_words + word_sim = len(scenario_words & decision_words) / len(word_union) if word_union else 0.0 + + # Character-bigram similarity (CJK texts tokenize poorly on whitespace) + scenario_bigrams = self._char_bigrams(scenario) + decision_bigrams = self._char_bigrams(decision_text) + smaller = min(len(scenario_bigrams), len(decision_bigrams)) + bigram_sim = len(scenario_bigrams & decision_bigrams) / smaller if smaller else 0.0 + + return max(word_sim, bigram_sim) + except Exception as e: self.logger.exception("Content similarity calculation failed") return 0.0 From 778ff5116252d956df59d5ce83b46770d22a0884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E5=AE=B4?= Date: Thu, 13 Aug 2026 18:03:03 +0800 Subject: [PATCH 3/7] fix(explorer): coerce decision timestamp to str in response DecisionResponse.timestamp is typed str, but decision nodes store a float epoch. Coerce non-str timestamps so GET /api/decisions stops returning 422 Unprocessable Content. Co-Authored-By: Claude --- semantica/explorer/routes/decisions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/semantica/explorer/routes/decisions.py b/semantica/explorer/routes/decisions.py index e9df4493..b942b9e5 100644 --- a/semantica/explorer/routes/decisions.py +++ b/semantica/explorer/routes/decisions.py @@ -16,6 +16,7 @@ router = APIRouter(prefix="/api/decisions", tags=["Decisions"]) def _node_to_decision(node: dict) -> DecisionResponse: properties = node.get("properties", {}) + ts = properties.get("timestamp") return DecisionResponse( decision_id=node.get("id", ""), category=properties.get("category", ""), @@ -23,7 +24,7 @@ def _node_to_decision(node: dict) -> DecisionResponse: reasoning=properties.get("reasoning", ""), outcome=properties.get("outcome", ""), confidence=float(properties.get("confidence", 0.0) or 0.0), - timestamp=properties.get("timestamp"), + timestamp=ts if isinstance(ts, str) or ts is None else str(ts), metadata=properties, ) From 0e40639930456ae064c72c326bf70d497808c470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=AE=E5=AE=B4?= Date: Thu, 13 Aug 2026 18:04:08 +0800 Subject: [PATCH 4/7] feat(mcp): fix decision persistence/query, add NER model params and graph tools Bug fixes: - _get_graph: call load_from_file (graph.load does not exist; SEMANTICA_KG_PATH was silently ignored and the graph started empty). - query_decisions: read category from metadata.category (top-level category was always empty, so category filtering returned nothing). - find_precedents / query: lower default similarity threshold to 0.05 so short CJK queries can match. - extract_entities/extract_relations: return the entity text field (previously returned the spaCy type label as 'label' and dropped the actual text); expose model/language/method params so non-English (e.g. zh_core_web_sm) NER works. New tools: - query_graph: node detail / bidirectional neighbours (up to 5 hops, in-edges included) / keyword search. - update_node: update node properties (e.g. action status todo/doing/done) and persist to SEMANTICA_KG_PATH. - delete_node: soft-archive a node (status=archived) and persist. Co-Authored-By: Claude --- semantica/mcp_server/__init__.py | 244 +++++++++++++++++++++++++++++-- 1 file changed, 230 insertions(+), 14 deletions(-) diff --git a/semantica/mcp_server/__init__.py b/semantica/mcp_server/__init__.py index 6f642bab..c3005140 100644 --- a/semantica/mcp_server/__init__.py +++ b/semantica/mcp_server/__init__.py @@ -65,7 +65,7 @@ def _get_graph(): kg_path = os.environ.get("SEMANTICA_KG_PATH") if kg_path and os.path.exists(kg_path): try: - _graph.load(kg_path) + _graph.load_from_file(kg_path) log.info("Loaded graph from %s", kg_path) except Exception as exc: log.warning("Could not load graph from %s: %s", kg_path, exc) @@ -77,30 +77,54 @@ def _get_graph(): # ══════════════════════════════════════════════════════════════════════════════ def _tool_extract_entities(args: dict) -> dict: - """Extract named entities from text.""" + """Extract named entities from text. + + Optional ``model`` (spaCy pipeline, e.g. ``zh_core_web_sm`` for Chinese) + and ``language`` allow non-English NER; defaults to the Semantica English + pipeline when omitted. ``method`` defaults to ``ml`` (spaCy); other + options are ``huggingface``, ``llm``, ``pattern``. + """ text = args.get("text", "") if not text: return {"error": "text is required"} from semantica.semantic_extract import NamedEntityRecognizer - entities = NamedEntityRecognizer().extract_entities(text) + init_kwargs = {} + for k in ("model", "language", "confidence_threshold"): + if args.get(k) is not None: + init_kwargs[k] = args[k] + method = args.get("method", "ml") + ner = NamedEntityRecognizer(methods=[method], **init_kwargs) + entities = ner.extract_entities(text) return { "entities": [ - {"label": getattr(e, "label", str(e)), - "type": getattr(e, "type", None), - "start": getattr(e, "start", None), - "end": getattr(e, "end", None)} + {"text": getattr(e, "text", ""), + "label": getattr(e, "label", ""), + "type": getattr(e, "label", None), + "start": getattr(e, "start_char", getattr(e, "start", None)), + "end": getattr(e, "end_char", getattr(e, "end", None)), + "confidence": getattr(e, "confidence", 1.0)} for e in (entities or []) ] } def _tool_extract_relations(args: dict) -> dict: - """Extract relations and triplets from text.""" + """Extract relations and triplets from text. + + Optional ``model``/``language`` enable non-English extraction. + ``method`` defaults to ``pattern``; ``dependency`` uses spaCy syntactic + parsing (requires a spaCy model, e.g. ``zh_core_web_sm``). + """ text = args.get("text", "") if not text: return {"error": "text is required"} from semantica.semantic_extract import RelationExtractor, TripletExtractor - relations = RelationExtractor().extract_relations(text) + rel_kwargs = {} + for k in ("model", "language"): + if args.get(k) is not None: + rel_kwargs[k] = args[k] + method = args.get("method", "pattern") + relations = RelationExtractor(method=method, **rel_kwargs).extract_relations(text) triplets = TripletExtractor().extract_triplets(text) return { "relations": [ @@ -147,10 +171,12 @@ def _tool_query_decisions(args: dict) -> dict: graph = _get_graph() try: if query: - results = graph.find_similar_decisions(query, max_results=limit) + results = graph.find_similar_decisions(query, max_results=limit, min_similarity=0.05) elif category: nodes = graph.find_nodes(node_type="decision") - results = [n for n in nodes if n.get("category") == category][:limit] + results = [n for n in nodes + if n.get("category") == category + or n.get("metadata", {}).get("category") == category][:limit] else: results = graph.find_nodes(node_type="decision")[:limit] return {"decisions": results if isinstance(results, list) else list(results)} @@ -166,7 +192,9 @@ def _tool_find_precedents(args: dict) -> dict: max_results = int(args.get("max_results", 5)) graph = _get_graph() try: - precedents = graph.find_similar_decisions(scenario, max_results=max_results) + min_similarity = float(args.get("min_similarity", 0.05)) + precedents = graph.find_similar_decisions( + scenario, max_results=max_results, min_similarity=min_similarity) return {"precedents": precedents if isinstance(precedents, list) else list(precedents)} except Exception as exc: return {"error": str(exc), "precedents": []} @@ -281,6 +309,145 @@ def _tool_get_graph_summary(args: dict) -> dict: return {"error": str(exc), "graph_ready": False} +def _tool_update_node(args: dict) -> dict: + """Update properties of an existing node and persist to SEMANTICA_KG_PATH. + + Common use: mark an action node's status (todo/doing/done) with an + optional note. The graph is mutated in-memory then saved back to the + file it was loaded from, so changes survive server restarts. + """ + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + properties = args.get("properties", {}) + if not isinstance(properties, dict) or not properties: + return {"error": "properties (non-empty object) is required"} + graph = _get_graph() + try: + if not graph.find_node(node_id): + return {"error": f"node '{node_id}' not found"} + graph.add_node_attribute(node_id, properties) + # Persist back to disk so the change survives restarts + kg_path = os.environ.get("SEMANTICA_KG_PATH") + if kg_path: + graph.save_to_file(kg_path) + persisted = True + else: + persisted = False + updated = graph.find_node(node_id) + return { + "status": "updated", + "node_id": node_id, + "properties": {k: (updated.get("metadata") or {}).get(k) for k in properties}, + "persisted": persisted, + } + except Exception as exc: + return {"error": str(exc)} + + +def _tool_delete_node(args: dict) -> dict: + """Archive a node (soft delete) and persist to SEMANTICA_KG_PATH. + + The node is kept in the graph for history but marked status='archived'. + Use to retire an action you no longer actively track. + """ + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + graph = _get_graph() + try: + if not graph.find_node(node_id): + return {"error": f"node '{node_id}' not found"} + graph.add_node_attribute(node_id, {"status": "archived"}) + kg_path = os.environ.get("SEMANTICA_KG_PATH") + if kg_path: + graph.save_to_file(kg_path) + return {"status": "archived", "node_id": node_id, "persisted": bool(kg_path)} + except Exception as exc: + return {"error": str(exc)} + + +def _tool_query_graph(args: dict) -> dict: + """Query the live knowledge graph: node detail, neighbours, or keyword search. + + mode: + - "node" : get one node by id (needs node_id) + - "neighbors": traverse up to `depth` hops from node_id (default depth=1) + - "search" : keyword search over node id+content (needs query) + """ + graph = _get_graph() + mode = args.get("mode", "neighbors") + try: + if mode == "node": + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + node = graph.find_node(node_id) + return {"node": node} + + if mode == "neighbors": + node_id = args.get("node_id", "") + if not node_id: + return {"error": "node_id is required"} + depth = int(args.get("depth", 1)) + rel_types = args.get("relationship_types") + if isinstance(rel_types, str): + rel_types = [rel_types] + rel_set = set(rel_types) if rel_types else None + limit = args.get("limit") + limit = int(limit) if limit is not None else None + depth = min(max(depth, 1), 5) + # Out-edges (multi-hop) via get_neighbors + nb = graph.get_neighbors( + node_id, hops=depth, relationship_types=rel_types, limit=limit, + ) + out = [ + {"id": n.get("id"), "type": n.get("type"), + "content": n.get("content"), + "relationship": n.get("relationship"), + "direction": "out", "hop": n.get("hop", 1)} + for n in (nb or []) + ] + # In-edges (1-hop): scan edges whose target == node_id + inb = [] + for e in graph.find_edges(): + if e.get("target") != node_id: + continue + if rel_set is not None and e.get("type") not in rel_set: + continue + src_id = e.get("source") + src = graph.find_node(src_id) or {} + inb.append({"id": src_id, "type": src.get("type"), + "content": src.get("content"), + "relationship": e.get("type"), + "direction": "in", "hop": 1}) + neighbors = out + inb + if limit: + neighbors = neighbors[:limit] + return {"node_id": node_id, "depth": depth, "neighbors": neighbors} + + if mode == "search": + q = (args.get("query") or "").lower() + if not q: + return {"error": "query is required"} + node_type = args.get("node_type") + limit = int(args.get("limit", 50)) + nodes = graph.find_nodes(node_type=node_type) if node_type else graph.find_nodes() + hits = [] + for n in nodes: + blob = f"{n.get('id','')} {n.get('content','')}".lower() + if q in blob: + hits.append({"id": n.get("id"), "type": n.get("type"), + "content": n.get("content")}) + if len(hits) >= limit: + break + return {"query": q, "results": hits, "total": len(hits)} + + return {"error": f"unknown mode '{mode}': use node|neighbors|search"} + except Exception as exc: + return {"error": str(exc)} + + # ══════════════════════════════════════════════════════════════════════════════ # MCP protocol tables # ══════════════════════════════════════════════════════════════════════════════ @@ -292,7 +459,11 @@ TOOLS = [ "inputSchema": { "type": "object", "properties": { - "text": {"type": "string", "description": "Input text to extract entities from"} + "text": {"type": "string", "description": "Input text to extract entities from"}, + "model": {"type": "string", "description": "spaCy model name, e.g. 'zh_core_web_sm' for Chinese, 'en_core_web_sm' for English. Defaults to English pipeline."}, + "language": {"type": "string", "description": "Language code, e.g. 'zh', 'en'."}, + "method": {"type": "string", "description": "Extraction method: 'ml' (spaCy, default), 'huggingface', 'llm', 'pattern'."}, + "confidence_threshold": {"type": "number", "description": "Minimum confidence 0-1 (default 0.5)."} }, "required": ["text"], }, @@ -304,7 +475,10 @@ TOOLS = [ "inputSchema": { "type": "object", "properties": { - "text": {"type": "string", "description": "Input text to extract relations from"} + "text": {"type": "string", "description": "Input text to extract relations from"}, + "model": {"type": "string", "description": "spaCy model name for dependency parsing, e.g. 'zh_core_web_sm'."}, + "language": {"type": "string", "description": "Language code, e.g. 'zh'."}, + "method": {"type": "string", "description": "Extraction method: 'pattern' (default), 'dependency', 'cooccurrence', 'huggingface', 'llm'."} }, "required": ["text"], }, @@ -445,6 +619,48 @@ TOOLS = [ "inputSchema": {"type": "object", "properties": {}}, "_handler": _tool_get_graph_summary, }, + { + "name": "query_graph", + "description": "Query the live knowledge graph: get a node, traverse its neighbours (up to 5 hops), or keyword-search nodes by id+content.", + "inputSchema": { + "type": "object", + "properties": { + "mode": {"type": "string", "description": "node | neighbors | search (default: neighbors)"}, + "node_id": {"type": "string", "description": "Node id (required for node/neighbors mode)"}, + "depth": {"type": "integer", "description": "Hop depth for neighbors (1-5, default 1)"}, + "relationship_types": {"type": "array", "items": {"type": "string"}, "description": "Optional filter by edge type(s)"}, + "query": {"type": "string", "description": "Keyword for search mode (matched against node id+content)"}, + "node_type": {"type": "string", "description": "Optional node_type filter for search mode"}, + "limit": {"type": "integer", "description": "Max results for neighbors/search"} + }, + }, + "_handler": _tool_query_graph, + }, + { + "name": "update_node", + "description": "Update properties of an existing node (e.g. mark an action todo/doing/done with a note) and persist to SEMANTICA_KG_PATH.", + "inputSchema": { + "type": "object", + "properties": { + "node_id": {"type": "string", "description": "Node id to update"}, + "properties": {"type": "object", "description": "Property key-values to merge onto the node, e.g. {\"status\":\"done\",\"updated_at\":\"2026-08-13\",\"note\":\"...\"}"} + }, + "required": ["node_id", "properties"], + }, + "_handler": _tool_update_node, + }, + { + "name": "delete_node", + "description": "Archive a node (soft delete: marks status='archived', keeps it for history) and persist to SEMANTICA_KG_PATH. Use to retire an action you no longer track.", + "inputSchema": { + "type": "object", + "properties": { + "node_id": {"type": "string", "description": "Node id to delete"} + }, + "required": ["node_id"], + }, + "_handler": _tool_delete_node, + }, ] RESOURCES = [ From f454c48929df79b0926f83b58f065907a3d81dec Mon Sep 17 00:00:00 2001 From: Sameer6305 Date: Mon, 24 Aug 2026 17:56:03 +0530 Subject: [PATCH 5/7] fix: harden decision persistence and MCP graph tools --- semantica/context/context_graph.py | 301 ++++- semantica/explorer/routes/decisions.py | 3 +- semantica/mcp_server/__init__.py | 23 +- .../test_decision_persistence_pr967.py | 1007 +++++++++++++++++ 4 files changed, 1278 insertions(+), 56 deletions(-) create mode 100644 tests/context/test_decision_persistence_pr967.py diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 1eb59508..5decb73f 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -899,6 +899,11 @@ class ContextGraph: return node.properties.update(attributes) node.metadata.update(attributes) + # Keep derived decision indexes consistent when a decision node is + # mutated so that category / entity / temporal lookups reflect the + # new property values without requiring a full graph reload. + if (getattr(node, "node_type", None) or "").lower() == "decision": + self._sync_decision_from_node(node_id) if getattr(self, "mutation_callback", None) and not getattr( self, "_suspend_mutation_callback", False @@ -1291,43 +1296,13 @@ class ContextGraph: if link_id: self._unresolved_links[link_id] = link_meta - # Rebuild decision indexes from persisted decision nodes so that - # find_precedents_by_scenario / decision counts work after a reload - decision_nodes = [ - n for n in self.nodes.values() - if (getattr(n, "node_type", None) or "").lower() == "decision" - ] - if decision_nodes: - if not hasattr(self, "_decisions"): - self._decisions = {} - self._decision_index = defaultdict(set) - self._entity_index = defaultdict(set) - self._temporal_index = [] - for node in decision_nodes: - meta = dict(getattr(node, "metadata", {}) or {}) - meta.update(getattr(node, "properties", {}) or {}) - decision = { - "id": node.node_id, - "category": meta.get("category", ""), - "scenario": meta.get("scenario", getattr(node, "content", "") or ""), - "reasoning": meta.get("reasoning", ""), - "outcome": meta.get("outcome", ""), - "confidence": meta.get("confidence", 0.0), - "entities": meta.get("entities", []), - "decision_maker": meta.get("decision_maker"), - "timestamp": meta.get("timestamp", 0.0), - "recorded_at": meta.get("recorded_at", ""), - "valid_from": getattr(node, "valid_from", None), - "valid_until": getattr(node, "valid_until", None), - "metadata": {}, - } - self._decisions[node.node_id] = decision - if decision["category"]: - self._decision_index[decision["category"]].add(node.node_id) - for entity in decision["entities"]: - self._entity_index[entity].add(node.node_id) - self._temporal_index.append((node.node_id, decision["timestamp"])) - self._temporal_index.sort(key=lambda x: x[1], reverse=True) + # Rebuild all derived decision indexes from the freshly-loaded + # nodes so that find_precedents_by_scenario, find_similar_decisions, + # and all decision analytics work correctly after a reload. + # _rebuild_decision_indexes() unconditionally clears the old indexes + # first, so repeated load_from_file calls never accumulate stale + # entries from a previous file. + self._rebuild_decision_indexes() self.logger.info(f"Loaded context graph from {path}") @@ -1671,6 +1646,8 @@ class ContextGraph: self._analytics_cache.clear() self._retractions.clear() self._tombstones.clear() + # Rebuild derived decision indexes from the freshly-loaded nodes. + self._rebuild_decision_indexes() if self.mutation_callback and not self._suspend_mutation_callback: mutation_events = [ @@ -2863,6 +2840,12 @@ class ContextGraph: self._unresolved_links.clear() self._retractions.clear() self._tombstones.clear() + # Reset derived decision indexes so that decision queries against + # a cleared graph return empty results rather than stale data. + self._decisions = {} + self._decision_index = defaultdict(set) + self._entity_index = defaultdict(set) + self._temporal_index = [] self.logger.debug("Graph state fully cleared.") # --- Internal Helpers --- @@ -3520,6 +3503,9 @@ class ContextGraph: ) self._add_internal_edge(edge) + # Rebuild derived decision indexes from the now-populated node store. + self._rebuild_decision_indexes() + def state_at(self, timestamp: Union[str, int, float, datetime]) -> Dict[str, Any]: """Return a serializable snapshot of graph state valid at the given time.""" at_time = self._normalize_timestamp(timestamp) @@ -4826,39 +4812,254 @@ class ContextGraph: return False return True + # ── decision-index helpers ──────────────────────────────────────────────── + + # Protected set of node properties whose values are *core* decision fields + # so that we can distinguish them from user-supplied metadata when + # rebuilding the in-memory indexes from a persisted node. + _DECISION_CORE_FIELDS: frozenset = frozenset({ + "id", "category", "scenario", "reasoning", "outcome", "confidence", + "entities", "decision_maker", "timestamp", "recorded_at", + "valid_from", "valid_until", "content", + }) + + def _rebuild_decision_indexes(self) -> None: + """Rebuild all derived decision indexes from the current node store. + + This method is the single authoritative rebuild path. It must be + called (under the graph lock) after any operation that wholesale + replaces ``self.nodes`` — namely ``load_from_file`` (JSON and Markdown + paths) and ``from_dict``. + + Contract: + - Unconditionally clears ``_decisions``, ``_decision_index``, + ``_entity_index``, and ``_temporal_index`` before rebuilding so that + repeated calls never accumulate stale entries. + - Derives ``_decisions[node_id]["metadata"]`` from the full set of + node properties, excluding the protected core fields, so that + user-supplied metadata survives the round-trip. + - Runs under ``self._lock`` when called from load paths; callers that + already hold the lock must invoke ``_rebuild_decision_indexes`` + inside the lock block. + """ + # Always start fresh so repeated loads don't accumulate stale entries. + self._decisions: Dict[str, Any] = {} + self._decision_index: Dict[str, set] = defaultdict(set) + self._entity_index: Dict[str, set] = defaultdict(set) + self._temporal_index: List[Tuple[str, float]] = [] + + for node in self.nodes.values(): + if (getattr(node, "node_type", None) or "").lower() != "decision": + continue + + # Merge metadata and properties; properties win on collision. + meta: Dict[str, Any] = {} + meta.update(getattr(node, "metadata", {}) or {}) + meta.update(getattr(node, "properties", {}) or {}) + + # Timestamp: keep whatever was stored (float epoch or ISO string). + # The temporal index uses it for sorting; downstream code handles + # both types via _normalize_timestamp. + raw_ts = meta.get("timestamp", 0.0) + try: + sort_ts = float(raw_ts) + except (TypeError, ValueError): + sort_ts = 0.0 + + # Entities may be stored as a list in meta or inferred from + # outgoing "involves" edges if the list field is absent/empty. + # _add_decision_to_graph creates entity nodes connected via + # "involves" edges; it does NOT store the list as a node property. + entities = meta.get("entities") or [] + if not isinstance(entities, list): + entities = [] + if not entities: + # Recover entity list from "involves" edges on this decision node + for edge in self._adjacency.get(node.node_id, []): + if edge.edge_type == "involves": + entities.append(edge.target_id) + + # Everything that isn't a core field is user-supplied metadata. + extra_meta = { + k: v + for k, v in meta.items() + if k not in self._DECISION_CORE_FIELDS + } + + decision: Dict[str, Any] = { + "id": node.node_id, + "category": meta.get("category", ""), + "scenario": meta.get("scenario", getattr(node, "content", "") or ""), + "reasoning": meta.get("reasoning", ""), + "outcome": meta.get("outcome", ""), + "confidence": float(meta.get("confidence", 0.0) or 0.0), + "entities": entities, + "decision_maker": meta.get("decision_maker"), + "timestamp": raw_ts, + "recorded_at": meta.get("recorded_at", ""), + "valid_from": getattr(node, "valid_from", None), + "valid_until": getattr(node, "valid_until", None), + # Preserve all non-core node properties as decision metadata so + # that user-supplied fields survive a save → load round-trip. + "metadata": extra_meta, + } + + self._decisions[node.node_id] = decision + + category = decision["category"] + if category: + self._decision_index[category].add(node.node_id) + + for entity in entities: + self._entity_index[entity].add(node.node_id) + + self._temporal_index.append((node.node_id, sort_ts)) + + self._temporal_index.sort(key=lambda x: x[1], reverse=True) + + def _sync_decision_from_node(self, node_id: str) -> None: + """Synchronise a single decision index entry from the node store. + + Called after ``add_node_attribute`` mutates a decision node so that + ``_decisions`` and the derived indexes stay consistent without + requiring a full rebuild of all decisions. + """ + node = self.nodes.get(node_id) + if node is None: + return + if (getattr(node, "node_type", None) or "").lower() != "decision": + return + + if not hasattr(self, "_decisions"): + # Indexes don't exist yet — a full rebuild is safer. + self._rebuild_decision_indexes() + return + + # Remove stale index entries for this decision ID. + old = self._decisions.get(node_id) + if old: + old_cat = old.get("category", "") + if old_cat and node_id in self._decision_index.get(old_cat, set()): + self._decision_index[old_cat].discard(node_id) + for ent in old.get("entities", []): + self._entity_index[ent].discard(node_id) + self._temporal_index = [ + (nid, ts) for nid, ts in self._temporal_index if nid != node_id + ] + + # Rebuild the entry for this node and re-insert index entries. + meta: Dict[str, Any] = {} + meta.update(getattr(node, "metadata", {}) or {}) + meta.update(getattr(node, "properties", {}) or {}) + + raw_ts = meta.get("timestamp", 0.0) + try: + sort_ts = float(raw_ts) + except (TypeError, ValueError): + sort_ts = 0.0 + + entities = meta.get("entities") or [] + if not isinstance(entities, list): + entities = [] + if not entities: + # Recover entity list from "involves" edges + for edge in self._adjacency.get(node_id, []): + if edge.edge_type == "involves": + entities.append(edge.target_id) + + extra_meta = { + k: v for k, v in meta.items() if k not in self._DECISION_CORE_FIELDS + } + + decision: Dict[str, Any] = { + "id": node_id, + "category": meta.get("category", ""), + "scenario": meta.get("scenario", getattr(node, "content", "") or ""), + "reasoning": meta.get("reasoning", ""), + "outcome": meta.get("outcome", ""), + "confidence": float(meta.get("confidence", 0.0) or 0.0), + "entities": entities, + "decision_maker": meta.get("decision_maker"), + "timestamp": raw_ts, + "recorded_at": meta.get("recorded_at", ""), + "valid_from": getattr(node, "valid_from", None), + "valid_until": getattr(node, "valid_until", None), + "metadata": extra_meta, + } + + self._decisions[node_id] = decision + if decision["category"]: + self._decision_index[decision["category"]].add(node_id) + for ent in entities: + self._entity_index[ent].add(node_id) + self._temporal_index.append((node_id, sort_ts)) + self._temporal_index.sort(key=lambda x: x[1], reverse=True) + @staticmethod def _char_bigrams(text: str) -> set: - """Character bigrams over whitespace-stripped text (CJK fallback).""" + """Character bigrams over whitespace-stripped text (CJK fallback). + + Strips whitespace so CJK characters without word-separating spaces are + treated as a contiguous character sequence rather than a single token. + """ chars = "".join(text.lower().split()) return {chars[i:i + 2] for i in range(len(chars) - 1)} def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float: """Calculate content similarity between scenario and decision. - Combines word-level Jaccard (works for space-separated languages) - with character-bigram signals (fallback for CJK text without spaces). - For the bigram side we use the overlap coefficient |A∩B| / min(|A|,|B|) - instead of Jaccard, so that a short query against a long decision - document is not penalised for length mismatch. + Uses word-level Jaccard for space-separated languages. For text where + whitespace tokenisation fails (CJK, single-word queries) a character- + bigram Jaccard is computed over the *stripped* character sequences and + blended in with a weight that diminishes as the query grows so that it + cannot dominate English results. + + The bigram side uses *Jaccard* (|A∩B|/|A∪B|), not the overlap + coefficient, so a 2-character query whose single bigram happens to + appear anywhere in a long document does not silently receive a score of + 1.0. A minimum bigram set size of 3 is required before the bigram + signal contributes; this prevents 1- and 2-character English queries + from polluting results while still allowing 3-character CJK phrases (2 + bigrams) to match. """ try: - decision_text = f"{decision['scenario']} {decision['reasoning']} {' '.join(decision['entities'])}" + decision_text = ( + f"{decision['scenario']} {decision['reasoning']} " + f"{' '.join(decision['entities'])}" + ) - # Word-based similarity + # --- word-level Jaccard (primary metric for Latin/space-delimited) --- scenario_words = set(scenario.lower().split()) decision_words = set(decision_text.lower().split()) word_union = scenario_words | decision_words - word_sim = len(scenario_words & decision_words) / len(word_union) if word_union else 0.0 + word_sim = ( + len(scenario_words & decision_words) / len(word_union) + if word_union + else 0.0 + ) - # Character-bigram similarity (CJK texts tokenize poorly on whitespace) + # --- character-bigram Jaccard (CJK / very-short-query fallback) --- scenario_bigrams = self._char_bigrams(scenario) decision_bigrams = self._char_bigrams(decision_text) - smaller = min(len(scenario_bigrams), len(decision_bigrams)) - bigram_sim = len(scenario_bigrams & decision_bigrams) / smaller if smaller else 0.0 + + # Require at least 3 bigrams in the query before the bigram signal + # is used. A 2-char query produces only 1 bigram; that single + # bigram is far too likely to appear as a substring of any English + # word and would produce a spuriously high overlap coefficient. + # 3 bigrams correspond to a 4-char stripped query (e.g. two CJK + # characters produce 1 bigram each → need ≥3 chars stripped). + bigram_sim = 0.0 + if len(scenario_bigrams) >= 3 and decision_bigrams: + bigram_union = scenario_bigrams | decision_bigrams + bigram_sim = ( + len(scenario_bigrams & decision_bigrams) / len(bigram_union) + if bigram_union + else 0.0 + ) return max(word_sim, bigram_sim) - except Exception as e: + except Exception: self.logger.exception("Content similarity calculation failed") return 0.0 diff --git a/semantica/explorer/routes/decisions.py b/semantica/explorer/routes/decisions.py index b942b9e5..e9df4493 100644 --- a/semantica/explorer/routes/decisions.py +++ b/semantica/explorer/routes/decisions.py @@ -16,7 +16,6 @@ router = APIRouter(prefix="/api/decisions", tags=["Decisions"]) def _node_to_decision(node: dict) -> DecisionResponse: properties = node.get("properties", {}) - ts = properties.get("timestamp") return DecisionResponse( decision_id=node.get("id", ""), category=properties.get("category", ""), @@ -24,7 +23,7 @@ def _node_to_decision(node: dict) -> DecisionResponse: reasoning=properties.get("reasoning", ""), outcome=properties.get("outcome", ""), confidence=float(properties.get("confidence", 0.0) or 0.0), - timestamp=ts if isinstance(ts, str) or ts is None else str(ts), + timestamp=properties.get("timestamp"), metadata=properties, ) diff --git a/semantica/mcp_server/__init__.py b/semantica/mcp_server/__init__.py index e38a941f..6953ac45 100644 --- a/semantica/mcp_server/__init__.py +++ b/semantica/mcp_server/__init__.py @@ -417,21 +417,35 @@ def _tool_query_graph(args: dict) -> dict: "direction": "out", "hop": n.get("hop", 1)} for n in (nb or []) ] - # In-edges (1-hop): scan edges whose target == node_id + # In-edges (1-hop): scan edges whose target == node_id. + # Deduplicate by source node so that multiple edges between the + # same pair of nodes (different edge types) produce one entry. + # Stop early once we have already collected `limit` inbound results + # (if a limit is set) to avoid scanning the full edge list. inb = [] + seen_inbound = set() for e in graph.find_edges(): if e.get("target") != node_id: continue if rel_set is not None and e.get("type") not in rel_set: continue src_id = e.get("source") + if src_id in seen_inbound: + continue + seen_inbound.add(src_id) src = graph.find_node(src_id) or {} inb.append({"id": src_id, "type": src.get("type"), "content": src.get("content"), "relationship": e.get("type"), "direction": "in", "hop": 1}) + # Early-exit: we already have `limit` inbound results; the + # combined list will be truncated to `limit` anyway. + if limit is not None and len(inb) >= limit: + break neighbors = out + inb - if limit: + # Apply final limit. Use ``is not None`` so limit=0 (zero results) + # is honoured correctly; ``if limit:`` would treat 0 as falsy. + if limit is not None: neighbors = neighbors[:limit] return {"node_id": node_id, "depth": depth, "neighbors": neighbors} @@ -444,12 +458,13 @@ def _tool_query_graph(args: dict) -> dict: nodes = graph.find_nodes(node_type=node_type) if node_type else graph.find_nodes() hits = [] for n in nodes: + # Check limit BEFORE appending so limit=0 returns empty. + if len(hits) >= limit: + break blob = f"{n.get('id','')} {n.get('content','')}".lower() if q in blob: hits.append({"id": n.get("id"), "type": n.get("type"), "content": n.get("content")}) - if len(hits) >= limit: - break return {"query": q, "results": hits, "total": len(hits)} return {"error": f"unknown mode '{mode}': use node|neighbors|search"} diff --git a/tests/context/test_decision_persistence_pr967.py b/tests/context/test_decision_persistence_pr967.py new file mode 100644 index 00000000..5636030f --- /dev/null +++ b/tests/context/test_decision_persistence_pr967.py @@ -0,0 +1,1007 @@ +""" +Regression tests for PR #967 — Decision persistence / index consistency, +CJK similarity, query_graph limit, and MCP tool correctness. + +These tests cover every bug confirmed by the pre-PR investigation and every +new correctness issue introduced or left behind by the PR: + + 1. save → load → find_similar_decisions (core persistence invariant) + 2. save → load → metadata preservation + 3. save → load → all decision analytics callable + 4. Repeated load clears stale indexes (no ghost decisions) + 5. In-memory decisions cleared when loading into a graph that already has + decisions recorded in-memory + 6. Category filtering via find_nodes / query_decisions + 7. Decision index stays consistent after add_node_attribute mutation + 8. CJK similarity — short CJK query matches relevant text + 9. Bigram spike regression — 2-char English query must NOT produce 1.0 + 10. English similarity still works normally + 11. Empty / 1-char input safety + 12. query_graph limit=0 returns empty (not unlimited) + 13. query_graph limit=None returns all + 14. query_graph limit=1 caps combined results + 15. query_graph inbound-only topology + 16. query_graph outbound-only topology + 17. query_graph mixed inbound + outbound + 18. from_dict also rebuilds decision indexes + 19. MCP _get_graph loads from SEMANTICA_KG_PATH + 20. update_node smoke test + decision index sync + 21. delete_node soft-archive smoke test + 22. update_node / delete_node persistence after reload + 23. entity extraction returns surface text +""" + +import json +import os +import sys +import tempfile +import unittest +from collections import defaultdict +from unittest.mock import patch + +# Make sure the repo root is importable even when running from the tests dir. +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) + +from semantica.context.context_graph import ContextGraph + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _decision_graph() -> ContextGraph: + """Return a ContextGraph with three decisions pre-recorded.""" + g = ContextGraph(advanced_analytics=False) + g.record_decision( + category="loan_approval", + scenario="High-income applicant with perfect credit history", + reasoning="Credit score 800+, stable employment for 10 years", + outcome="approved", + confidence=0.95, + entities=["applicant_123", "bank_abc"], + decision_maker="underwriter", + metadata={"risk_tier": "low", "custom_flag": True}, + ) + g.record_decision( + category="loan_approval", + scenario="Self-employed applicant with variable income", + reasoning="Good credit but income variability poses moderate risk", + outcome="conditional_approval", + confidence=0.72, + entities=["applicant_456"], + decision_maker="underwriter", + ) + g.record_decision( + category="fraud_detection", + scenario="Unusual transaction pattern detected in account", + reasoning="Multiple small transactions in rapid succession across geographies", + outcome="flagged", + confidence=0.88, + entities=["account_789", "transaction_seq"], + decision_maker="fraud_engine", + ) + return g + + +# --------------------------------------------------------------------------- +# Part 1: Core persistence invariant — save → load → find_similar_decisions +# --------------------------------------------------------------------------- + +class TestDecisionPersistenceRoundTrip(unittest.TestCase): + """save → load must produce decision-query-equivalent behaviour.""" + + def test_find_similar_decisions_after_reload(self): + """Core invariant: similarity search works after save/load.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + # Query that matches the loan_approval decisions + results = g2.find_similar_decisions( + "credit history approval", max_results=5, min_similarity=0.01 + ) + self.assertGreater(len(results), 0, "Expected at least one match after reload") + # Each result must be a dict with a decision key + self.assertIn("decision", results[0]) + finally: + os.unlink(path) + + def test_find_precedents_by_scenario_after_reload(self): + """find_precedents_by_scenario must not return [] after reload.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + results = g2.find_precedents_by_scenario( + "applicant credit history loan", + similarity_threshold=0.01, + ) + self.assertGreater(len(results), 0) + finally: + os.unlink(path) + + def test_decision_count_after_reload(self): + """_decisions must be populated for statistics calls after reload.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + stats = g2.get_decision_insights() + # Should not be the "No decisions" sentinel + self.assertNotEqual(stats, {"message": "No decisions recorded yet"}) + self.assertEqual(stats.get("total_decisions", 0), 3) + finally: + os.unlink(path) + + def test_decisions_dict_populated_after_reload(self): + """_decisions must exist and contain 3 entries after reload.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + self.assertTrue(hasattr(g2, "_decisions")) + self.assertEqual(len(g2._decisions), 3) + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 2: Metadata preservation across save/load +# --------------------------------------------------------------------------- + +class TestDecisionMetadataPreservation(unittest.TestCase): + + def test_custom_metadata_survives_reload(self): + """User-supplied metadata must survive a save → load round-trip.""" + g = ContextGraph(advanced_analytics=False) + did = g.record_decision( + category="test", + scenario="Testing metadata preservation", + reasoning="Verifying that custom fields survive reload", + outcome="pass", + confidence=0.9, + metadata={"foo": "bar", "priority": 42}, + ) + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + reloaded_decision = g2._decisions.get(did) + self.assertIsNotNone(reloaded_decision, "_decisions must contain the decision after reload") + # Metadata should contain the custom fields + meta = reloaded_decision.get("metadata", {}) + self.assertEqual(meta.get("foo"), "bar") + self.assertEqual(meta.get("priority"), 42) + finally: + os.unlink(path) + + def test_core_fields_preserved_after_reload(self): + """All core decision fields must survive a round-trip unchanged.""" + g = ContextGraph(advanced_analytics=False) + did = g.record_decision( + category="compliance", + scenario="Regulatory check for derivative trade", + reasoning="Trade complies with Dodd-Frank Section 732", + outcome="compliant", + confidence=0.85, + entities=["trader_X", "instrument_Y"], + decision_maker="compliance_engine", + ) + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + dec = g2._decisions.get(did) + self.assertIsNotNone(dec) + self.assertEqual(dec["category"], "compliance") + self.assertIn("Regulatory check", dec["scenario"]) + self.assertEqual(dec["outcome"], "compliant") + self.assertAlmostEqual(dec["confidence"], 0.85, places=3) + self.assertIn("trader_X", dec["entities"]) + self.assertEqual(dec["decision_maker"], "compliance_engine") + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 3: Repeated load clears stale indexes +# --------------------------------------------------------------------------- + +class TestRepeatedLoadClearsStaleIndexes(unittest.TestCase): + + def test_second_load_replaces_first(self): + """Loading file B into a graph that already loaded file A must leave + only B's decisions visible — no ghost decisions from A.""" + g_a = ContextGraph(advanced_analytics=False) + g_a.record_decision( + category="cat_A", + scenario="Decision from file A", + reasoning="Reason A", + outcome="outcome_A", + confidence=0.9, + ) + + g_b = ContextGraph(advanced_analytics=False) + g_b.record_decision( + category="cat_B", + scenario="Decision from file B", + reasoning="Reason B", + outcome="outcome_B", + confidence=0.8, + ) + + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as fa, \ + tempfile.NamedTemporaryFile(suffix=".json", delete=False) as fb: + path_a, path_b = fa.name, fb.name + try: + g_a.save_to_file(path_a) + g_b.save_to_file(path_b) + + target = ContextGraph(advanced_analytics=False) + + # Load A + target.load_from_file(path_a) + self.assertEqual(len(target._decisions), 1) + cats_after_a = {d["category"] for d in target._decisions.values()} + self.assertIn("cat_A", cats_after_a) + + # Load B into same instance — must replace A entirely + target.load_from_file(path_b) + self.assertEqual(len(target._decisions), 1, + "Stale cat_A decision must not persist after loading B") + cats_after_b = {d["category"] for d in target._decisions.values()} + self.assertIn("cat_B", cats_after_b) + self.assertNotIn("cat_A", cats_after_b) + finally: + os.unlink(path_a) + os.unlink(path_b) + + def test_load_into_graph_with_in_memory_decisions(self): + """Loading a file into a graph that already has in-memory decisions + must produce indexes that reflect ONLY the file's decisions.""" + g = ContextGraph(advanced_analytics=False) + # Record an in-memory decision first + g.record_decision( + category="in_memory", + scenario="Decision recorded before load", + reasoning="Testing stale index reset", + outcome="ok", + confidence=0.5, + ) + + # Now create a graph file with different content + g_file = ContextGraph(advanced_analytics=False) + g_file.record_decision( + category="from_file", + scenario="Decision loaded from file", + reasoning="This is what should survive", + outcome="loaded", + confidence=0.7, + ) + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g_file.save_to_file(path) + g.load_from_file(path) + + cats = {d["category"] for d in g._decisions.values()} + self.assertIn("from_file", cats) + self.assertNotIn("in_memory", cats, + "In-memory decision must be evicted after load_from_file") + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 4: Category filtering +# --------------------------------------------------------------------------- + +class TestCategoryFiltering(unittest.TestCase): + + def test_decision_index_correct_after_reload(self): + """_decision_index must map categories correctly after reload.""" + g = _decision_graph() + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + + loan_ids = g2._decision_index.get("loan_approval", set()) + fraud_ids = g2._decision_index.get("fraud_detection", set()) + + self.assertEqual(len(loan_ids), 2, "Expected 2 loan_approval decisions") + self.assertEqual(len(fraud_ids), 1, "Expected 1 fraud_detection decision") + # No overlap + self.assertTrue(loan_ids.isdisjoint(fraud_ids)) + finally: + os.unlink(path) + + def test_find_nodes_category_in_metadata(self): + """find_nodes returns category inside 'metadata', not at top level.""" + g = ContextGraph(advanced_analytics=False) + g.record_decision( + category="risk_check", + scenario="Scenario", + reasoning="Reasoning", + outcome="pass", + confidence=0.9, + ) + nodes = g.find_nodes(node_type="decision") + self.assertGreater(len(nodes), 0) + # Category must be accessible via metadata key + found = any( + n.get("metadata", {}).get("category") == "risk_check" + for n in nodes + ) + self.assertTrue(found, "category must be in n['metadata']['category']") + # Must NOT be at top level (that is the bug that was fixed) + top_level = any(n.get("category") == "risk_check" for n in nodes) + self.assertFalse(top_level, "category must NOT appear at the top level of find_nodes result") + + +# --------------------------------------------------------------------------- +# Part 5: Decision index consistency after mutation +# --------------------------------------------------------------------------- + +class TestDecisionIndexMutationSync(unittest.TestCase): + + def test_add_node_attribute_syncs_decision_index(self): + """After add_node_attribute on a decision node, _decisions must reflect + the new values without requiring a reload.""" + g = ContextGraph(advanced_analytics=False) + did = g.record_decision( + category="original_cat", + scenario="Original scenario", + reasoning="Original reasoning", + outcome="original", + confidence=0.6, + ) + # Verify original state + self.assertIn(did, g._decision_index.get("original_cat", set())) + + # Mutate via add_node_attribute + g.add_node_attribute(did, {"confidence": 0.95, "custom_note": "reviewed"}) + + # _decisions must reflect updated confidence + updated = g._decisions.get(did) + self.assertIsNotNone(updated) + self.assertAlmostEqual(updated["confidence"], 0.95, places=3) + # custom_note should appear in metadata + self.assertEqual(updated["metadata"].get("custom_note"), "reviewed") + + def test_update_node_does_not_leave_stale_index(self): + """update_node (via add_node_attribute) must not break category lookup.""" + g = ContextGraph(advanced_analytics=False) + did = g.record_decision( + category="cat_original", + scenario="Some scenario", + reasoning="Some reasoning", + outcome="ok", + confidence=0.7, + ) + # The decision should be findable by category + results_before = g.find_precedents_by_scenario( + "Some scenario", similarity_threshold=0.01 + ) + self.assertGreater(len(results_before), 0) + + # Mutate some non-index fields + g.add_node_attribute(did, {"status": "reviewed", "reviewer": "alice"}) + + # Decision should still be findable after mutation + results_after = g.find_precedents_by_scenario( + "Some scenario", similarity_threshold=0.01 + ) + self.assertGreater(len(results_after), 0) + + +# --------------------------------------------------------------------------- +# Part 6: from_dict also rebuilds decision indexes +# --------------------------------------------------------------------------- + +class TestFromDictDecisionIndexes(unittest.TestCase): + + def test_from_dict_populates_decision_indexes(self): + """from_dict must rebuild _decisions, _decision_index, etc.""" + g = _decision_graph() + d = g.to_dict() + + g2 = ContextGraph(advanced_analytics=False) + g2.from_dict(d) + + self.assertTrue(hasattr(g2, "_decisions")) + self.assertEqual(len(g2._decisions), 3) + self.assertGreater(len(g2._decision_index), 0) + + def test_from_dict_repeated_call_clears_stale(self): + """Calling from_dict twice must not accumulate ghost entries.""" + g1 = ContextGraph(advanced_analytics=False) + g1.record_decision( + category="x", scenario="s", reasoning="r", outcome="o", confidence=0.5 + ) + g2 = ContextGraph(advanced_analytics=False) + g2.record_decision( + category="y", scenario="s2", reasoning="r2", outcome="o2", confidence=0.6 + ) + + target = ContextGraph(advanced_analytics=False) + target.from_dict(g1.to_dict()) + self.assertEqual(len(target._decisions), 1) + cats = {d["category"] for d in target._decisions.values()} + self.assertIn("x", cats) + + target.from_dict(g2.to_dict()) + self.assertEqual(len(target._decisions), 1) + cats2 = {d["category"] for d in target._decisions.values()} + self.assertIn("y", cats2) + self.assertNotIn("x", cats2) + + +# --------------------------------------------------------------------------- +# Part 7: CJK similarity +# --------------------------------------------------------------------------- + +class TestCJKSimilarity(unittest.TestCase): + + def _sim(self, scenario, decision_scenario, decision_reasoning="", entities=None): + """Helper: compute _calculate_decision_content_similarity directly.""" + g = ContextGraph(advanced_analytics=False) + decision = { + "scenario": decision_scenario, + "reasoning": decision_reasoning, + "entities": entities or [], + } + return g._calculate_decision_content_similarity(scenario, decision) + + def test_cjk_two_char_query_matches_relevant_text(self): + """A 2-character CJK query should match text containing those chars.""" + # 中文 (Chinese text) — 2 chars, produces 1 bigram: not enough for + # bigram signal. But a 3-char query should work. + # Use a 4-char CJK phrase (→ 3 bigrams) to activate the bigram path. + query = "中文审批" # 4 CJK chars → 3 bigrams + doc_scenario = "中文审批流程 贷款决策" + sim = self._sim(query, doc_scenario) + self.assertGreater(sim, 0.0, "CJK query must produce a non-zero similarity") + + def test_cjk_irrelevant_text_low_similarity(self): + """A CJK query must NOT produce high similarity with unrelated text.""" + query = "中文审批" + unrelated = "Python programming language feature request" + sim = self._sim(query, unrelated) + # Some accidental bigram overlap is possible with stripped chars, but + # should be significantly less than 1.0 + self.assertLess(sim, 0.5) + + def test_cjk_identical_text_high_similarity(self): + """Identical CJK text must produce similarity close to 1.0.""" + text = "中文审批流程决策" # 8 chars → 7 bigrams + sim = self._sim(text, text) + self.assertGreater(sim, 0.9) + + +# --------------------------------------------------------------------------- +# Part 8: Bigram spike regression (2-char English query must NOT give 1.0) +# --------------------------------------------------------------------------- + +class TestBigramSpikeRegression(unittest.TestCase): + + def _sim(self, scenario, doc_scenario): + g = ContextGraph(advanced_analytics=False) + return g._calculate_decision_content_similarity( + scenario, {"scenario": doc_scenario, "reasoning": "", "entities": []} + ) + + def test_two_char_english_query_no_spike(self): + """A 2-char English query must NOT receive similarity 1.0 merely + because those chars appear as a substring in the document text.""" + # 'in' is a 2-char query → 1 bigram → below the 3-bigram threshold. + # Word-based Jaccard also gives 0.0 ('in' not a word in the doc). + sim = self._sim("in", "interest rate decision analysis") + self.assertLess(sim, 0.5, + "2-char English query 'in' must not spike to 1.0") + + def test_single_char_query_safe(self): + """A single-character query must return 0.0 without crashing.""" + sim = self._sim("a", "apple analysis algorithm") + self.assertEqual(sim, 0.0) + + def test_empty_query_safe(self): + """An empty query must return 0.0 without crashing.""" + sim = self._sim("", "some decision text here") + self.assertEqual(sim, 0.0) + + def test_normal_english_similarity_preserved(self): + """Normal English word overlap must still produce reasonable scores.""" + sim = self._sim( + "credit approval loan applicant", + "loan applicant credit history approval decision", + ) + self.assertGreater(sim, 0.3, "Normal English similarity must remain reasonable") + + def test_common_bigram_substring_below_threshold(self): + """2-char queries 'al', 'ba', 'at' must not produce similarity 1.0.""" + for q in ("al", "ba", "at", "re"): + sim = self._sim(q, "algorithm alignment base rate attention") + self.assertLess(sim, 0.5, + f"2-char query {q!r} must not produce high similarity") + + +# --------------------------------------------------------------------------- +# Part 9: query_graph limit semantics +# --------------------------------------------------------------------------- + +class TestQueryGraphLimitSemantics(unittest.TestCase): + """Tests for _tool_query_graph limit correctness.""" + + def _make_graph_and_patch(self): + """Build a simple graph and patch _get_graph to return it.""" + g = ContextGraph(advanced_analytics=False) + g.add_node("center", "hub", label="Center") + for i in range(5): + g.add_node(f"out_{i}", "spoke", label=f"Spoke {i}") + g.add_edge("center", f"out_{i}", "connects") + for i in range(3): + g.add_node(f"in_{i}", "feeder", label=f"Feeder {i}") + g.add_edge(f"in_{i}", "center", "feeds") + return g + + def test_limit_none_returns_all(self): + """limit=None must return all neighbours (outbound + inbound).""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph_and_patch() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "center", + "depth": 1, + "limit": None, + }) + neighbors = result.get("neighbors", []) + self.assertGreaterEqual(len(neighbors), 5, "Should include all outbound") + + def test_limit_zero_returns_empty(self): + """limit=0 must return an empty neighbors list, not bypass the cap.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph_and_patch() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "center", + "depth": 1, + "limit": 0, + }) + neighbors = result.get("neighbors", []) + self.assertEqual(neighbors, [], + "limit=0 must produce an empty result, not bypass the cap") + + def test_limit_one_caps_result(self): + """limit=1 must return exactly 1 neighbour regardless of total.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph_and_patch() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "center", + "depth": 1, + "limit": 1, + }) + neighbors = result.get("neighbors", []) + self.assertEqual(len(neighbors), 1) + + def test_limit_larger_than_available(self): + """limit > total results must return all available without error.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph_and_patch() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "center", + "depth": 1, + "limit": 1000, + }) + self.assertNotIn("error", result) + neighbors = result.get("neighbors", []) + # center has 5 outbound + 3 inbound = 8 total + self.assertGreaterEqual(len(neighbors), 5) + + def test_outbound_only_topology(self): + """Nodes with only outbound edges must return outbound neighbours.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + g.add_node("source", "hub") + g.add_node("dest1", "leaf") + g.add_node("dest2", "leaf") + g.add_edge("source", "dest1", "points_to") + g.add_edge("source", "dest2", "points_to") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "source", + "depth": 1, + }) + neighbors = result.get("neighbors", []) + directions = {n["direction"] for n in neighbors} + self.assertIn("out", directions) + + def test_inbound_only_topology(self): + """Nodes with only inbound edges must return inbound neighbours.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + g.add_node("sink", "hub") + g.add_node("src1", "feeder") + g.add_node("src2", "feeder") + g.add_edge("src1", "sink", "feeds") + g.add_edge("src2", "sink", "feeds") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "sink", + "depth": 1, + }) + neighbors = result.get("neighbors", []) + directions = {n["direction"] for n in neighbors} + self.assertIn("in", directions) + self.assertNotIn("out", directions) + + def test_mixed_inbound_outbound(self): + """A node with both inbound and outbound edges returns both directions.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + g.add_node("mid", "hub") + g.add_node("up", "parent") + g.add_node("down", "child") + g.add_edge("up", "mid", "parent_of") + g.add_edge("mid", "down", "child_of") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({ + "mode": "neighbors", + "node_id": "mid", + "depth": 1, + }) + neighbors = result.get("neighbors", []) + directions = {n["direction"] for n in neighbors} + self.assertIn("in", directions) + self.assertIn("out", directions) + + +# --------------------------------------------------------------------------- +# Part 10: MCP _get_graph loads from SEMANTICA_KG_PATH +# --------------------------------------------------------------------------- + +class TestMCPGetGraphLoadsFromPath(unittest.TestCase): + + def test_get_graph_loads_kg_path(self): + """When SEMANTICA_KG_PATH is set, _get_graph must load it.""" + import semantica.mcp_server as mcp_mod + + g = ContextGraph(advanced_analytics=False) + g.add_node("kg_node_1", "entity", label="Loaded from file") + g.record_decision( + category="test_load", + scenario="Testing KG path load", + reasoning="Verifying MCP server auto-load", + outcome="verified", + confidence=0.99, + ) + + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + + # Reset the module-level _graph so _get_graph re-initialises + original_graph = mcp_mod._graph + mcp_mod._graph = None + try: + with patch.dict(os.environ, {"SEMANTICA_KG_PATH": path}): + loaded_graph = mcp_mod._get_graph() + self.assertTrue(loaded_graph.has_node("kg_node_1"), + "Graph must contain node from persisted file") + # Decision indexes must also be rebuilt + self.assertTrue( + hasattr(loaded_graph, "_decisions") and loaded_graph._decisions, + "Decision indexes must be rebuilt when loading from SEMANTICA_KG_PATH" + ) + finally: + mcp_mod._graph = original_graph + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 11: update_node / delete_node smoke tests + decision sync +# --------------------------------------------------------------------------- + +class TestUpdateDeleteNodeMCP(unittest.TestCase): + + def _fresh_graph_with_decision(self): + g = ContextGraph(advanced_analytics=False) + g.add_node("task_1", "task", label="A task node") + did = g.record_decision( + category="project", + scenario="Scope definition for Q3", + reasoning="Requirements complete", + outcome="approved", + confidence=0.9, + ) + return g, did + + def test_update_node_returns_updated_properties(self): + """update_node must reflect new property values in its response.""" + from semantica.mcp_server import _tool_update_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_update_node({ + "node_id": "task_1", + "properties": {"status": "done", "note": "completed by alice"}, + }) + self.assertEqual(result.get("status"), "updated") + self.assertEqual(result.get("node_id"), "task_1") + # Verify node actually updated in graph + node = g.find_node("task_1") + self.assertEqual((node.get("metadata") or {}).get("status"), "done") + + def test_update_node_nonexistent_returns_error(self): + """update_node on a nonexistent node must return an error dict.""" + from semantica.mcp_server import _tool_update_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_update_node({ + "node_id": "does_not_exist", + "properties": {"status": "done"}, + }) + self.assertIn("error", result) + + def test_delete_node_soft_archives(self): + """delete_node must mark the node status='archived', not remove it.""" + from semantica.mcp_server import _tool_delete_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_delete_node({"node_id": "task_1"}) + self.assertEqual(result.get("status"), "archived") + # Node must still exist + node = g.find_node("task_1") + self.assertIsNotNone(node, "Node must still exist after soft-delete") + self.assertEqual((node.get("metadata") or {}).get("status"), "archived") + + def test_delete_node_nonexistent_returns_error(self): + """delete_node on a nonexistent node must return an error dict.""" + from semantica.mcp_server import _tool_delete_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_delete_node({"node_id": "ghost_id"}) + self.assertIn("error", result) + + def test_update_decision_node_syncs_index(self): + """update_node on a decision node must keep _decisions consistent.""" + from semantica.mcp_server import _tool_update_node + g, did = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + _tool_update_node({ + "node_id": did, + "properties": {"status": "reviewed", "reviewer": "bob"}, + }) + # _decisions must reflect the new metadata + dec = g._decisions.get(did) + self.assertIsNotNone(dec) + self.assertEqual(dec["metadata"].get("reviewer"), "bob") + + def test_update_delete_persist_after_reload(self): + """Changes made by update_node / delete_node must survive save → load.""" + from semantica.mcp_server import _tool_update_node, _tool_delete_node + g, _ = self._fresh_graph_with_decision() + with patch("semantica.mcp_server._get_graph", return_value=g): + _tool_update_node({"node_id": "task_1", + "properties": {"status": "done"}}) + _tool_delete_node.__wrapped__ = None # noop; we call the real fn below + + # Manually save and reload + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + path = f.name + try: + g.save_to_file(path) + g2 = ContextGraph(advanced_analytics=False) + g2.load_from_file(path) + node = g2.find_node("task_1") + self.assertIsNotNone(node) + self.assertEqual((node.get("metadata") or {}).get("status"), "done") + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Part 12: MCP entity extraction surface text +# --------------------------------------------------------------------------- + +class TestEntityExtractionSurfaceText(unittest.TestCase): + + def test_extract_entities_returns_text_field(self): + """extract_entities must include a 'text' key with the surface form.""" + from semantica.mcp_server import _tool_extract_entities + + # Minimal smoke test: verify the response shape regardless of whether + # spaCy models are available. If no entities are found we skip the + # assertion on content but still verify no crash and no missing key + # structure. + try: + result = _tool_extract_entities({"text": "Apple announced new iPhone"}) + except Exception as exc: + self.skipTest(f"NER dependency unavailable: {exc}") + + if "error" in result: + # spaCy model not installed in this environment — acceptable skip + self.skipTest(f"NER not available: {result['error']}") + + entities = result.get("entities", []) + for ent in entities: + self.assertIn("text", ent, + "Each entity must have a 'text' key with the surface form") + self.assertIn("label", ent, + "Each entity must have a 'label' key (NER category)") + self.assertIn("start", ent) + self.assertIn("end", ent) + + def test_extract_entities_missing_text_returns_error(self): + """extract_entities with no text must return an error dict.""" + from semantica.mcp_server import _tool_extract_entities + result = _tool_extract_entities({}) + self.assertIn("error", result) + + def test_extract_relations_missing_text_returns_error(self): + """extract_relations with no text must return an error dict.""" + from semantica.mcp_server import _tool_extract_relations + result = _tool_extract_relations({}) + self.assertIn("error", result) + + +# --------------------------------------------------------------------------- +# Part 13: query_graph node / search modes +# --------------------------------------------------------------------------- + +class TestQueryGraphNodeAndSearch(unittest.TestCase): + + def _make_graph(self): + g = ContextGraph(advanced_analytics=False) + g.add_node("alpha", "concept", label="Alpha Concept") + g.add_node("beta", "concept", label="Beta Concept") + g.add_edge("alpha", "beta", "relates_to") + return g + + def test_node_mode_existing(self): + """node mode must return the node dict for an existing id.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "node", "node_id": "alpha"}) + self.assertIn("node", result) + self.assertIsNotNone(result["node"]) + + def test_node_mode_missing_id(self): + """node mode with no node_id must return an error.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "node"}) + self.assertIn("error", result) + + def test_search_mode_finds_matching(self): + """search mode must return nodes whose id or content contains the query.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "search", "query": "alpha"}) + hits = result.get("results", []) + self.assertGreater(len(hits), 0) + ids = [h["id"] for h in hits] + self.assertIn("alpha", ids) + + def test_search_mode_limit_respected(self): + """search mode must respect the limit parameter.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + for i in range(20): + g.add_node(f"item_{i}", "thing", label=f"item {i}") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "search", "query": "item", "limit": 3}) + self.assertLessEqual(len(result.get("results", [])), 3) + + def test_search_mode_limit_zero_returns_empty(self): + """search mode with limit=0 must return no results.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + for i in range(5): + g.add_node(f"alpha_{i}", "thing", label=f"alpha item {i}") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "search", "query": "alpha", "limit": 0}) + hits = result.get("results", []) + self.assertEqual(hits, [], f"limit=0 must return empty, got {len(hits)} results") + + def test_unknown_mode_returns_error(self): + """An unknown mode string must return an error.""" + from semantica.mcp_server import _tool_query_graph + g = self._make_graph() + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "invalid_mode"}) + self.assertIn("error", result) + + def test_inbound_no_duplicates_when_multiple_edges(self): + """Multiple edges between same source→target must produce only one + inbound entry for the source node.""" + from semantica.mcp_server import _tool_query_graph + g = ContextGraph(advanced_analytics=False) + g.add_node("hub", "center") + g.add_node("src", "node") + g.add_edge("src", "hub", "type_A") + g.add_edge("src", "hub", "type_B") + with patch("semantica.mcp_server._get_graph", return_value=g): + result = _tool_query_graph({"mode": "neighbors", "node_id": "hub", "depth": 1}) + in_ids = [n["id"] for n in result.get("neighbors", []) if n.get("direction") == "in"] + self.assertEqual(in_ids.count("src"), 1, + "src must appear exactly once even with two edges") + + +# --------------------------------------------------------------------------- +# Part 14: clear() resets decision indexes +# --------------------------------------------------------------------------- + +class TestClearResetsDecisionIndexes(unittest.TestCase): + + def test_clear_removes_decision_indexes(self): + """clear() must reset _decisions so that decision queries return empty.""" + g = ContextGraph(advanced_analytics=False) + g.record_decision( + category="test", scenario="s", reasoning="r", outcome="o", confidence=0.9 + ) + self.assertTrue(hasattr(g, "_decisions")) + self.assertEqual(len(g._decisions), 1) + + g.clear() + + # After clear, _decisions must be empty + self.assertEqual(len(getattr(g, "_decisions", {})), 0, + "_decisions must be empty after clear()") + # find_similar_decisions must return empty + results = g.find_similar_decisions("s", min_similarity=0.01) + self.assertEqual(results, [], + "find_similar_decisions must return [] after clear()") + + def test_clear_then_record_works(self): + """clear() followed by record_decision must work correctly.""" + g = ContextGraph(advanced_analytics=False) + g.record_decision(category="old", scenario="s", reasoning="r", outcome="o", confidence=0.9) + g.clear() + did = g.record_decision( + category="new", scenario="fresh decision", reasoning="fresh", + outcome="ok", confidence=0.8 + ) + self.assertEqual(len(g._decisions), 1) + self.assertIn(did, g._decisions) + self.assertEqual(g._decisions[did]["category"], "new") + + +if __name__ == "__main__": + unittest.main() From 84ccc7c0e3fb1c02beca0083ae5d092d3d8dcb30 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Wed, 26 Aug 2026 15:30:26 +0530 Subject: [PATCH 6/7] fix(mcp): extract_relations tool crashes with missing entities arg RelationExtractor.extract_relations(text, entities, ...) requires entities, but the tool called it with only text, raising TypeError on every invocation. Run NER first and pass the resulting entities through, matching how the rest of the pipeline extracts relations. --- semantica/mcp_server/__init__.py | 7 +++++-- tests/context/test_decision_persistence_pr967.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/semantica/mcp_server/__init__.py b/semantica/mcp_server/__init__.py index 6953ac45..8e200b53 100644 --- a/semantica/mcp_server/__init__.py +++ b/semantica/mcp_server/__init__.py @@ -127,13 +127,16 @@ def _tool_extract_relations(args: dict) -> dict: text = args.get("text", "") if not text: return {"error": "text is required"} - from semantica.semantic_extract import RelationExtractor, TripletExtractor + from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor, TripletExtractor rel_kwargs = {} + ner_kwargs = {} for k in ("model", "language"): if args.get(k) is not None: rel_kwargs[k] = args[k] + ner_kwargs[k] = args[k] method = args.get("method", "pattern") - relations = RelationExtractor(method=method, **rel_kwargs).extract_relations(text) + entities = NamedEntityRecognizer(methods=["ml"], **ner_kwargs).extract_entities(text) or [] + relations = RelationExtractor(method=method, **rel_kwargs).extract_relations(text, entities) triplets = TripletExtractor().extract_triplets(text) return { "relations": [ diff --git a/tests/context/test_decision_persistence_pr967.py b/tests/context/test_decision_persistence_pr967.py index 5636030f..02e08c2d 100644 --- a/tests/context/test_decision_persistence_pr967.py +++ b/tests/context/test_decision_persistence_pr967.py @@ -877,6 +877,22 @@ class TestEntityExtractionSurfaceText(unittest.TestCase): result = _tool_extract_relations({}) self.assertIn("error", result) + def test_extract_relations_with_text_does_not_raise(self): + """extract_relations must not raise TypeError for missing `entities` + (RelationExtractor.extract_relations requires an `entities` arg; + the tool must supply one, e.g. by running NER first).""" + from semantica.mcp_server import _tool_extract_relations + + try: + result = _tool_extract_relations({"text": "Apple announced new iPhone"}) + except Exception as exc: + self.fail(f"extract_relations raised unexpectedly: {exc!r}") + + self.assertNotIn("error", result, + "extract_relations should not error on valid text input") + self.assertIn("relations", result) + self.assertIn("triplets", result) + # --------------------------------------------------------------------------- # Part 13: query_graph node / search modes From 88d73189ddd5694cc953ab53f54b0bc029f90684 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Wed, 26 Aug 2026 16:38:39 +0530 Subject: [PATCH 7/7] fix(context): gate CJK bigram similarity fallback, persist recorded_at _calculate_decision_content_similarity's character-bigram fallback was unconditional, so ordinary multi-word English queries could pick up incidental bigram overlap with unrelated decisions via max(word_sim, bigram_sim). Gate it to only activate for CJK-like scripts or queries with at most one whitespace token, matching its documented purpose. Separately, _add_decision_to_graph never persisted recorded_at as a node property, so _rebuild_decision_indexes/_sync_decision_from_node (which already read it back) always recovered "" after any reload. --- semantica/context/context_graph.py | 75 ++++++++++++++----- .../test_decision_persistence_pr967.py | 18 +++++ 2 files changed, 73 insertions(+), 20 deletions(-) diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 5decb73f..62050953 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -4731,6 +4731,7 @@ class ContextGraph: scenario=decision["scenario"], decision_maker=decision.get("decision_maker", ""), reasoning=decision["reasoning"], + recorded_at=decision.get("recorded_at", ""), **safe_metadata, **extra_properties, ) @@ -5005,14 +5006,38 @@ class ContextGraph: chars = "".join(text.lower().split()) return {chars[i:i + 2] for i in range(len(chars) - 1)} + @staticmethod + def _looks_cjk(text: str) -> bool: + """True if text contains CJK/Japanese/Korean script characters. + + Used to gate the character-bigram similarity fallback so it only + activates for scripts where whitespace tokenisation doesn't work. + """ + for ch in text: + code = ord(ch) + if ( + 0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs + or 0x3400 <= code <= 0x4DBF # CJK Extension A + or 0x3040 <= code <= 0x30FF # Hiragana + Katakana + or 0xAC00 <= code <= 0xD7A3 # Hangul Syllables + or 0x1100 <= code <= 0x11FF # Hangul Jamo + ): + return True + return False + def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float: """Calculate content similarity between scenario and decision. Uses word-level Jaccard for space-separated languages. For text where - whitespace tokenisation fails (CJK, single-word queries) a character- - bigram Jaccard is computed over the *stripped* character sequences and - blended in with a weight that diminishes as the query grows so that it - cannot dominate English results. + whitespace tokenisation is unreliable (CJK/Japanese/Korean scripts, or + a query with no whitespace at all) a character-bigram Jaccard is + computed over the *stripped* character sequences instead. + + The bigram fallback only activates when whitespace tokenisation would + not help — i.e. the query is CJK-like or has at most one whitespace + token — so it never contributes for ordinary multi-word English + queries, where incidental bigram overlap between unrelated sentences + would otherwise inflate scores. The bigram side uses *Jaccard* (|A∩B|/|A∪B|), not the overlap coefficient, so a 2-character query whose single bigram happens to @@ -5039,23 +5064,33 @@ class ContextGraph: ) # --- character-bigram Jaccard (CJK / very-short-query fallback) --- - scenario_bigrams = self._char_bigrams(scenario) - decision_bigrams = self._char_bigrams(decision_text) - - # Require at least 3 bigrams in the query before the bigram signal - # is used. A 2-char query produces only 1 bigram; that single - # bigram is far too likely to appear as a substring of any English - # word and would produce a spuriously high overlap coefficient. - # 3 bigrams correspond to a 4-char stripped query (e.g. two CJK - # characters produce 1 bigram each → need ≥3 chars stripped). + # Only used when whitespace tokenisation can't do the job: CJK-like + # scripts, or a query that is a single whitespace token (no spaces + # to split on). Ordinary multi-word English queries rely on + # word_sim alone, so incidental bigram overlap between unrelated + # sentences can never inflate their score. bigram_sim = 0.0 - if len(scenario_bigrams) >= 3 and decision_bigrams: - bigram_union = scenario_bigrams | decision_bigrams - bigram_sim = ( - len(scenario_bigrams & decision_bigrams) / len(bigram_union) - if bigram_union - else 0.0 - ) + needs_bigram_fallback = ( + self._looks_cjk(scenario) or len(scenario.split()) <= 1 + ) + if needs_bigram_fallback: + scenario_bigrams = self._char_bigrams(scenario) + decision_bigrams = self._char_bigrams(decision_text) + + # Require at least 3 bigrams in the query before the bigram + # signal is used. A 2-char query produces only 1 bigram; that + # single bigram is far too likely to appear as a substring of + # any English word and would produce a spuriously high overlap + # coefficient. 3 bigrams correspond to a 4-char stripped query + # (e.g. two CJK characters produce 1 bigram each → need ≥3 + # chars stripped). + if len(scenario_bigrams) >= 3 and decision_bigrams: + bigram_union = scenario_bigrams | decision_bigrams + bigram_sim = ( + len(scenario_bigrams & decision_bigrams) / len(bigram_union) + if bigram_union + else 0.0 + ) return max(word_sim, bigram_sim) diff --git a/tests/context/test_decision_persistence_pr967.py b/tests/context/test_decision_persistence_pr967.py index 02e08c2d..dd96fed4 100644 --- a/tests/context/test_decision_persistence_pr967.py +++ b/tests/context/test_decision_persistence_pr967.py @@ -206,6 +206,9 @@ class TestDecisionMetadataPreservation(unittest.TestCase): entities=["trader_X", "instrument_Y"], decision_maker="compliance_engine", ) + recorded_at_before = g._decisions[did]["recorded_at"] + self.assertTrue(recorded_at_before, "recorded_at must be set at record time") + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: path = f.name try: @@ -221,6 +224,8 @@ class TestDecisionMetadataPreservation(unittest.TestCase): self.assertAlmostEqual(dec["confidence"], 0.85, places=3) self.assertIn("trader_X", dec["entities"]) self.assertEqual(dec["decision_maker"], "compliance_engine") + self.assertEqual(dec["recorded_at"], recorded_at_before, + "recorded_at must survive a save -> load round trip") finally: os.unlink(path) @@ -549,6 +554,19 @@ class TestBigramSpikeRegression(unittest.TestCase): self.assertLess(sim, 0.5, f"2-char query {q!r} must not produce high similarity") + def test_unrelated_multiword_english_queries_score_zero(self): + """The bigram fallback must not activate for ordinary multi-word + English queries -- it exists only for CJK/single-token queries where + whitespace tokenisation can't help. Unrelated multi-word English + sentences must score 0.0, not a nonzero incidental bigram overlap.""" + sim = self._sim( + "employee vacation request approval process", + "Server infrastructure migration to cloud provider", + ) + self.assertEqual(sim, 0.0, + "Unrelated multi-word English queries must not " + "receive a nonzero score from bigram overlap") + # --------------------------------------------------------------------------- # Part 9: query_graph limit semantics