From 4d3259df319a2cb9abd4943772fc42e8d0f0aa2c Mon Sep 17 00:00:00 2001 From: T1mn <136770748@qq.com> Date: Wed, 19 Aug 2026 20:15:20 +0800 Subject: [PATCH 01/11] fix(kg): validate entity_id aliases --- semantica/kg/graph_validator.py | 47 ++++++++++++++++++++------------ tests/kg/test_graph_validator.py | 34 +++++++++++++++++++++++ 2 files changed, 63 insertions(+), 18 deletions(-) create mode 100644 tests/kg/test_graph_validator.py diff --git a/semantica/kg/graph_validator.py b/semantica/kg/graph_validator.py index 9d6a2cbb..a8d126ca 100644 --- a/semantica/kg/graph_validator.py +++ b/semantica/kg/graph_validator.py @@ -31,6 +31,7 @@ from dataclasses import dataclass, field from enum import Enum import networkx as nx +from ..utils.entity_ids import get_entity_id from ..utils.logging import get_logger class ValidationSeverity(Enum): @@ -147,19 +148,23 @@ class GraphValidator: # 2. Entity Validation entity_ids = set() for entity in entities: + eid = get_entity_id(entity) + # Check required fields missing = self.required_entity_fields - set(entity.keys()) + missing.discard("id") + if eid is None: + missing.add("id or entity_id") if missing: issues.append(ValidationIssue( code="MISSING_FIELD", message=f"Entity missing required fields: {missing}", severity=ValidationSeverity.ERROR, - element_id=entity.get("id", "unknown"), + element_id=eid or "unknown", element_type="entity" )) # Check ID uniqueness - eid = entity.get("id") if eid: if eid in entity_ids: issues.append(ValidationIssue( @@ -183,18 +188,29 @@ class GraphValidator: element_type="entity" )) + def get_relationship_endpoint(rel, field, alias): + endpoint = rel.get(field) + if endpoint is None: + endpoint = rel.get(alias) + return endpoint + + def is_valid_id(node_id): + if node_id is None: + return False + try: + return node_id in entity_ids + except TypeError: + # Not hashable, so it can't be in the set of string IDs + return False + # 3. Relationship Validation for i, rel in enumerate(relationships): # Endpoints may use either the legacy ``source``/``target`` keys or # the canonical ``source_id``/``target_id`` keys emitted by # ``ContextGraph.to_kg_dict()``. Accept either variant so both # representations validate consistently. - src = rel.get("source") - if src is None: - src = rel.get("source_id") - tgt = rel.get("target") - if tgt is None: - tgt = rel.get("target_id") + src = get_relationship_endpoint(rel, "source", "source_id") + tgt = get_relationship_endpoint(rel, "target", "target_id") # Check required fields: ``type`` plus a resolvable source/target. missing = set() @@ -215,15 +231,6 @@ class GraphValidator: continue # Check Dangling Edges - def is_valid_id(node_id): - if node_id is None: - return False - try: - return node_id in entity_ids - except TypeError: - # Not hashable, so it can't be in the set of string IDs - return False - if not is_valid_id(src): issues.append(ValidationIssue( code="DANGLING_EDGE", @@ -259,7 +266,11 @@ class GraphValidator: try: nx_graph = nx.DiGraph() nx_graph.add_nodes_from(entity_ids) - nx_graph.add_edges_from([(r["source"], r["target"]) for r in relationships if r.get("source") in entity_ids and r.get("target") in entity_ids]) + for relationship in relationships: + src = get_relationship_endpoint(relationship, "source", "source_id") + tgt = get_relationship_endpoint(relationship, "target", "target_id") + if is_valid_id(src) and is_valid_id(tgt): + nx_graph.add_edge(src, tgt) # Check for cycles try: diff --git a/tests/kg/test_graph_validator.py b/tests/kg/test_graph_validator.py new file mode 100644 index 00000000..a0c57add --- /dev/null +++ b/tests/kg/test_graph_validator.py @@ -0,0 +1,34 @@ +from semantica.kg.graph_builder import GraphBuilder +from semantica.kg.graph_validator import GraphValidator + + +def test_entity_id_aliases_are_validated_across_graph_structure(): + """Entity aliases should work for schema and structural validation.""" + graph = GraphBuilder( + merge_entities=True, + entity_resolution_strategy="exact", + resolve_conflicts=False, + ).build( + { + "entities": [ + {"entity_id": "alice:1", "name": "Alice", "type": "Person"}, + {"entity_id": "alice:2", "name": " Alice ", "type": "Person"}, + {"entity_id": "org:1", "name": "Acme", "type": "Organization"}, + ], + "relationships": [ + { + "source_id": "alice:2", + "target_id": "org:1", + "type": "WORKS_FOR", + } + ], + } + ) + + result = GraphValidator().validate(graph) + + assert result.is_valid + assert not any( + issue.code in {"MISSING_FIELD", "DANGLING_EDGE", "ORPHAN_NODES"} + for issue in result.issues + ) From e5c5cf0efa323f2b4ef58c90a94e67b750c2453c Mon Sep 17 00:00:00 2001 From: T1mn <136770748@qq.com> Date: Wed, 19 Aug 2026 23:19:04 +0800 Subject: [PATCH 02/11] fix(kg): harden validator alias handling --- semantica/kg/graph_validator.py | 21 ++++++++++++----- tests/kg/test_graph_validator.py | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/semantica/kg/graph_validator.py b/semantica/kg/graph_validator.py index a8d126ca..a24fd16e 100644 --- a/semantica/kg/graph_validator.py +++ b/semantica/kg/graph_validator.py @@ -166,15 +166,24 @@ class GraphValidator: # Check ID uniqueness if eid: - if eid in entity_ids: + try: + if eid in entity_ids: + issues.append(ValidationIssue( + code="DUPLICATE_ID", + message=f"Duplicate entity ID found: {eid}", + severity=ValidationSeverity.CRITICAL, + element_id=eid, + element_type="entity" + )) + entity_ids.add(eid) + except TypeError: issues.append(ValidationIssue( - code="DUPLICATE_ID", - message=f"Duplicate entity ID found: {eid}", - severity=ValidationSeverity.CRITICAL, - element_id=eid, + code="INVALID_ID", + message=f"Entity ID is not hashable: {eid}", + severity=ValidationSeverity.ERROR, + element_id=str(eid), element_type="entity" )) - entity_ids.add(eid) # Schema Check (if schema provided) if self.schema and "entity_types" in self.schema: diff --git a/tests/kg/test_graph_validator.py b/tests/kg/test_graph_validator.py index a0c57add..103e67b5 100644 --- a/tests/kg/test_graph_validator.py +++ b/tests/kg/test_graph_validator.py @@ -32,3 +32,43 @@ def test_entity_id_aliases_are_validated_across_graph_structure(): issue.code in {"MISSING_FIELD", "DANGLING_EDGE", "ORPHAN_NODES"} for issue in result.issues ) + + +def test_entity_id_and_relationship_aliases_validate_without_builder(): + """Validator endpoint fallbacks should be tested without normalization.""" + result = GraphValidator().validate( + { + "entities": [ + {"entity_id": "alice:1", "name": "Alice", "type": "Person"}, + {"entity_id": "org:1", "name": "Acme", "type": "Organization"}, + ], + "relationships": [ + { + "source_id": "alice:1", + "target_id": "org:1", + "type": "WORKS_FOR", + } + ], + } + ) + + assert result.is_valid + assert not any( + issue.code in {"MISSING_FIELD", "DANGLING_EDGE", "ORPHAN_NODES"} + for issue in result.issues + ) + + +def test_unhashable_entity_id_returns_validation_issue(): + """Invalid unhashable IDs should produce an issue instead of crashing.""" + result = GraphValidator().validate( + { + "entities": [ + {"entity_id": ["alice:1"], "name": "Alice", "type": "Person"} + ], + "relationships": [], + } + ) + + assert not result.is_valid + assert any(issue.code == "INVALID_ID" for issue in result.issues) From 60eb595d6216588dda8d50d9bd0ff2ba821fb90d Mon Sep 17 00:00:00 2001 From: FABIOTESS Date: Thu, 20 Aug 2026 10:14:46 +0100 Subject: [PATCH 03/11] fix(export): keep JSON-LD payloads in the default graph A JSON-LD document with a top-level @id and a top-level @graph is a named graph. Its members become quads named by that @id, and the default graph is left empty. rdflib.Graph.parse() keeps the default graph and discards the rest without reporting anything, so every consumer that loads an export the ordinary way saw the document header and none of the data. _convert_to_jsonld wrote the payload into @graph and then stamped a document @id beside it, which named every list export and every generic-dict export. export_knowledge_graph made it worse: it converted the graph to JSON-LD and handed the finished document back to export(), which converted it a second time. The converted document no longer carries entities/relationships keys, so the second pass treated it as opaque and buried the whole knowledge graph inside @graph, under a name that is a wall-clock timestamp. A two-entity, one-relationship graph exported to JSON-LD parsed as 2 triples with Graph() and 21 quads with Dataset(). The 19 missing triples were the entire knowledge graph. The document node now goes inside @graph when the payload lives there, and is the document itself otherwise, so no export names its own graph by accident. An already-converted document is merged rather than nested, which also stops the export carrying two document nodes and two @context blocks. Semantica's reader has the mirror of this bug (#1129), so these exports could not be read back by Semantica either. --- semantica/export/json_exporter.py | 71 ++++++++++- tests/export/test_jsonld_default_graph.py | 143 ++++++++++++++++++++++ 2 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 tests/export/test_jsonld_default_graph.py diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index c9c587de..c1906250 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -34,6 +34,25 @@ from ..utils.progress_tracker import get_progress_tracker from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri +def _is_jsonld_document(data: Dict[str, Any]) -> bool: + """ + Report whether a dictionary is already a JSON-LD document. + + ``export_knowledge_graph`` converts a knowledge graph to JSON-LD and then + hands the finished document to ``export()``, which converted it a second + time. The converted document no longer carries ``entities``/ + ``relationships`` keys, so the second pass treated it as an opaque value and + buried it inside ``@graph``. + + Args: + data: Dictionary to test + + Returns: + True when the dictionary declares a JSON-LD context + """ + return "@context" in data + + class JSONExporter: """ JSON exporter for knowledge graphs and semantic data. @@ -397,7 +416,16 @@ class JSONExporter: # Convert data based on type if isinstance(data, dict): - if "entities" in data or "relationships" in data: + if _is_jsonld_document(data): + # Already JSON-LD: merge it rather than nesting it. Wrapping a + # converted document in @graph re-typed the payload as a named + # graph and doubled the @context, which is what happened when + # export_knowledge_graph handed its own output back to export(). + context = data.get("@context") + if isinstance(context, dict): + jsonld["@context"].update(context) + jsonld.update({k: v for k, v in data.items() if k != "@context"}) + elif "entities" in data or "relationships" in data: # Knowledge graph structure - use specialized conversion jsonld.update(self._convert_kg_to_jsonld(data, **options)) else: @@ -412,13 +440,46 @@ class JSONExporter: # Add metadata and provenance if requested if include_metadata: - jsonld["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" - if include_provenance: - jsonld["semantica:exportedAt"] = utc_now_iso() - jsonld["semantica:format"] = "json-ld" + self._attach_document_metadata(jsonld, include_provenance) return jsonld + @staticmethod + def _attach_document_metadata( + jsonld: Dict[str, Any], include_provenance: bool + ) -> None: + """ + Attach the export's own metadata without naming the graph. + + A top-level ``@id`` alongside a top-level ``@graph`` is a *named graph*: + the members of ``@graph`` become quads named by that ``@id`` and leave + the default graph empty. ``rdflib.Graph.parse()`` keeps only the default + graph, so every statement in the export was discarded without an error + (2 of 21 statements survived a two-entity knowledge graph). When the + payload lives in ``@graph``, the document node goes in beside it as one + more node; otherwise it is the document itself. + + Args: + jsonld: Document being built, modified in place + include_provenance: Whether to record how and when it was exported + """ + document: Dict[str, Any] = {} + # Do not overwrite an identifier the payload already carries: the + # knowledge-graph conversion names its own document node. + if "@id" not in jsonld: + document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" + if include_provenance: + document["semantica:exportedAt"] = utc_now_iso() + document["semantica:format"] = "json-ld" + + if "@graph" in jsonld: + document.setdefault( + "@id", f"https://semantica.dev/data/{utc_now_iso()}" + ) + jsonld["@graph"] = list(jsonld["@graph"]) + [document] + else: + jsonld.update(document) + def _convert_kg_to_json(self, kg: Dict[str, Any], **options) -> Dict[str, Any]: """ Convert knowledge graph to JSON format. diff --git a/tests/export/test_jsonld_default_graph.py b/tests/export/test_jsonld_default_graph.py new file mode 100644 index 00000000..8d3f08a6 --- /dev/null +++ b/tests/export/test_jsonld_default_graph.py @@ -0,0 +1,143 @@ +"""Every JSON-LD export must put its payload in the default graph. + +A JSON-LD document carrying a top-level ``@id`` *and* a top-level ``@graph`` is +a **named graph**: the contents of ``@graph`` are quads named by that ``@id``, +not triples in the default graph. ``rdflib.Graph.parse()`` — the ordinary way a +Python consumer loads RDF — keeps the default graph and discards the rest, +without an error. ``JSONExporter`` emitted exactly that shape: + +* ``_convert_to_jsonld`` wrote the payload into ``@graph`` and then stamped a + document ``@id`` beside it, so every list export and every generic-dict + export was named; +* ``export_knowledge_graph`` converted the graph to JSON-LD and handed the + finished document back to ``export()``, which converted it a *second* time. + The converted document no longer has ``entities``/``relationships`` keys, so + the second pass treated it as opaque and wrapped it in ``@graph`` — burying + a whole knowledge graph, entities, relationships and all, inside a named + graph whose name is a wall-clock timestamp. + +Measured on v0.6.6: a two-entity, one-relationship graph exported to JSON-LD +parsed as **2 triples** with ``Graph()`` and 21 quads with ``Dataset()``. The 19 +missing triples were the entire knowledge graph, and nothing reported a +problem. Semantica's own reader has the mirror of this bug (#1129), so the +export could not even be read back by Semantica. +""" + +import json + +import pytest +from rdflib import Dataset, Graph + +from semantica.export.json_exporter import JSONExporter + +KG = { + "entities": [ + {"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"}, + {"id": "https://example.org/e2", "text": "Jane Roe", "type": "PERSON"}, + ], + "relationships": [ + { + "source_id": "https://example.org/e1", + "target_id": "https://example.org/e2", + "type": "employs", + } + ], + "metadata": {"source_document": "contract.pdf"}, +} + +RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label" + +PAYLOADS = { + "knowledge_graph": KG, + "list": [ + {"@id": "https://example.org/a", RDFS_LABEL: "A"}, + {"@id": "https://example.org/b", RDFS_LABEL: "B"}, + ], + "generic_dict": {"@id": "https://example.org/x", RDFS_LABEL: "X"}, +} + + +def _write(payload, tmp_path, name="out.jsonld", **options): + path = tmp_path / name + JSONExporter().export(payload, path, format="json-ld", **options) + return path + + +def _counts(path): + """Triples a default-graph reader sees, and quads a quad reader sees.""" + graph = Graph() + graph.parse(str(path), format="json-ld") + dataset = Dataset() + dataset.parse(str(path), format="json-ld") + return len(graph), sum(1 for _ in dataset.quads((None, None, None, None))) + + +@pytest.mark.parametrize("name", sorted(PAYLOADS)) +def test_no_export_hides_its_payload_in_a_named_graph(name, tmp_path): + """A top-level @id beside a top-level @graph names the graph.""" + path = _write(PAYLOADS[name], tmp_path, f"{name}.jsonld") + document = json.loads(path.read_text()) + + assert not ("@id" in document and "@graph" in document), ( + f"{name}: @id + @graph at the top level makes a named graph, " + "which a default-graph reader discards in full" + ) + + +@pytest.mark.parametrize("name", sorted(PAYLOADS)) +def test_a_plain_graph_reader_loses_nothing(name, tmp_path): + """Graph() and Dataset() must agree: no triple may live outside the default graph.""" + path = _write(PAYLOADS[name], tmp_path, f"{name}.jsonld") + triples, quads = _counts(path) + + assert triples == quads, ( + f"{name}: Graph() read {triples} of {quads} statements; " + f"{quads - triples} were dropped silently" + ) + + +def test_exported_knowledge_graph_survives_a_default_graph_read(tmp_path): + """The entities and the relationship must be there after a plain parse.""" + path = tmp_path / "kg.jsonld" + JSONExporter().export_knowledge_graph(KG, path, format="json-ld") + + graph = Graph() + graph.parse(str(path), format="json-ld") + subjects = {str(s) for s in graph.subjects()} + objects = {str(o) for o in graph.objects()} + + assert "https://example.org/e1" in subjects + assert "https://example.org/e2" in subjects + assert "Acme Corp" in objects + assert "Jane Roe" in objects + assert "employs" in objects + + +def test_knowledge_graph_is_not_converted_twice(tmp_path): + """A nested @context is the signature of the document being re-converted.""" + path = tmp_path / "kg.jsonld" + JSONExporter().export_knowledge_graph(KG, path, format="json-ld") + document = json.loads(path.read_text()) + + nested = [ + node + for node in document.get("@graph", []) + if isinstance(node, dict) and "@context" in node + ] + assert nested == [], "the knowledge graph was converted, then converted again" + + +def test_document_provenance_still_reaches_the_default_graph(tmp_path): + """Keeping the payload readable must not cost the export its own metadata.""" + path = _write(PAYLOADS["list"], tmp_path) + + graph = Graph() + graph.parse(str(path), format="json-ld") + predicates = {str(p) for p in graph.predicates()} + + assert "https://semantica.dev/ns#exportedAt" in predicates + assert "https://semantica.dev/ns#format" in predicates + assert {"https://example.org/a", "https://example.org/b"} <= { + str(s) for s in graph.subjects() + } + assert {"A", "B"} <= {str(o) for o in graph.objects()} From 8f6948f85dd26cb6fec33900c272bf9c7f6aceec Mon Sep 17 00:00:00 2001 From: FABIOTESS Date: Thu, 20 Aug 2026 10:22:18 +0100 Subject: [PATCH 04/11] fix(export): close the four review gaps in the default-graph change All four are in the branch that recognises an already-converted document, which has to survive every shape JSON-LD allows rather than the one shape Semantica happens to produce. A knowledge graph carrying a context of its own took the already-JSON-LD branch and skipped its own conversion, leaving entity ids, relationship endpoints, types and confidences as raw keys. The entities/relationships test now runs first, and a converted document never has those keys, so the double-conversion guard is unaffected. A context that is a URL or an array cannot be merged key by key, and was being dropped in favour of Semantica's defaults, silently changing how every term expands. Both are kept as an array now, the caller's winning, which is the same precedence the dictionary branch already used. An explicit null is left alone on purpose: in an array it resets the active context and would take the semantica prefix with it. @graph may be a single node object as well as an array. list() on a dictionary yields its keys, so an object-valued graph was replaced by a list of strings. A caller may hand us a document that is deliberately a named graph. That name is theirs to keep, so it is no longer flattened; it is nested one level and the export's own provenance goes beside it, in the default graph, where a plain reader can see it. Four tests, one per case, all failing before this commit. --- semantica/export/json_exporter.py | 42 ++++++++++--- tests/export/test_jsonld_default_graph.py | 75 +++++++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index c1906250..0c3b492c 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -416,7 +416,13 @@ class JSONExporter: # Convert data based on type if isinstance(data, dict): - if _is_jsonld_document(data): + # A knowledge graph is converted even when it carries a context of + # its own: the specialized conversion is what mints entity ids and + # relationship endpoints, and skipping it leaves them raw keys. + if "entities" in data or "relationships" in data: + # Knowledge graph structure - use specialized conversion + jsonld.update(self._convert_kg_to_jsonld(data, **options)) + elif _is_jsonld_document(data): # Already JSON-LD: merge it rather than nesting it. Wrapping a # converted document in @graph re-typed the payload as a named # graph and doubled the @context, which is what happened when @@ -424,10 +430,15 @@ class JSONExporter: context = data.get("@context") if isinstance(context, dict): jsonld["@context"].update(context) + elif context is not None: + # A context may also be a URL or an array of them, which + # cannot be merged key by key. Keeping both as an array + # preserves the caller's term expansion, which wins over + # ours, while still defining the semantica prefix. An + # explicit null is left alone: in an array it would reset + # the active context and take our own terms with it. + jsonld["@context"] = [jsonld["@context"], context] jsonld.update({k: v for k, v in data.items() if k != "@context"}) - elif "entities" in data or "relationships" in data: - # Knowledge graph structure - use specialized conversion - jsonld.update(self._convert_kg_to_jsonld(data, **options)) else: # Generic dictionary - wrap in @graph jsonld["@graph"] = [data] @@ -463,20 +474,31 @@ class JSONExporter: jsonld: Document being built, modified in place include_provenance: Whether to record how and when it was exported """ + # A caller may hand us a document that is deliberately a named graph. + # That name is theirs to keep, but our own statements must not end up + # inside it, where a default-graph reader would never see them. + payload_is_named_graph = "@id" in jsonld and "@graph" in jsonld + document: Dict[str, Any] = {} # Do not overwrite an identifier the payload already carries: the # knowledge-graph conversion names its own document node. - if "@id" not in jsonld: + if "@id" not in jsonld or payload_is_named_graph: document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" if include_provenance: document["semantica:exportedAt"] = utc_now_iso() document["semantica:format"] = "json-ld" - if "@graph" in jsonld: - document.setdefault( - "@id", f"https://semantica.dev/data/{utc_now_iso()}" - ) - jsonld["@graph"] = list(jsonld["@graph"]) + [document] + if payload_is_named_graph: + named = {key: value for key, value in jsonld.items() if key != "@context"} + for key in [key for key in jsonld if key != "@context"]: + del jsonld[key] + jsonld["@graph"] = [named, document] + elif "@graph" in jsonld: + # @graph may be a single node object as well as an array. list() on + # a dictionary yields its keys, which would discard the node. + members = jsonld["@graph"] + members = list(members) if isinstance(members, list) else [members] + jsonld["@graph"] = members + [document] else: jsonld.update(document) diff --git a/tests/export/test_jsonld_default_graph.py b/tests/export/test_jsonld_default_graph.py index 8d3f08a6..c66b9a1a 100644 --- a/tests/export/test_jsonld_default_graph.py +++ b/tests/export/test_jsonld_default_graph.py @@ -141,3 +141,78 @@ def test_document_provenance_still_reaches_the_default_graph(tmp_path): str(s) for s in graph.subjects() } assert {"A", "B"} <= {str(o) for o in graph.objects()} + + +# The document a caller hands to export() need not be one Semantica built, and +# the branch that recognises an already-converted document has to survive every +# shape JSON-LD allows. Each of the four cases below regressed when that branch +# was first written. + + +def test_a_url_valued_context_is_not_thrown_away(tmp_path): + """@context may be a URL or an array, not only an object.""" + payload = { + "@context": "https://schema.org/", + "@id": "https://example.org/thing", + "name": "Acme Corp", + } + path = _write(payload, tmp_path) + context = json.loads(path.read_text())["@context"] + + flattened = context if isinstance(context, list) else [context] + assert "https://schema.org/" in flattened, ( + "the caller's context was replaced by Semantica's defaults, " + "which silently changes how every term expands" + ) + + +def test_a_graph_given_as_one_node_object_survives(tmp_path): + """@graph may be a single node object; list() on it yields its keys.""" + payload = { + "@context": {"rdfs": "http://www.w3.org/2000/01/rdf-schema#"}, + "@graph": {"@id": "https://example.org/only", "rdfs:label": "Only"}, + } + path = _write(payload, tmp_path) + + graph = Graph() + graph.parse(str(path), format="json-ld") + assert "Only" in {str(o) for o in graph.objects()} + + +def test_a_caller_supplied_named_graph_keeps_our_provenance_readable(tmp_path): + """A deliberate named graph stays named, but must not swallow the export's own metadata.""" + payload = { + "@context": {"rdfs": "http://www.w3.org/2000/01/rdf-schema#"}, + "@id": "https://example.org/named", + "@graph": [{"@id": "https://example.org/n1", "rdfs:label": "N1"}], + } + path = _write(payload, tmp_path) + document = json.loads(path.read_text()) + + assert "@id" not in document or "@graph" not in document + + graph = Graph() + graph.parse(str(path), format="json-ld") + predicates = {str(p) for p in graph.predicates()} + assert "https://semantica.dev/ns#exportedAt" in predicates, ( + "the export's provenance was written inside the caller's named graph, " + "where a default-graph reader cannot see it" + ) + + dataset = Dataset() + dataset.parse(str(path), format="json-ld") + names = {str(c.identifier) for c in dataset.graphs()} + assert "https://example.org/named" in names, "the caller's graph lost its name" + + +def test_a_knowledge_graph_carrying_a_context_is_still_converted(tmp_path): + """entities/relationships must win over the already-JSON-LD branch.""" + payload = dict(KG, **{"@context": {"ex": "https://example.org/ns#"}}) + path = _write(payload, tmp_path) + document = json.loads(path.read_text()) + + assert "semantica:entities" in document, ( + "the knowledge graph skipped its own conversion, so entity ids, " + "endpoints, types and confidences were left as raw keys" + ) + assert "entities" not in document From 729f4fe932f29277e35f6c423595508d00974db2 Mon Sep 17 00:00:00 2001 From: Dwiti Thaker <138315448+DwitiThaker@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:51:45 +0530 Subject: [PATCH 05/11] fix(docker): use Python 3.13 for gensim compatibility (#1172) Docker build was broken on python:3.14-slim because gensim doesn't ship a 3.14 wheel yet (typical of bleeding edge Python), so pip tries to compile it from source and there's no gcc in the slim image. gensim's a core dependency so every build hit this. Went back to 3.13 instead of installing a compiler : simpler, and 3.14 was just a jump from an automated bump PR anyway. Fixes #1025. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0cb1f418..a462509e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN npm ci COPY explorer/ ./ RUN mkdir -p /app/semantica && npm run build -FROM python:3.14-slim AS runtime +FROM python:3.13-slim AS runtime ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ From d4fdc1f0d341a869487d6d5101f3c34d05d161c8 Mon Sep 17 00:00:00 2001 From: hari Date: Sat, 22 Aug 2026 14:00:35 +0530 Subject: [PATCH 06/11] fix(normalize): accept unit aliases during conversion (#939) Convert_units() was validating categories on raw input like "kg" or "ft" instead of the normalized unit name, so aliases got checked against a category list that only has canonical names in it. Any alias-based conversion that should've worked just raised ValidationError instead. Fixed by normalizing both units before the category check runs. Also added foot/yard/mile/gallon to the alias map - they already had conversion factors but weren't mapped to their canonical names, so they'd still have failed even after the above fix. Turned out there was a second bug hiding behind the first one: the category check defaults both sides to None, and None == None is True, so two aliases from different categories that neither resolved to a real category would silently pass instead of raising. kg -> ft would just return a number instead of erroring. Normalizing first fixes this too, since aliases now resolve to their actual categories and the mismatch gets caught. Added a regression test locking that second one down - kg->ft and gal->lb now raise ValidationError instead of silently converting. Fixes #931. --- semantica/normalize/number_normalizer.py | 24 +++++++++++++++++++---- tests/normalize/test_number_normalizer.py | 15 ++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/semantica/normalize/number_normalizer.py b/semantica/normalize/number_normalizer.py index 29911cb9..0846465d 100644 --- a/semantica/normalize/number_normalizer.py +++ b/semantica/normalize/number_normalizer.py @@ -370,8 +370,12 @@ class UnitConverter: Raises: ValidationError: If units are incompatible or not in same category """ - from_unit = from_unit.lower() - to_unit = to_unit.lower() + # Normalize aliases before validating categories and looking up factors. + # The public API documents abbreviations such as ``kg`` and ``km``; + # validating those raw aliases against the canonical category lists + # incorrectly rejected otherwise supported conversions. + from_unit = self.normalize_unit(from_unit) + to_unit = self.normalize_unit(to_unit) # Validate units if not self.validate_units(from_unit, to_unit): @@ -401,8 +405,8 @@ class UnitConverter: Returns: bool: True if units are compatible (same category), False otherwise """ - from_unit = from_unit.lower() - to_unit = to_unit.lower() + from_unit = self.normalize_unit(from_unit) + to_unit = self.normalize_unit(to_unit) # Check if both units exist if ( @@ -498,6 +502,18 @@ class UnitConverter: "ml": "milliliter", "milliliter": "milliliter", "milliliters": "milliliter", + "ft": "foot", + "foot": "foot", + "feet": "foot", + "yd": "yard", + "yard": "yard", + "yards": "yard", + "mi": "mile", + "mile": "mile", + "miles": "mile", + "gal": "gallon", + "gallon": "gallon", + "gallons": "gallon", } return unit_map.get(unit_lower, unit_lower) diff --git a/tests/normalize/test_number_normalizer.py b/tests/normalize/test_number_normalizer.py index a7cf359c..2e06260c 100644 --- a/tests/normalize/test_number_normalizer.py +++ b/tests/normalize/test_number_normalizer.py @@ -1,4 +1,6 @@ import unittest + +from semantica.utils.exceptions import ValidationError from semantica.normalize.number_normalizer import ( NumberNormalizer, UnitConverter, @@ -32,6 +34,19 @@ class TestUnitConverter(unittest.TestCase): # 1 kg = 1000 g self.assertEqual(self.converter.convert_units(1, "kg", "g"), 1000.0) + def test_convert_accepts_aliases_for_category_validation(self): + # Aliases are part of the documented API, not just parsing syntax. + self.assertEqual(self.converter.convert_units(1, "feet", "m"), 0.3048) + self.assertEqual(self.converter.convert_units(1, "gal", "liter"), 3.78541) + + def test_convert_rejects_mismatched_categories_even_for_aliases(self): + # Both units normalize to canonical names first, so the category + # check sees real categories and rejects cross-category conversions. + with self.assertRaises(ValidationError): + self.converter.convert_units(1, "kg", "ft") + with self.assertRaises(ValidationError): + self.converter.convert_units(1, "gal", "lb") + def test_normalize_unit(self): self.assertEqual(self.converter.normalize_unit("km"), "kilometer") self.assertEqual(self.converter.normalize_unit("kgs"), "kilogram") From 8e9f7c5526800d7b4c4a2616afb653dd2770151e Mon Sep 17 00:00:00 2001 From: Aldrin Joseph Date: Sat, 22 Aug 2026 14:22:56 +0530 Subject: [PATCH 07/11] fix(utils): bound caller-controlled keys in validation error messages (#1088) * fix(utils): bound caller-controlled keys in validation error messages (#1001) _require_recognized_keys() and _require_nothing_dropped() interpolated supplied keys directly into ValidationError messages, so a megabyte-long key produced a megabyte-long exception and, through the export wrappers that log the full exception, an equally large log entry. Keys are now rendered through _truncate_key(), which bounds the display at 64 characters with an ellipsis; the supplied payload is never modified. Co-Authored-By: Claude * fix(utils): bound the count of keys shown in validation error messages (#1001) Review feedback: per-key truncation did not bound the number of keys shown, so a payload carrying many short unknown keys could still size the message (and the log entry that records it). _truncate_key_list() caps the display at 8 keys and appends "and N more", keeping the message actionable without letting the payload size it. Co-Authored-By: Claude --------- Co-authored-by: Claude --- semantica/utils/helpers.py | 33 +++++++++- tests/utils/test_normalize_graph_payload.py | 71 +++++++++++++++++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.py index 05ece01f..48065855 100644 --- a/semantica/utils/helpers.py +++ b/semantica/utils/helpers.py @@ -680,6 +680,35 @@ _TRIPLET_KEYS = ("triplets",) # 'metadata' and 'count'. _CONTEXT_KEYS = ("metadata", "statistics", "count") +# Validation errors below interpolate caller-controlled keys. A pathological +# key (megabytes long) would otherwise size the exception string and, through +# the export wrappers that log the full exception, the log entry. The display +# keeps the offending key recognizable while bounding the message. +_MAX_KEY_DISPLAY = 64 + + +def _truncate_key(key: Any) -> str: + """Render a mapping key for an error message, bounded in length.""" + value = str(key) + if len(value) > _MAX_KEY_DISPLAY: + return value[:_MAX_KEY_DISPLAY] + "…" + return value + + +# Truncating each key bounds the per-key cost; capping the count of keys +# shown bounds the total, so a payload carrying many unknown keys cannot +# size the message (or the log entry that records it) either. +_MAX_KEYS_DISPLAY = 8 + + +def _truncate_key_list(keys: Iterable[Any]) -> str: + """Render keys for an error message, bounded in count and length.""" + rendered = [_truncate_key(key) for key in keys] + if len(rendered) <= _MAX_KEYS_DISPLAY: + return ", ".join(f"'{key}'" for key in rendered) + shown = ", ".join(f"'{key}'" for key in rendered[:_MAX_KEYS_DISPLAY]) + return f"{shown}, and {len(rendered) - _MAX_KEYS_DISPLAY} more" + def _require_recognized_keys( payload: Mapping, recognized_keys: Sequence[str], *, what: str @@ -702,7 +731,7 @@ def _require_recognized_keys( if not payload or any(key in payload for key in recognized_keys): return - supplied = ", ".join(f"'{key}'" for key in sorted(map(str, payload))) + supplied = _truncate_key_list(sorted(map(str, payload))) expected = ", ".join(f"'{key}'" for key in recognized_keys) raise ValidationError( f"{what} has no recognized key. Supplied: {supplied}. " @@ -753,7 +782,7 @@ def _require_nothing_dropped( if not dropped: return - named = ", ".join(f"'{key}'" for key in dropped) + named = _truncate_key_list(dropped) expected = ", ".join(f"'{key}'" for key in recognized_keys) raise ValidationError( f"{what} resolved to nothing, but {named} still holds records. " diff --git a/tests/utils/test_normalize_graph_payload.py b/tests/utils/test_normalize_graph_payload.py index 02b53219..6f64aa22 100644 --- a/tests/utils/test_normalize_graph_payload.py +++ b/tests/utils/test_normalize_graph_payload.py @@ -474,6 +474,77 @@ class TestIsRecordBoundary(unittest.TestCase): ) +class TestKeyDisplayBounds(unittest.TestCase): + """Exception messages must not scale with caller-controlled keys (#1001). + + The validation boundary interpolates supplied keys straight into error + messages, so an extremely large key produced an equally large exception + string -- and, through the export wrappers that log the full exception, + an equally large log entry. The displayed key is truncated to a bounded + length while the supplied payload itself is never modified. + """ + + def test_unrecognized_key_display_is_bounded(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"x" * 1_000_000: [ENTITY]}) + + message = str(ctx.exception) + self.assertLess(len(message), 300) + self.assertIn("x" * 64 + "…", message) + # Truncating the supplied key must not cost the actionable part. + self.assertIn("no recognized key", message) + self.assertIn("entities", message) + + def test_dropped_record_key_display_is_bounded(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"entities": [], "y" * 1_000_000: [ENTITY]}) + + message = str(ctx.exception) + self.assertLess(len(message), 300) + self.assertIn("y" * 64 + "…", message) + self.assertIn("holds records", message) + + def test_short_keys_are_displayed_in_full(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"short_key": [ENTITY]}) + + self.assertIn("'short_key'", str(ctx.exception)) + + def test_bounded_display_does_not_mutate_the_payload(self): + big_key = "z" * 1_000_000 + payload = {big_key: [ENTITY]} + + with self.assertRaises(ValidationError): + normalize_graph_payload(payload) + + self.assertEqual(list(payload), [big_key]) + self.assertEqual(payload[big_key], [ENTITY]) + + def test_many_unrecognized_keys_are_summarized(self): + """Per-key truncation does not bound the number of keys shown. + + A payload carrying many short unrecognized keys would still size the + message (and the log entry that records it), so the count of + displayed keys is bounded too. + """ + payload = {f"key_{i}": [ENTITY] for i in range(100)} + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload(payload) + + message = str(ctx.exception) + self.assertLess(len(message), 500) + self.assertIn("and 92 more", message) + + def test_many_dropped_record_keys_are_summarized(self): + payload = {"entities": [], **{f"data_{i}": [ENTITY] for i in range(100)}} + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload(payload) + + message = str(ctx.exception) + self.assertLess(len(message), 500) + self.assertIn("and 92 more", message) + + @dataclass class _DataclassNode: id: str From 394ce5fe61cb4d664439ae471a2623ed13457051 Mon Sep 17 00:00:00 2001 From: Aldrin Joseph Date: Sat, 22 Aug 2026 14:30:28 +0530 Subject: [PATCH 08/11] fix(reasoning): refuse SPARQL query execution instead of returning empty results (#1087) * fix(reasoning): refuse SPARQL query execution instead of returning empty results (#1083) SPARQLReasoner.execute_query() never executed the query: both branches returned an empty SPARQLQueryResult, with or without a triplet store, so callers that trust an empty result as "no matches" silently drew wrong conclusions. Until a real triplet-store execution path lands, the method raises NotImplementedError with an explanation, per the issue's suggestion. The dead cache/inference scaffolding after the execution point is removed along with it. Co-Authored-By: Claude * docs(reasoning): align execute_query() docs with the NotImplementedError contract (#1087) Review feedback: the docstring still carried a "Returns" section and the reasoning guide showed execute_query() returning bindings, both of which now mislead. The docstring documents Raises only, the guide demonstrates expand_query() and points to rdflib for execution until the triplet-store path lands, and query_cache/clear_cache() are marked as reserved for that future execution path. Co-Authored-By: Claude --------- Co-authored-by: Claude --- docs/guides/reasoning.md | 21 ++--- semantica/reasoning/sparql_reasoner.py | 86 +++++-------------- tests/reasoning/test_specialized_reasoners.py | 23 +++++ 3 files changed, 50 insertions(+), 80 deletions(-) diff --git a/docs/guides/reasoning.md b/docs/guides/reasoning.md index 6e43e23f..4df1d010 100644 --- a/docs/guides/reasoning.md +++ b/docs/guides/reasoning.md @@ -269,7 +269,7 @@ print("Loaded {} facts from graph".format(count)) ## Step 5 — SPARQL queries over enriched working memory -After forward chaining has derived new facts, `SPARQLReasoner` lets you query the enriched working memory using SPARQL triple-pattern matching with optional inference expansion: +After forward chaining has derived new facts, `SPARQLReasoner` prepares SPARQL queries over the enriched working memory with optional inference expansion: ```python from semantica.reasoning import SPARQLReasoner @@ -288,22 +288,13 @@ query = """ } """ -# execute_query() runs: expansion → inference → deduplication -result = sparql.execute_query(query) - -for binding in result.bindings: - print("Actor: {:15s} CVE: {}".format( - binding.get("actor", "?"), - binding.get("cve", "?"), - )) - -# metadata shows how many results came from inference vs ground facts -print("Original: {} Inferred: {}".format( - result.metadata.get("original_count", 0), - result.metadata.get("inferred_count", 0), -)) +# expand_query() applies inference rules to the query text: +expanded = sparql.expand_query(query) +print(expanded) ``` +`execute_query()` is not implemented yet: no triplet-store execution path exists, so it raises `NotImplementedError` rather than returning an empty result set that callers would misread as "no matches". Until execution lands, run the expanded query against your RDF store directly (for example with `rdflib`). + Inspect the expanded query before running it: ```python diff --git a/semantica/reasoning/sparql_reasoner.py b/semantica/reasoning/sparql_reasoner.py index 58e0c351..16be2a9f 100644 --- a/semantica/reasoning/sparql_reasoner.py +++ b/semantica/reasoning/sparql_reasoner.py @@ -84,6 +84,9 @@ class SPARQLReasoner: self.triplet_store = self.config.get("triplet_store") self.enable_inference = self.config.get("enable_inference", True) + # Reserved for query caching once a triplet-store execution path + # lands. execute_query() raises NotImplementedError until then, so + # the cache cannot be populated through any public path yet. self.query_cache: Dict[str, Any] = {} def expand_query(self, query: str, **options) -> str: @@ -330,79 +333,32 @@ class SPARQLReasoner: """ Execute SPARQL query with reasoning. + Not implemented: no triplet-store execution path exists yet, so the + query is refused loudly instead of returning an empty result set + that callers would read as "no matches" (issue #1083). + Args: query: SPARQL query string **options: Additional options - Returns: - Query results + Raises: + NotImplementedError: always, until a triplet-store execution + path lands. """ - tracking_id = self.progress_tracker.start_tracking( - module="reasoning", - submodule="SPARQLReasoner", - message="Executing SPARQL query with reasoning", + raise NotImplementedError( + "SPARQLReasoner.execute_query() is not implemented: no " + "triplet-store execution path exists yet. Returning an empty " + "result set would be misread as 'no matches', so the query " + "is refused instead." ) - try: - # Check cache - self.progress_tracker.update_tracking( - tracking_id, message="Checking query cache..." - ) - if query in self.query_cache: - self.logger.debug("Returning cached query result") - self.progress_tracker.stop_tracking( - tracking_id, - status="completed", - message="Returned cached query result", - ) - return self.query_cache[query] - - # Expand query - self.progress_tracker.update_tracking( - tracking_id, message="Expanding query with inference rules..." - ) - expanded_query = self.expand_query(query, **options) - - # Execute query (if triplet store available) - self.progress_tracker.update_tracking( - tracking_id, message="Executing query..." - ) - if self.triplet_store: - # This would call the triplet store's query method - # For now, return empty result - result = SPARQLQueryResult(bindings=[], variables=[]) - else: - # Mock result for testing - result = SPARQLQueryResult(bindings=[], variables=[]) - - # Infer additional results - if self.enable_inference: - self.progress_tracker.update_tracking( - tracking_id, message="Inferring additional results..." - ) - result = self.infer_results(result, **options) - - # Cache result - self.progress_tracker.update_tracking( - tracking_id, message="Caching query result..." - ) - self.query_cache[query] = result - - self.progress_tracker.stop_tracking( - tracking_id, - status="completed", - message=f"Query executed: {len(result.bindings)} results", - ) - return result - - except Exception as e: - self.progress_tracker.stop_tracking( - tracking_id, status="failed", message=str(e) - ) - raise - def clear_cache(self) -> None: - """Clear query cache.""" + """Clear query cache. + + Reserved for when a triplet-store execution path lands: until then, + ``execute_query()`` raises ``NotImplementedError`` and nothing can + populate the cache. + """ self.query_cache.clear() def add_inference_rule(self, rule_definition: str, **options) -> Rule: diff --git a/tests/reasoning/test_specialized_reasoners.py b/tests/reasoning/test_specialized_reasoners.py index 3a519171..51dbdc9a 100644 --- a/tests/reasoning/test_specialized_reasoners.py +++ b/tests/reasoning/test_specialized_reasoners.py @@ -30,6 +30,29 @@ class TestSpecializedReasoners(unittest.TestCase): binding_types = [b.get("x_type") for b in inferred.bindings] self.assertIn("Human", binding_types) + def test_execute_query_raises_not_implemented(self): + """Empty results must not pass as a valid answer (issue #1083). + + Both branches returned ``SPARQLQueryResult(bindings=[], variables=[])`` + -- with or without a triplet store -- so callers that trust an empty + result as "no matches" silently drew wrong conclusions. Until a real + execution path lands, refusing loudly is safer. + """ + reasoner = SPARQLReasoner() + with self.assertRaises(NotImplementedError): + reasoner.execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") + + def test_execute_query_with_triplet_store_raises_not_implemented(self): + reasoner = SPARQLReasoner(triplet_store=object()) + with self.assertRaises(NotImplementedError): + reasoner.execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") + + def test_execute_query_error_explains_why_the_query_is_refused(self): + reasoner = SPARQLReasoner() + with self.assertRaises(NotImplementedError) as ctx: + reasoner.execute_query("SELECT ?s WHERE { ?s ?p ?o }") + self.assertIn("not implemented", str(ctx.exception)) + def test_abductive_reasoner_generate_hypotheses(self): reasoner = AbductiveReasoner() reasoner.reasoner.add_rule("IF Disease(Flu) THEN Symptom(Fever)") From 58125a0a93da6a31bae439417ab959d25d3ae1b2 Mon Sep 17 00:00:00 2001 From: Kevin Date: Sat, 22 Aug 2026 17:13:45 +0800 Subject: [PATCH 09/11] fix(dedup): never merge entities with different explicit types (closes #1137) (#1149) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dedup): never merge entities with different explicit types (fixes #1137) The duplicate candidate confidence scoring only rewarded same-type pairs but never penalized different-type pairs, so a Person 'Alice' and an Organization 'Acme' (different id, type, and name) passed the confidence threshold and were merged, silently dropping one entity. Add a type guard: when both entities carry a non-empty type and they differ, the pair is never a duplicate candidate (confidence 0, reason 'type_mismatch'). Untyped entities and genuinely duplicate same-type pairs keep their previous behavior. Regression tests cover all three cases. * fix(dedup): honor Entity.type and exclude mismatch structurally (review fixes) Two gaps from code review (#1149): 1. _get_entity_value mapped object 'type' exclusively to .label, which Entity objects never have — their type lives on .type. The mismatch guard therefore never saw the type of Entity objects, and differently typed objects could still merge. Read .type first, fall back to .label. 2. The mismatch branch returned a normal candidate with confidence 0.0, but detection filters with >= confidence_threshold, and 0.0 is a documented valid threshold, so mismatches slipped through. Exclude type_mismatch candidates structurally at both filter sites regardless of threshold. Adds tests for Entity objects with different types and for confidence_threshold=0.0. 94 dedup tests pass. --------- --- semantica/deduplication/duplicate_detector.py | 44 ++++++++-- tests/deduplication/test_deduplication.py | 80 +++++++++++++++++++ 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/semantica/deduplication/duplicate_detector.py b/semantica/deduplication/duplicate_detector.py index c7a29d49..e5b91c11 100644 --- a/semantica/deduplication/duplicate_detector.py +++ b/semantica/deduplication/duplicate_detector.py @@ -278,8 +278,12 @@ class DuplicateDetector: for i, (entity1, entity2, score) in enumerate(similarities): candidate = self._create_duplicate_candidate(entity1, entity2, score) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are excluded + # structurally so no threshold value can admit them. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) remaining = total_similarities - (i + 1) @@ -624,8 +628,12 @@ class DuplicateDetector: new_entity, existing_entity, similarity.score ) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are + # excluded structurally regardless of the threshold. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) processed += 1 @@ -723,7 +731,9 @@ class DuplicateDetector: if key == "name": return getattr(entity, "text", default) if key == "type": - return getattr(entity, "label", default) + # Entity objects store the type on .type; extraction entities + # may expose .label. Missing .label never means "no type". + return getattr(entity, "type", default) or getattr(entity, "label", default) if key == "properties": # Check metadata for properties metadata = getattr(entity, "metadata", {}) @@ -757,6 +767,25 @@ class DuplicateDetector: reasons = [] confidence = similarity_score + # Check entity type mismatch first: two entities with different + # explicit types are not duplicates, whatever their similarity. + entity_type1 = self._get_entity_value(entity1, "type") + entity_type2 = self._get_entity_value(entity2, "type") + if entity_type1 and entity_type2 and entity_type1 != entity_type2: + return DuplicateCandidate( + entity1=entity1, + entity2=entity2, + similarity_score=similarity_score, + confidence=0.0, + reasons=["type_mismatch"], + metadata={ + "name_match": False, + "common_properties": 0, + "type_match": False, + "type_mismatch": True, + }, + ) + # Check for exact name match (strong indicator) name1 = str(self._get_entity_value(entity1, "name", "")).lower().strip() name2 = str(self._get_entity_value(entity2, "name", "")).lower().strip() @@ -779,9 +808,8 @@ class DuplicateDetector: # Boost confidence for each matching property confidence += 0.05 * prop_matches - # Check entity type match - entity_type1 = self._get_entity_value(entity1, "type") - entity_type2 = self._get_entity_value(entity2, "type") + # Check entity type match (only boosts when types are equal; mismatch + # is handled above) if entity_type1 and entity_type2 and entity_type1 == entity_type2: reasons.append("same_type") confidence += 0.05 diff --git a/tests/deduplication/test_deduplication.py b/tests/deduplication/test_deduplication.py index dd351642..8dc7b7c2 100644 --- a/tests/deduplication/test_deduplication.py +++ b/tests/deduplication/test_deduplication.py @@ -12,6 +12,7 @@ from semantica.deduplication.cluster_builder import ClusterBuilder from semantica.deduplication.registry import MethodRegistry from semantica.deduplication.config import DeduplicationConfig from semantica.deduplication.methods import get_deduplication_method +from semantica.utils.types import Entity from semantica.utils.progress_tracker import ConsoleProgressDisplay class TestDeduplication(unittest.TestCase): @@ -87,6 +88,85 @@ class TestDeduplication(unittest.TestCase): # One group should have at least 2 entities (the Apple ones) apple_group = next((g for g in groups if len(g.entities) >= 2), None) self.assertIsNotNone(apple_group) + + def test_different_types_are_never_duplicates(self): + """Entities with different non-empty types must not merge (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Person 'Alice' and Organization 'Acme' must not be duplicate candidates", + ) + # GraphBuilder with merge_entities=True must keep both entities + from semantica.kg import GraphBuilder + graph = GraphBuilder(merge_entities=True).build( + {"entities": entities, "relationships": []} + ) + self.assertEqual(len(graph["entities"]), 2) + + def test_same_type_same_name_still_merges(self): + """Type guard must not break legitimate dedup of same-type entities.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Person", "name": "Alice", "text": "Alice"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Same-type same-name entities must still be detected as duplicates" + ) + + def test_untyped_same_name_still_merges(self): + """Entities with no type must retain previous behavior (merge on similarity).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "x1", "name": "Apple"}, + {"id": "x2", "name": "Apple"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Untyped same-name entities must still be detected as duplicates" + ) + + def test_entity_objects_different_types_not_duplicates(self): + """Entity objects expose their type via .type, not .label (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + Entity(id="e1", text="Alice", type="Person"), + Entity(id="e2", text="Acme", type="Organization"), + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Entity objects with different types must never be detected as duplicates", + ) + + def test_zero_threshold_still_excludes_type_mismatch(self): + """Type mismatch must be excluded structurally, not just by confidence 0.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Different-type candidates must be excluded even with confidence_threshold=0.0", + ) def test_entity_merger(self): """Test entity merging.""" From 14091d21fb0887f5b33a39a0f06f90e73a55855a Mon Sep 17 00:00:00 2001 From: cxzg007 <108442142+cxzg007@users.noreply.github.com> Date: Sat, 22 Aug 2026 17:24:47 +0800 Subject: [PATCH 10/11] fix(kg): compute real relationship duration for temporal stability metric (#1143) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit analyze_evolution() previously appended a constant placeholder (durations.append(1)) for every bounded relationship, so the stability metric was always 1.0 when any bounded relationship existed and 0 otherwise, never reflecting actual valid-time durations. Stability now computes the mean valid-time duration in seconds ((valid_until - valid_from).total_seconds()) across relationships with both bounds set; unbounded/half-open intervals are skipped and non-positive intervals clamped to 0. Adds unit tests and a CHANGELOG entry. Co-authored-by: 江俊杰 --- CHANGELOG.md | 5 +++ semantica/kg/temporal_query.py | 12 +++++-- tests/kg/test_kg.py | 63 ++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ac9df42..481a64af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **The temporal-evolution `stability` metric was a hardcoded placeholder, not a duration** + - `TemporalGraphQuery.analyze_evolution()` documents `stability` as a "relationship duration/stability measure", but the implementation appended a constant `1` for every relationship with both `valid_from` and `valid_until` set (`durations.append(1) # Placeholder`). The reported stability was therefore always `1.0` when any bounded relationship existed and `0` otherwise — it never reflected how long relationships actually stayed valid, so it could not distinguish a graph of decade-long relationships from one of one-second relationships + - `stability` now computes the mean valid-time duration in seconds (`(valid_until - valid_from).total_seconds()`) across relationships that have both bounds set. Relationships with a missing or open `valid_from`/`valid_until` are skipped (their duration is unbounded), and non-positive intervals are clamped to `0`; an empty set still reports `0` + - New tests in `tests/kg/test_kg.py` assert the mean-duration result, the skipping of unbounded/half-open intervals, and the empty-graph zero case + - **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai - `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it - In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have diff --git a/semantica/kg/temporal_query.py b/semantica/kg/temporal_query.py index f5bc227a..c335c95c 100644 --- a/semantica/kg/temporal_query.py +++ b/semantica/kg/temporal_query.py @@ -510,6 +510,9 @@ class TemporalGraphQuery: - "count": Number of relationships - "diversity": Number of unique relationship types - "stability": Relationship duration/stability measure + (mean valid-time duration in seconds across + relationships that have both ``valid_from`` and + ``valid_until`` set) **options: Additional analysis options (unused) Returns: @@ -582,14 +585,17 @@ class TemporalGraphQuery: result["diversity"] = len(rel_types) if "stability" in metrics: - # Calculate stability based on relationship duration + # Stability is the mean duration (in seconds) that relationships + # remain valid. Relationships without a bounded validity interval + # (missing/open ``valid_from`` or ``valid_until``) are skipped, and + # non-positive intervals are clamped to zero. durations = [] for rel in relationships: valid_from = self._parse_time(rel.get("valid_from")) valid_until = self._parse_time(rel.get("valid_until")) if valid_from and valid_until: - # Simplified duration calculation - durations.append(1) # Placeholder + duration_seconds = (valid_until - valid_from).total_seconds() + durations.append(max(0.0, duration_seconds)) result["stability"] = sum(durations) / len(durations) if durations else 0 return result diff --git a/tests/kg/test_kg.py b/tests/kg/test_kg.py index a983694d..4d02b1e4 100644 --- a/tests/kg/test_kg.py +++ b/tests/kg/test_kg.py @@ -407,6 +407,69 @@ class TestTemporalGraphQuery(unittest.TestCase): self.assertEqual(result["num_relationships"], 1) + def test_analyze_evolution_stability_is_mean_duration_seconds(self): + day = 86400.0 + graph = { + "relationships": [ + { + "source": "1", + "target": "2", + "type": "a", + "valid_from": "2024-01-01", + "valid_until": "2024-01-02", # 1 day + }, + { + "source": "2", + "target": "3", + "type": "b", + "valid_from": "2024-01-01", + "valid_until": "2024-01-04", # 3 days + }, + ] + } + + result = self.query_engine.analyze_evolution(graph, metrics=["stability"]) + + # Mean of 1-day and 3-day durations == 2 days in seconds. + self.assertAlmostEqual(result["stability"], 2 * day) + + def test_analyze_evolution_stability_skips_unbounded_intervals(self): + graph = { + "relationships": [ + { + "source": "1", + "target": "2", + "type": "bounded", + "valid_from": "2024-01-01", + "valid_until": "2024-01-02", + }, + { + "source": "2", + "target": "3", + "type": "open", + "valid_from": "2024-01-01", + "valid_until": TemporalBound.OPEN, + }, + { + "source": "3", + "target": "4", + "type": "no-start", + "valid_until": "2024-06-01", + }, + ] + } + + result = self.query_engine.analyze_evolution(graph, metrics=["stability"]) + + # Only the fully bounded relationship contributes (1 day). + self.assertAlmostEqual(result["stability"], 86400.0) + + def test_analyze_evolution_stability_empty_is_zero(self): + result = self.query_engine.analyze_evolution( + {"relationships": []}, metrics=["stability"] + ) + self.assertEqual(result["stability"], 0) + def test_query_at_time_legacy_transaction_axis_uses_valid_from_when_recorded_missing(self): graph = { "relationships": [ From 483f53aaa6368783ca4d1dd95a02544027a2cf08 Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:12:52 +0530 Subject: [PATCH 11/11] fix(tests): use exact-equality check to clear CodeQL substring-URL false positive (#1183) CodeQL (py/incomplete-url-substring-sanitization) flagged the "https://schema.org/" in flattened check because it pattern-matches on URL-ish strings tested with `in`. flattened is always a list here, so the check was already exact membership, not a substring test on untrusted input, but the ambiguous idiom tripped the scanner. Rewrite as an explicit equality comparison so the intent is unambiguous. --- tests/export/test_jsonld_default_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/export/test_jsonld_default_graph.py b/tests/export/test_jsonld_default_graph.py index c66b9a1a..e9f8dc09 100644 --- a/tests/export/test_jsonld_default_graph.py +++ b/tests/export/test_jsonld_default_graph.py @@ -160,7 +160,7 @@ def test_a_url_valued_context_is_not_thrown_away(tmp_path): context = json.loads(path.read_text())["@context"] flattened = context if isinstance(context, list) else [context] - assert "https://schema.org/" in flattened, ( + assert any(entry == "https://schema.org/" for entry in flattened), ( "the caller's context was replaced by Semantica's defaults, " "which silently changes how every term expands" )