diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bb3d263..88795f46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **The temporal-evolution `stability` metric was a hardcoded placeholder, not a duration** + - `TemporalGraphQuery.analyze_evolution()` documents `stability` as a "relationship duration/stability measure", but the implementation appended a constant `1` for every relationship with both `valid_from` and `valid_until` set (`durations.append(1) # Placeholder`). The reported stability was therefore always `1.0` when any bounded relationship existed and `0` otherwise — it never reflected how long relationships actually stayed valid, so it could not distinguish a graph of decade-long relationships from one of one-second relationships + - `stability` now computes the mean valid-time duration in seconds (`(valid_until - valid_from).total_seconds()`) across relationships that have both bounds set. Relationships with a missing or open `valid_from`/`valid_until` are skipped (their duration is unbounded), and non-positive intervals are clamped to `0`; an empty set still reports `0` + - New tests in `tests/kg/test_kg.py` assert the mean-duration result, the skipping of unbounded/half-open intervals, and the empty-graph zero case + - **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai - `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it - In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have diff --git a/Dockerfile b/Dockerfile index 0cb1f418..a462509e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN npm ci COPY explorer/ ./ RUN mkdir -p /app/semantica && npm run build -FROM python:3.14-slim AS runtime +FROM python:3.13-slim AS runtime ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ diff --git a/docs/guides/reasoning.md b/docs/guides/reasoning.md index 6e43e23f..4df1d010 100644 --- a/docs/guides/reasoning.md +++ b/docs/guides/reasoning.md @@ -269,7 +269,7 @@ print("Loaded {} facts from graph".format(count)) ## Step 5 — SPARQL queries over enriched working memory -After forward chaining has derived new facts, `SPARQLReasoner` lets you query the enriched working memory using SPARQL triple-pattern matching with optional inference expansion: +After forward chaining has derived new facts, `SPARQLReasoner` prepares SPARQL queries over the enriched working memory with optional inference expansion: ```python from semantica.reasoning import SPARQLReasoner @@ -288,22 +288,13 @@ query = """ } """ -# execute_query() runs: expansion → inference → deduplication -result = sparql.execute_query(query) - -for binding in result.bindings: - print("Actor: {:15s} CVE: {}".format( - binding.get("actor", "?"), - binding.get("cve", "?"), - )) - -# metadata shows how many results came from inference vs ground facts -print("Original: {} Inferred: {}".format( - result.metadata.get("original_count", 0), - result.metadata.get("inferred_count", 0), -)) +# expand_query() applies inference rules to the query text: +expanded = sparql.expand_query(query) +print(expanded) ``` +`execute_query()` is not implemented yet: no triplet-store execution path exists, so it raises `NotImplementedError` rather than returning an empty result set that callers would misread as "no matches". Until execution lands, run the expanded query against your RDF store directly (for example with `rdflib`). + Inspect the expanded query before running it: ```python diff --git a/semantica/deduplication/duplicate_detector.py b/semantica/deduplication/duplicate_detector.py index c7a29d49..e5b91c11 100644 --- a/semantica/deduplication/duplicate_detector.py +++ b/semantica/deduplication/duplicate_detector.py @@ -278,8 +278,12 @@ class DuplicateDetector: for i, (entity1, entity2, score) in enumerate(similarities): candidate = self._create_duplicate_candidate(entity1, entity2, score) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are excluded + # structurally so no threshold value can admit them. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) remaining = total_similarities - (i + 1) @@ -624,8 +628,12 @@ class DuplicateDetector: new_entity, existing_entity, similarity.score ) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are + # excluded structurally regardless of the threshold. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) processed += 1 @@ -723,7 +731,9 @@ class DuplicateDetector: if key == "name": return getattr(entity, "text", default) if key == "type": - return getattr(entity, "label", default) + # Entity objects store the type on .type; extraction entities + # may expose .label. Missing .label never means "no type". + return getattr(entity, "type", default) or getattr(entity, "label", default) if key == "properties": # Check metadata for properties metadata = getattr(entity, "metadata", {}) @@ -757,6 +767,25 @@ class DuplicateDetector: reasons = [] confidence = similarity_score + # Check entity type mismatch first: two entities with different + # explicit types are not duplicates, whatever their similarity. + entity_type1 = self._get_entity_value(entity1, "type") + entity_type2 = self._get_entity_value(entity2, "type") + if entity_type1 and entity_type2 and entity_type1 != entity_type2: + return DuplicateCandidate( + entity1=entity1, + entity2=entity2, + similarity_score=similarity_score, + confidence=0.0, + reasons=["type_mismatch"], + metadata={ + "name_match": False, + "common_properties": 0, + "type_match": False, + "type_mismatch": True, + }, + ) + # Check for exact name match (strong indicator) name1 = str(self._get_entity_value(entity1, "name", "")).lower().strip() name2 = str(self._get_entity_value(entity2, "name", "")).lower().strip() @@ -779,9 +808,8 @@ class DuplicateDetector: # Boost confidence for each matching property confidence += 0.05 * prop_matches - # Check entity type match - entity_type1 = self._get_entity_value(entity1, "type") - entity_type2 = self._get_entity_value(entity2, "type") + # Check entity type match (only boosts when types are equal; mismatch + # is handled above) if entity_type1 and entity_type2 and entity_type1 == entity_type2: reasons.append("same_type") confidence += 0.05 diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index c9c587de..0c3b492c 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -34,6 +34,25 @@ from ..utils.progress_tracker import get_progress_tracker from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri +def _is_jsonld_document(data: Dict[str, Any]) -> bool: + """ + Report whether a dictionary is already a JSON-LD document. + + ``export_knowledge_graph`` converts a knowledge graph to JSON-LD and then + hands the finished document to ``export()``, which converted it a second + time. The converted document no longer carries ``entities``/ + ``relationships`` keys, so the second pass treated it as an opaque value and + buried it inside ``@graph``. + + Args: + data: Dictionary to test + + Returns: + True when the dictionary declares a JSON-LD context + """ + return "@context" in data + + class JSONExporter: """ JSON exporter for knowledge graphs and semantic data. @@ -397,9 +416,29 @@ class JSONExporter: # Convert data based on type if isinstance(data, dict): + # A knowledge graph is converted even when it carries a context of + # its own: the specialized conversion is what mints entity ids and + # relationship endpoints, and skipping it leaves them raw keys. if "entities" in data or "relationships" in data: # Knowledge graph structure - use specialized conversion jsonld.update(self._convert_kg_to_jsonld(data, **options)) + elif _is_jsonld_document(data): + # Already JSON-LD: merge it rather than nesting it. Wrapping a + # converted document in @graph re-typed the payload as a named + # graph and doubled the @context, which is what happened when + # export_knowledge_graph handed its own output back to export(). + context = data.get("@context") + if isinstance(context, dict): + jsonld["@context"].update(context) + elif context is not None: + # A context may also be a URL or an array of them, which + # cannot be merged key by key. Keeping both as an array + # preserves the caller's term expansion, which wins over + # ours, while still defining the semantica prefix. An + # explicit null is left alone: in an array it would reset + # the active context and take our own terms with it. + jsonld["@context"] = [jsonld["@context"], context] + jsonld.update({k: v for k, v in data.items() if k != "@context"}) else: # Generic dictionary - wrap in @graph jsonld["@graph"] = [data] @@ -412,13 +451,57 @@ class JSONExporter: # Add metadata and provenance if requested if include_metadata: - jsonld["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" - if include_provenance: - jsonld["semantica:exportedAt"] = utc_now_iso() - jsonld["semantica:format"] = "json-ld" + self._attach_document_metadata(jsonld, include_provenance) return jsonld + @staticmethod + def _attach_document_metadata( + jsonld: Dict[str, Any], include_provenance: bool + ) -> None: + """ + Attach the export's own metadata without naming the graph. + + A top-level ``@id`` alongside a top-level ``@graph`` is a *named graph*: + the members of ``@graph`` become quads named by that ``@id`` and leave + the default graph empty. ``rdflib.Graph.parse()`` keeps only the default + graph, so every statement in the export was discarded without an error + (2 of 21 statements survived a two-entity knowledge graph). When the + payload lives in ``@graph``, the document node goes in beside it as one + more node; otherwise it is the document itself. + + Args: + jsonld: Document being built, modified in place + include_provenance: Whether to record how and when it was exported + """ + # A caller may hand us a document that is deliberately a named graph. + # That name is theirs to keep, but our own statements must not end up + # inside it, where a default-graph reader would never see them. + payload_is_named_graph = "@id" in jsonld and "@graph" in jsonld + + document: Dict[str, Any] = {} + # Do not overwrite an identifier the payload already carries: the + # knowledge-graph conversion names its own document node. + if "@id" not in jsonld or payload_is_named_graph: + document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" + if include_provenance: + document["semantica:exportedAt"] = utc_now_iso() + document["semantica:format"] = "json-ld" + + if payload_is_named_graph: + named = {key: value for key, value in jsonld.items() if key != "@context"} + for key in [key for key in jsonld if key != "@context"]: + del jsonld[key] + jsonld["@graph"] = [named, document] + elif "@graph" in jsonld: + # @graph may be a single node object as well as an array. list() on + # a dictionary yields its keys, which would discard the node. + members = jsonld["@graph"] + members = list(members) if isinstance(members, list) else [members] + jsonld["@graph"] = members + [document] + else: + jsonld.update(document) + def _convert_kg_to_json(self, kg: Dict[str, Any], **options) -> Dict[str, Any]: """ Convert knowledge graph to JSON format. diff --git a/semantica/kg/graph_validator.py b/semantica/kg/graph_validator.py index 9d6a2cbb..a24fd16e 100644 --- a/semantica/kg/graph_validator.py +++ b/semantica/kg/graph_validator.py @@ -31,6 +31,7 @@ from dataclasses import dataclass, field from enum import Enum import networkx as nx +from ..utils.entity_ids import get_entity_id from ..utils.logging import get_logger class ValidationSeverity(Enum): @@ -147,29 +148,42 @@ class GraphValidator: # 2. Entity Validation entity_ids = set() for entity in entities: + eid = get_entity_id(entity) + # Check required fields missing = self.required_entity_fields - set(entity.keys()) + missing.discard("id") + if eid is None: + missing.add("id or entity_id") if missing: issues.append(ValidationIssue( code="MISSING_FIELD", message=f"Entity missing required fields: {missing}", severity=ValidationSeverity.ERROR, - element_id=entity.get("id", "unknown"), + element_id=eid or "unknown", element_type="entity" )) # Check ID uniqueness - eid = entity.get("id") if eid: - if eid in entity_ids: + try: + if eid in entity_ids: + issues.append(ValidationIssue( + code="DUPLICATE_ID", + message=f"Duplicate entity ID found: {eid}", + severity=ValidationSeverity.CRITICAL, + element_id=eid, + element_type="entity" + )) + entity_ids.add(eid) + except TypeError: issues.append(ValidationIssue( - code="DUPLICATE_ID", - message=f"Duplicate entity ID found: {eid}", - severity=ValidationSeverity.CRITICAL, - element_id=eid, + code="INVALID_ID", + message=f"Entity ID is not hashable: {eid}", + severity=ValidationSeverity.ERROR, + element_id=str(eid), element_type="entity" )) - entity_ids.add(eid) # Schema Check (if schema provided) if self.schema and "entity_types" in self.schema: @@ -183,18 +197,29 @@ class GraphValidator: element_type="entity" )) + def get_relationship_endpoint(rel, field, alias): + endpoint = rel.get(field) + if endpoint is None: + endpoint = rel.get(alias) + return endpoint + + def is_valid_id(node_id): + if node_id is None: + return False + try: + return node_id in entity_ids + except TypeError: + # Not hashable, so it can't be in the set of string IDs + return False + # 3. Relationship Validation for i, rel in enumerate(relationships): # Endpoints may use either the legacy ``source``/``target`` keys or # the canonical ``source_id``/``target_id`` keys emitted by # ``ContextGraph.to_kg_dict()``. Accept either variant so both # representations validate consistently. - src = rel.get("source") - if src is None: - src = rel.get("source_id") - tgt = rel.get("target") - if tgt is None: - tgt = rel.get("target_id") + src = get_relationship_endpoint(rel, "source", "source_id") + tgt = get_relationship_endpoint(rel, "target", "target_id") # Check required fields: ``type`` plus a resolvable source/target. missing = set() @@ -215,15 +240,6 @@ class GraphValidator: continue # Check Dangling Edges - def is_valid_id(node_id): - if node_id is None: - return False - try: - return node_id in entity_ids - except TypeError: - # Not hashable, so it can't be in the set of string IDs - return False - if not is_valid_id(src): issues.append(ValidationIssue( code="DANGLING_EDGE", @@ -259,7 +275,11 @@ class GraphValidator: try: nx_graph = nx.DiGraph() nx_graph.add_nodes_from(entity_ids) - nx_graph.add_edges_from([(r["source"], r["target"]) for r in relationships if r.get("source") in entity_ids and r.get("target") in entity_ids]) + for relationship in relationships: + src = get_relationship_endpoint(relationship, "source", "source_id") + tgt = get_relationship_endpoint(relationship, "target", "target_id") + if is_valid_id(src) and is_valid_id(tgt): + nx_graph.add_edge(src, tgt) # Check for cycles try: diff --git a/semantica/kg/temporal_query.py b/semantica/kg/temporal_query.py index f5bc227a..c335c95c 100644 --- a/semantica/kg/temporal_query.py +++ b/semantica/kg/temporal_query.py @@ -510,6 +510,9 @@ class TemporalGraphQuery: - "count": Number of relationships - "diversity": Number of unique relationship types - "stability": Relationship duration/stability measure + (mean valid-time duration in seconds across + relationships that have both ``valid_from`` and + ``valid_until`` set) **options: Additional analysis options (unused) Returns: @@ -582,14 +585,17 @@ class TemporalGraphQuery: result["diversity"] = len(rel_types) if "stability" in metrics: - # Calculate stability based on relationship duration + # Stability is the mean duration (in seconds) that relationships + # remain valid. Relationships without a bounded validity interval + # (missing/open ``valid_from`` or ``valid_until``) are skipped, and + # non-positive intervals are clamped to zero. durations = [] for rel in relationships: valid_from = self._parse_time(rel.get("valid_from")) valid_until = self._parse_time(rel.get("valid_until")) if valid_from and valid_until: - # Simplified duration calculation - durations.append(1) # Placeholder + duration_seconds = (valid_until - valid_from).total_seconds() + durations.append(max(0.0, duration_seconds)) result["stability"] = sum(durations) / len(durations) if durations else 0 return result diff --git a/semantica/normalize/number_normalizer.py b/semantica/normalize/number_normalizer.py index 29911cb9..0846465d 100644 --- a/semantica/normalize/number_normalizer.py +++ b/semantica/normalize/number_normalizer.py @@ -370,8 +370,12 @@ class UnitConverter: Raises: ValidationError: If units are incompatible or not in same category """ - from_unit = from_unit.lower() - to_unit = to_unit.lower() + # Normalize aliases before validating categories and looking up factors. + # The public API documents abbreviations such as ``kg`` and ``km``; + # validating those raw aliases against the canonical category lists + # incorrectly rejected otherwise supported conversions. + from_unit = self.normalize_unit(from_unit) + to_unit = self.normalize_unit(to_unit) # Validate units if not self.validate_units(from_unit, to_unit): @@ -401,8 +405,8 @@ class UnitConverter: Returns: bool: True if units are compatible (same category), False otherwise """ - from_unit = from_unit.lower() - to_unit = to_unit.lower() + from_unit = self.normalize_unit(from_unit) + to_unit = self.normalize_unit(to_unit) # Check if both units exist if ( @@ -498,6 +502,18 @@ class UnitConverter: "ml": "milliliter", "milliliter": "milliliter", "milliliters": "milliliter", + "ft": "foot", + "foot": "foot", + "feet": "foot", + "yd": "yard", + "yard": "yard", + "yards": "yard", + "mi": "mile", + "mile": "mile", + "miles": "mile", + "gal": "gallon", + "gallon": "gallon", + "gallons": "gallon", } return unit_map.get(unit_lower, unit_lower) diff --git a/semantica/reasoning/sparql_reasoner.py b/semantica/reasoning/sparql_reasoner.py index 58e0c351..16be2a9f 100644 --- a/semantica/reasoning/sparql_reasoner.py +++ b/semantica/reasoning/sparql_reasoner.py @@ -84,6 +84,9 @@ class SPARQLReasoner: self.triplet_store = self.config.get("triplet_store") self.enable_inference = self.config.get("enable_inference", True) + # Reserved for query caching once a triplet-store execution path + # lands. execute_query() raises NotImplementedError until then, so + # the cache cannot be populated through any public path yet. self.query_cache: Dict[str, Any] = {} def expand_query(self, query: str, **options) -> str: @@ -330,79 +333,32 @@ class SPARQLReasoner: """ Execute SPARQL query with reasoning. + Not implemented: no triplet-store execution path exists yet, so the + query is refused loudly instead of returning an empty result set + that callers would read as "no matches" (issue #1083). + Args: query: SPARQL query string **options: Additional options - Returns: - Query results + Raises: + NotImplementedError: always, until a triplet-store execution + path lands. """ - tracking_id = self.progress_tracker.start_tracking( - module="reasoning", - submodule="SPARQLReasoner", - message="Executing SPARQL query with reasoning", + raise NotImplementedError( + "SPARQLReasoner.execute_query() is not implemented: no " + "triplet-store execution path exists yet. Returning an empty " + "result set would be misread as 'no matches', so the query " + "is refused instead." ) - try: - # Check cache - self.progress_tracker.update_tracking( - tracking_id, message="Checking query cache..." - ) - if query in self.query_cache: - self.logger.debug("Returning cached query result") - self.progress_tracker.stop_tracking( - tracking_id, - status="completed", - message="Returned cached query result", - ) - return self.query_cache[query] - - # Expand query - self.progress_tracker.update_tracking( - tracking_id, message="Expanding query with inference rules..." - ) - expanded_query = self.expand_query(query, **options) - - # Execute query (if triplet store available) - self.progress_tracker.update_tracking( - tracking_id, message="Executing query..." - ) - if self.triplet_store: - # This would call the triplet store's query method - # For now, return empty result - result = SPARQLQueryResult(bindings=[], variables=[]) - else: - # Mock result for testing - result = SPARQLQueryResult(bindings=[], variables=[]) - - # Infer additional results - if self.enable_inference: - self.progress_tracker.update_tracking( - tracking_id, message="Inferring additional results..." - ) - result = self.infer_results(result, **options) - - # Cache result - self.progress_tracker.update_tracking( - tracking_id, message="Caching query result..." - ) - self.query_cache[query] = result - - self.progress_tracker.stop_tracking( - tracking_id, - status="completed", - message=f"Query executed: {len(result.bindings)} results", - ) - return result - - except Exception as e: - self.progress_tracker.stop_tracking( - tracking_id, status="failed", message=str(e) - ) - raise - def clear_cache(self) -> None: - """Clear query cache.""" + """Clear query cache. + + Reserved for when a triplet-store execution path lands: until then, + ``execute_query()`` raises ``NotImplementedError`` and nothing can + populate the cache. + """ self.query_cache.clear() def add_inference_rule(self, rule_definition: str, **options) -> Rule: diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.py index 05ece01f..48065855 100644 --- a/semantica/utils/helpers.py +++ b/semantica/utils/helpers.py @@ -680,6 +680,35 @@ _TRIPLET_KEYS = ("triplets",) # 'metadata' and 'count'. _CONTEXT_KEYS = ("metadata", "statistics", "count") +# Validation errors below interpolate caller-controlled keys. A pathological +# key (megabytes long) would otherwise size the exception string and, through +# the export wrappers that log the full exception, the log entry. The display +# keeps the offending key recognizable while bounding the message. +_MAX_KEY_DISPLAY = 64 + + +def _truncate_key(key: Any) -> str: + """Render a mapping key for an error message, bounded in length.""" + value = str(key) + if len(value) > _MAX_KEY_DISPLAY: + return value[:_MAX_KEY_DISPLAY] + "…" + return value + + +# Truncating each key bounds the per-key cost; capping the count of keys +# shown bounds the total, so a payload carrying many unknown keys cannot +# size the message (or the log entry that records it) either. +_MAX_KEYS_DISPLAY = 8 + + +def _truncate_key_list(keys: Iterable[Any]) -> str: + """Render keys for an error message, bounded in count and length.""" + rendered = [_truncate_key(key) for key in keys] + if len(rendered) <= _MAX_KEYS_DISPLAY: + return ", ".join(f"'{key}'" for key in rendered) + shown = ", ".join(f"'{key}'" for key in rendered[:_MAX_KEYS_DISPLAY]) + return f"{shown}, and {len(rendered) - _MAX_KEYS_DISPLAY} more" + def _require_recognized_keys( payload: Mapping, recognized_keys: Sequence[str], *, what: str @@ -702,7 +731,7 @@ def _require_recognized_keys( if not payload or any(key in payload for key in recognized_keys): return - supplied = ", ".join(f"'{key}'" for key in sorted(map(str, payload))) + supplied = _truncate_key_list(sorted(map(str, payload))) expected = ", ".join(f"'{key}'" for key in recognized_keys) raise ValidationError( f"{what} has no recognized key. Supplied: {supplied}. " @@ -753,7 +782,7 @@ def _require_nothing_dropped( if not dropped: return - named = ", ".join(f"'{key}'" for key in dropped) + named = _truncate_key_list(dropped) expected = ", ".join(f"'{key}'" for key in recognized_keys) raise ValidationError( f"{what} resolved to nothing, but {named} still holds records. " diff --git a/tests/deduplication/test_deduplication.py b/tests/deduplication/test_deduplication.py index dd351642..8dc7b7c2 100644 --- a/tests/deduplication/test_deduplication.py +++ b/tests/deduplication/test_deduplication.py @@ -12,6 +12,7 @@ from semantica.deduplication.cluster_builder import ClusterBuilder from semantica.deduplication.registry import MethodRegistry from semantica.deduplication.config import DeduplicationConfig from semantica.deduplication.methods import get_deduplication_method +from semantica.utils.types import Entity from semantica.utils.progress_tracker import ConsoleProgressDisplay class TestDeduplication(unittest.TestCase): @@ -87,6 +88,85 @@ class TestDeduplication(unittest.TestCase): # One group should have at least 2 entities (the Apple ones) apple_group = next((g for g in groups if len(g.entities) >= 2), None) self.assertIsNotNone(apple_group) + + def test_different_types_are_never_duplicates(self): + """Entities with different non-empty types must not merge (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Person 'Alice' and Organization 'Acme' must not be duplicate candidates", + ) + # GraphBuilder with merge_entities=True must keep both entities + from semantica.kg import GraphBuilder + graph = GraphBuilder(merge_entities=True).build( + {"entities": entities, "relationships": []} + ) + self.assertEqual(len(graph["entities"]), 2) + + def test_same_type_same_name_still_merges(self): + """Type guard must not break legitimate dedup of same-type entities.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Person", "name": "Alice", "text": "Alice"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Same-type same-name entities must still be detected as duplicates" + ) + + def test_untyped_same_name_still_merges(self): + """Entities with no type must retain previous behavior (merge on similarity).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "x1", "name": "Apple"}, + {"id": "x2", "name": "Apple"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Untyped same-name entities must still be detected as duplicates" + ) + + def test_entity_objects_different_types_not_duplicates(self): + """Entity objects expose their type via .type, not .label (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + Entity(id="e1", text="Alice", type="Person"), + Entity(id="e2", text="Acme", type="Organization"), + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Entity objects with different types must never be detected as duplicates", + ) + + def test_zero_threshold_still_excludes_type_mismatch(self): + """Type mismatch must be excluded structurally, not just by confidence 0.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Different-type candidates must be excluded even with confidence_threshold=0.0", + ) def test_entity_merger(self): """Test entity merging.""" diff --git a/tests/export/test_jsonld_default_graph.py b/tests/export/test_jsonld_default_graph.py new file mode 100644 index 00000000..e9f8dc09 --- /dev/null +++ b/tests/export/test_jsonld_default_graph.py @@ -0,0 +1,218 @@ +"""Every JSON-LD export must put its payload in the default graph. + +A JSON-LD document carrying a top-level ``@id`` *and* a top-level ``@graph`` is +a **named graph**: the contents of ``@graph`` are quads named by that ``@id``, +not triples in the default graph. ``rdflib.Graph.parse()`` — the ordinary way a +Python consumer loads RDF — keeps the default graph and discards the rest, +without an error. ``JSONExporter`` emitted exactly that shape: + +* ``_convert_to_jsonld`` wrote the payload into ``@graph`` and then stamped a + document ``@id`` beside it, so every list export and every generic-dict + export was named; +* ``export_knowledge_graph`` converted the graph to JSON-LD and handed the + finished document back to ``export()``, which converted it a *second* time. + The converted document no longer has ``entities``/``relationships`` keys, so + the second pass treated it as opaque and wrapped it in ``@graph`` — burying + a whole knowledge graph, entities, relationships and all, inside a named + graph whose name is a wall-clock timestamp. + +Measured on v0.6.6: a two-entity, one-relationship graph exported to JSON-LD +parsed as **2 triples** with ``Graph()`` and 21 quads with ``Dataset()``. The 19 +missing triples were the entire knowledge graph, and nothing reported a +problem. Semantica's own reader has the mirror of this bug (#1129), so the +export could not even be read back by Semantica. +""" + +import json + +import pytest +from rdflib import Dataset, Graph + +from semantica.export.json_exporter import JSONExporter + +KG = { + "entities": [ + {"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"}, + {"id": "https://example.org/e2", "text": "Jane Roe", "type": "PERSON"}, + ], + "relationships": [ + { + "source_id": "https://example.org/e1", + "target_id": "https://example.org/e2", + "type": "employs", + } + ], + "metadata": {"source_document": "contract.pdf"}, +} + +RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label" + +PAYLOADS = { + "knowledge_graph": KG, + "list": [ + {"@id": "https://example.org/a", RDFS_LABEL: "A"}, + {"@id": "https://example.org/b", RDFS_LABEL: "B"}, + ], + "generic_dict": {"@id": "https://example.org/x", RDFS_LABEL: "X"}, +} + + +def _write(payload, tmp_path, name="out.jsonld", **options): + path = tmp_path / name + JSONExporter().export(payload, path, format="json-ld", **options) + return path + + +def _counts(path): + """Triples a default-graph reader sees, and quads a quad reader sees.""" + graph = Graph() + graph.parse(str(path), format="json-ld") + dataset = Dataset() + dataset.parse(str(path), format="json-ld") + return len(graph), sum(1 for _ in dataset.quads((None, None, None, None))) + + +@pytest.mark.parametrize("name", sorted(PAYLOADS)) +def test_no_export_hides_its_payload_in_a_named_graph(name, tmp_path): + """A top-level @id beside a top-level @graph names the graph.""" + path = _write(PAYLOADS[name], tmp_path, f"{name}.jsonld") + document = json.loads(path.read_text()) + + assert not ("@id" in document and "@graph" in document), ( + f"{name}: @id + @graph at the top level makes a named graph, " + "which a default-graph reader discards in full" + ) + + +@pytest.mark.parametrize("name", sorted(PAYLOADS)) +def test_a_plain_graph_reader_loses_nothing(name, tmp_path): + """Graph() and Dataset() must agree: no triple may live outside the default graph.""" + path = _write(PAYLOADS[name], tmp_path, f"{name}.jsonld") + triples, quads = _counts(path) + + assert triples == quads, ( + f"{name}: Graph() read {triples} of {quads} statements; " + f"{quads - triples} were dropped silently" + ) + + +def test_exported_knowledge_graph_survives_a_default_graph_read(tmp_path): + """The entities and the relationship must be there after a plain parse.""" + path = tmp_path / "kg.jsonld" + JSONExporter().export_knowledge_graph(KG, path, format="json-ld") + + graph = Graph() + graph.parse(str(path), format="json-ld") + subjects = {str(s) for s in graph.subjects()} + objects = {str(o) for o in graph.objects()} + + assert "https://example.org/e1" in subjects + assert "https://example.org/e2" in subjects + assert "Acme Corp" in objects + assert "Jane Roe" in objects + assert "employs" in objects + + +def test_knowledge_graph_is_not_converted_twice(tmp_path): + """A nested @context is the signature of the document being re-converted.""" + path = tmp_path / "kg.jsonld" + JSONExporter().export_knowledge_graph(KG, path, format="json-ld") + document = json.loads(path.read_text()) + + nested = [ + node + for node in document.get("@graph", []) + if isinstance(node, dict) and "@context" in node + ] + assert nested == [], "the knowledge graph was converted, then converted again" + + +def test_document_provenance_still_reaches_the_default_graph(tmp_path): + """Keeping the payload readable must not cost the export its own metadata.""" + path = _write(PAYLOADS["list"], tmp_path) + + graph = Graph() + graph.parse(str(path), format="json-ld") + predicates = {str(p) for p in graph.predicates()} + + assert "https://semantica.dev/ns#exportedAt" in predicates + assert "https://semantica.dev/ns#format" in predicates + assert {"https://example.org/a", "https://example.org/b"} <= { + str(s) for s in graph.subjects() + } + assert {"A", "B"} <= {str(o) for o in graph.objects()} + + +# The document a caller hands to export() need not be one Semantica built, and +# the branch that recognises an already-converted document has to survive every +# shape JSON-LD allows. Each of the four cases below regressed when that branch +# was first written. + + +def test_a_url_valued_context_is_not_thrown_away(tmp_path): + """@context may be a URL or an array, not only an object.""" + payload = { + "@context": "https://schema.org/", + "@id": "https://example.org/thing", + "name": "Acme Corp", + } + path = _write(payload, tmp_path) + context = json.loads(path.read_text())["@context"] + + flattened = context if isinstance(context, list) else [context] + assert any(entry == "https://schema.org/" for entry in flattened), ( + "the caller's context was replaced by Semantica's defaults, " + "which silently changes how every term expands" + ) + + +def test_a_graph_given_as_one_node_object_survives(tmp_path): + """@graph may be a single node object; list() on it yields its keys.""" + payload = { + "@context": {"rdfs": "http://www.w3.org/2000/01/rdf-schema#"}, + "@graph": {"@id": "https://example.org/only", "rdfs:label": "Only"}, + } + path = _write(payload, tmp_path) + + graph = Graph() + graph.parse(str(path), format="json-ld") + assert "Only" in {str(o) for o in graph.objects()} + + +def test_a_caller_supplied_named_graph_keeps_our_provenance_readable(tmp_path): + """A deliberate named graph stays named, but must not swallow the export's own metadata.""" + payload = { + "@context": {"rdfs": "http://www.w3.org/2000/01/rdf-schema#"}, + "@id": "https://example.org/named", + "@graph": [{"@id": "https://example.org/n1", "rdfs:label": "N1"}], + } + path = _write(payload, tmp_path) + document = json.loads(path.read_text()) + + assert "@id" not in document or "@graph" not in document + + graph = Graph() + graph.parse(str(path), format="json-ld") + predicates = {str(p) for p in graph.predicates()} + assert "https://semantica.dev/ns#exportedAt" in predicates, ( + "the export's provenance was written inside the caller's named graph, " + "where a default-graph reader cannot see it" + ) + + dataset = Dataset() + dataset.parse(str(path), format="json-ld") + names = {str(c.identifier) for c in dataset.graphs()} + assert "https://example.org/named" in names, "the caller's graph lost its name" + + +def test_a_knowledge_graph_carrying_a_context_is_still_converted(tmp_path): + """entities/relationships must win over the already-JSON-LD branch.""" + payload = dict(KG, **{"@context": {"ex": "https://example.org/ns#"}}) + path = _write(payload, tmp_path) + document = json.loads(path.read_text()) + + assert "semantica:entities" in document, ( + "the knowledge graph skipped its own conversion, so entity ids, " + "endpoints, types and confidences were left as raw keys" + ) + assert "entities" not in document diff --git a/tests/kg/test_graph_validator.py b/tests/kg/test_graph_validator.py new file mode 100644 index 00000000..103e67b5 --- /dev/null +++ b/tests/kg/test_graph_validator.py @@ -0,0 +1,74 @@ +from semantica.kg.graph_builder import GraphBuilder +from semantica.kg.graph_validator import GraphValidator + + +def test_entity_id_aliases_are_validated_across_graph_structure(): + """Entity aliases should work for schema and structural validation.""" + graph = GraphBuilder( + merge_entities=True, + entity_resolution_strategy="exact", + resolve_conflicts=False, + ).build( + { + "entities": [ + {"entity_id": "alice:1", "name": "Alice", "type": "Person"}, + {"entity_id": "alice:2", "name": " Alice ", "type": "Person"}, + {"entity_id": "org:1", "name": "Acme", "type": "Organization"}, + ], + "relationships": [ + { + "source_id": "alice:2", + "target_id": "org:1", + "type": "WORKS_FOR", + } + ], + } + ) + + result = GraphValidator().validate(graph) + + assert result.is_valid + assert not any( + issue.code in {"MISSING_FIELD", "DANGLING_EDGE", "ORPHAN_NODES"} + for issue in result.issues + ) + + +def test_entity_id_and_relationship_aliases_validate_without_builder(): + """Validator endpoint fallbacks should be tested without normalization.""" + result = GraphValidator().validate( + { + "entities": [ + {"entity_id": "alice:1", "name": "Alice", "type": "Person"}, + {"entity_id": "org:1", "name": "Acme", "type": "Organization"}, + ], + "relationships": [ + { + "source_id": "alice:1", + "target_id": "org:1", + "type": "WORKS_FOR", + } + ], + } + ) + + assert result.is_valid + assert not any( + issue.code in {"MISSING_FIELD", "DANGLING_EDGE", "ORPHAN_NODES"} + for issue in result.issues + ) + + +def test_unhashable_entity_id_returns_validation_issue(): + """Invalid unhashable IDs should produce an issue instead of crashing.""" + result = GraphValidator().validate( + { + "entities": [ + {"entity_id": ["alice:1"], "name": "Alice", "type": "Person"} + ], + "relationships": [], + } + ) + + assert not result.is_valid + assert any(issue.code == "INVALID_ID" for issue in result.issues) diff --git a/tests/kg/test_kg.py b/tests/kg/test_kg.py index a983694d..4d02b1e4 100644 --- a/tests/kg/test_kg.py +++ b/tests/kg/test_kg.py @@ -407,6 +407,69 @@ class TestTemporalGraphQuery(unittest.TestCase): self.assertEqual(result["num_relationships"], 1) + def test_analyze_evolution_stability_is_mean_duration_seconds(self): + day = 86400.0 + graph = { + "relationships": [ + { + "source": "1", + "target": "2", + "type": "a", + "valid_from": "2024-01-01", + "valid_until": "2024-01-02", # 1 day + }, + { + "source": "2", + "target": "3", + "type": "b", + "valid_from": "2024-01-01", + "valid_until": "2024-01-04", # 3 days + }, + ] + } + + result = self.query_engine.analyze_evolution(graph, metrics=["stability"]) + + # Mean of 1-day and 3-day durations == 2 days in seconds. + self.assertAlmostEqual(result["stability"], 2 * day) + + def test_analyze_evolution_stability_skips_unbounded_intervals(self): + graph = { + "relationships": [ + { + "source": "1", + "target": "2", + "type": "bounded", + "valid_from": "2024-01-01", + "valid_until": "2024-01-02", + }, + { + "source": "2", + "target": "3", + "type": "open", + "valid_from": "2024-01-01", + "valid_until": TemporalBound.OPEN, + }, + { + "source": "3", + "target": "4", + "type": "no-start", + "valid_until": "2024-06-01", + }, + ] + } + + result = self.query_engine.analyze_evolution(graph, metrics=["stability"]) + + # Only the fully bounded relationship contributes (1 day). + self.assertAlmostEqual(result["stability"], 86400.0) + + def test_analyze_evolution_stability_empty_is_zero(self): + result = self.query_engine.analyze_evolution( + {"relationships": []}, metrics=["stability"] + ) + self.assertEqual(result["stability"], 0) + def test_query_at_time_legacy_transaction_axis_uses_valid_from_when_recorded_missing(self): graph = { "relationships": [ diff --git a/tests/normalize/test_number_normalizer.py b/tests/normalize/test_number_normalizer.py index a7cf359c..2e06260c 100644 --- a/tests/normalize/test_number_normalizer.py +++ b/tests/normalize/test_number_normalizer.py @@ -1,4 +1,6 @@ import unittest + +from semantica.utils.exceptions import ValidationError from semantica.normalize.number_normalizer import ( NumberNormalizer, UnitConverter, @@ -32,6 +34,19 @@ class TestUnitConverter(unittest.TestCase): # 1 kg = 1000 g self.assertEqual(self.converter.convert_units(1, "kg", "g"), 1000.0) + def test_convert_accepts_aliases_for_category_validation(self): + # Aliases are part of the documented API, not just parsing syntax. + self.assertEqual(self.converter.convert_units(1, "feet", "m"), 0.3048) + self.assertEqual(self.converter.convert_units(1, "gal", "liter"), 3.78541) + + def test_convert_rejects_mismatched_categories_even_for_aliases(self): + # Both units normalize to canonical names first, so the category + # check sees real categories and rejects cross-category conversions. + with self.assertRaises(ValidationError): + self.converter.convert_units(1, "kg", "ft") + with self.assertRaises(ValidationError): + self.converter.convert_units(1, "gal", "lb") + def test_normalize_unit(self): self.assertEqual(self.converter.normalize_unit("km"), "kilometer") self.assertEqual(self.converter.normalize_unit("kgs"), "kilogram") diff --git a/tests/reasoning/test_specialized_reasoners.py b/tests/reasoning/test_specialized_reasoners.py index 3a519171..51dbdc9a 100644 --- a/tests/reasoning/test_specialized_reasoners.py +++ b/tests/reasoning/test_specialized_reasoners.py @@ -30,6 +30,29 @@ class TestSpecializedReasoners(unittest.TestCase): binding_types = [b.get("x_type") for b in inferred.bindings] self.assertIn("Human", binding_types) + def test_execute_query_raises_not_implemented(self): + """Empty results must not pass as a valid answer (issue #1083). + + Both branches returned ``SPARQLQueryResult(bindings=[], variables=[])`` + -- with or without a triplet store -- so callers that trust an empty + result as "no matches" silently drew wrong conclusions. Until a real + execution path lands, refusing loudly is safer. + """ + reasoner = SPARQLReasoner() + with self.assertRaises(NotImplementedError): + reasoner.execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") + + def test_execute_query_with_triplet_store_raises_not_implemented(self): + reasoner = SPARQLReasoner(triplet_store=object()) + with self.assertRaises(NotImplementedError): + reasoner.execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") + + def test_execute_query_error_explains_why_the_query_is_refused(self): + reasoner = SPARQLReasoner() + with self.assertRaises(NotImplementedError) as ctx: + reasoner.execute_query("SELECT ?s WHERE { ?s ?p ?o }") + self.assertIn("not implemented", str(ctx.exception)) + def test_abductive_reasoner_generate_hypotheses(self): reasoner = AbductiveReasoner() reasoner.reasoner.add_rule("IF Disease(Flu) THEN Symptom(Fever)") diff --git a/tests/utils/test_normalize_graph_payload.py b/tests/utils/test_normalize_graph_payload.py index 02b53219..6f64aa22 100644 --- a/tests/utils/test_normalize_graph_payload.py +++ b/tests/utils/test_normalize_graph_payload.py @@ -474,6 +474,77 @@ class TestIsRecordBoundary(unittest.TestCase): ) +class TestKeyDisplayBounds(unittest.TestCase): + """Exception messages must not scale with caller-controlled keys (#1001). + + The validation boundary interpolates supplied keys straight into error + messages, so an extremely large key produced an equally large exception + string -- and, through the export wrappers that log the full exception, + an equally large log entry. The displayed key is truncated to a bounded + length while the supplied payload itself is never modified. + """ + + def test_unrecognized_key_display_is_bounded(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"x" * 1_000_000: [ENTITY]}) + + message = str(ctx.exception) + self.assertLess(len(message), 300) + self.assertIn("x" * 64 + "…", message) + # Truncating the supplied key must not cost the actionable part. + self.assertIn("no recognized key", message) + self.assertIn("entities", message) + + def test_dropped_record_key_display_is_bounded(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"entities": [], "y" * 1_000_000: [ENTITY]}) + + message = str(ctx.exception) + self.assertLess(len(message), 300) + self.assertIn("y" * 64 + "…", message) + self.assertIn("holds records", message) + + def test_short_keys_are_displayed_in_full(self): + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload({"short_key": [ENTITY]}) + + self.assertIn("'short_key'", str(ctx.exception)) + + def test_bounded_display_does_not_mutate_the_payload(self): + big_key = "z" * 1_000_000 + payload = {big_key: [ENTITY]} + + with self.assertRaises(ValidationError): + normalize_graph_payload(payload) + + self.assertEqual(list(payload), [big_key]) + self.assertEqual(payload[big_key], [ENTITY]) + + def test_many_unrecognized_keys_are_summarized(self): + """Per-key truncation does not bound the number of keys shown. + + A payload carrying many short unrecognized keys would still size the + message (and the log entry that records it), so the count of + displayed keys is bounded too. + """ + payload = {f"key_{i}": [ENTITY] for i in range(100)} + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload(payload) + + message = str(ctx.exception) + self.assertLess(len(message), 500) + self.assertIn("and 92 more", message) + + def test_many_dropped_record_keys_are_summarized(self): + payload = {"entities": [], **{f"data_{i}": [ENTITY] for i in range(100)}} + with self.assertRaises(ValidationError) as ctx: + normalize_graph_payload(payload) + + message = str(ctx.exception) + self.assertLess(len(message), 500) + self.assertIn("and 92 more", message) + + @dataclass class _DataclassNode: id: str