Merge remote-tracking branch 'origin/main' into codex/harden-markdown-import-symlinks

This commit is contained in:
Saurabh Meena
2026-08-22 19:38:20 +05:30
17 changed files with 804 additions and 126 deletions
+5
View File
@@ -108,6 +108,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- **The temporal-evolution `stability` metric was a hardcoded placeholder, not a duration**
- `TemporalGraphQuery.analyze_evolution()` documents `stability` as a "relationship duration/stability measure", but the implementation appended a constant `1` for every relationship with both `valid_from` and `valid_until` set (`durations.append(1) # Placeholder`). The reported stability was therefore always `1.0` when any bounded relationship existed and `0` otherwise — it never reflected how long relationships actually stayed valid, so it could not distinguish a graph of decade-long relationships from one of one-second relationships
- `stability` now computes the mean valid-time duration in seconds (`(valid_until - valid_from).total_seconds()`) across relationships that have both bounds set. Relationships with a missing or open `valid_from`/`valid_until` are skipped (their duration is unbounded), and non-positive intervals are clamped to `0`; an empty set still reports `0`
- New tests in `tests/kg/test_kg.py` assert the mean-duration result, the skipping of unbounded/half-open intervals, and the empty-graph zero case
- **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai
- `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it
- In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have
+1 -1
View File
@@ -9,7 +9,7 @@ RUN npm ci
COPY explorer/ ./
RUN mkdir -p /app/semantica && npm run build
FROM python:3.14-slim AS runtime
FROM python:3.13-slim AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
+6 -15
View File
@@ -269,7 +269,7 @@ print("Loaded {} facts from graph".format(count))
## Step 5 — SPARQL queries over enriched working memory
After forward chaining has derived new facts, `SPARQLReasoner` lets you query the enriched working memory using SPARQL triple-pattern matching with optional inference expansion:
After forward chaining has derived new facts, `SPARQLReasoner` prepares SPARQL queries over the enriched working memory with optional inference expansion:
```python
from semantica.reasoning import SPARQLReasoner
@@ -288,22 +288,13 @@ query = """
}
"""
# execute_query() runs: expansion → inference → deduplication
result = sparql.execute_query(query)
for binding in result.bindings:
print("Actor: {:15s} CVE: {}".format(
binding.get("actor", "?"),
binding.get("cve", "?"),
))
# metadata shows how many results came from inference vs ground facts
print("Original: {} Inferred: {}".format(
result.metadata.get("original_count", 0),
result.metadata.get("inferred_count", 0),
))
# expand_query() applies inference rules to the query text:
expanded = sparql.expand_query(query)
print(expanded)
```
`execute_query()` is not implemented yet: no triplet-store execution path exists, so it raises `NotImplementedError` rather than returning an empty result set that callers would misread as "no matches". Until execution lands, run the expanded query against your RDF store directly (for example with `rdflib`).
Inspect the expanded query before running it:
```python
+36 -8
View File
@@ -278,8 +278,12 @@ class DuplicateDetector:
for i, (entity1, entity2, score) in enumerate(similarities):
candidate = self._create_duplicate_candidate(entity1, entity2, score)
# Filter by confidence threshold
if candidate.confidence >= self.confidence_threshold:
# Filter by confidence threshold; type mismatches are excluded
# structurally so no threshold value can admit them.
if (
candidate.confidence >= self.confidence_threshold
and "type_mismatch" not in candidate.reasons
):
candidates.append(candidate)
remaining = total_similarities - (i + 1)
@@ -624,8 +628,12 @@ class DuplicateDetector:
new_entity, existing_entity, similarity.score
)
# Filter by confidence threshold
if candidate.confidence >= self.confidence_threshold:
# Filter by confidence threshold; type mismatches are
# excluded structurally regardless of the threshold.
if (
candidate.confidence >= self.confidence_threshold
and "type_mismatch" not in candidate.reasons
):
candidates.append(candidate)
processed += 1
@@ -723,7 +731,9 @@ class DuplicateDetector:
if key == "name":
return getattr(entity, "text", default)
if key == "type":
return getattr(entity, "label", default)
# Entity objects store the type on .type; extraction entities
# may expose .label. Missing .label never means "no type".
return getattr(entity, "type", default) or getattr(entity, "label", default)
if key == "properties":
# Check metadata for properties
metadata = getattr(entity, "metadata", {})
@@ -757,6 +767,25 @@ class DuplicateDetector:
reasons = []
confidence = similarity_score
# Check entity type mismatch first: two entities with different
# explicit types are not duplicates, whatever their similarity.
entity_type1 = self._get_entity_value(entity1, "type")
entity_type2 = self._get_entity_value(entity2, "type")
if entity_type1 and entity_type2 and entity_type1 != entity_type2:
return DuplicateCandidate(
entity1=entity1,
entity2=entity2,
similarity_score=similarity_score,
confidence=0.0,
reasons=["type_mismatch"],
metadata={
"name_match": False,
"common_properties": 0,
"type_match": False,
"type_mismatch": True,
},
)
# Check for exact name match (strong indicator)
name1 = str(self._get_entity_value(entity1, "name", "")).lower().strip()
name2 = str(self._get_entity_value(entity2, "name", "")).lower().strip()
@@ -779,9 +808,8 @@ class DuplicateDetector:
# Boost confidence for each matching property
confidence += 0.05 * prop_matches
# Check entity type match
entity_type1 = self._get_entity_value(entity1, "type")
entity_type2 = self._get_entity_value(entity2, "type")
# Check entity type match (only boosts when types are equal; mismatch
# is handled above)
if entity_type1 and entity_type2 and entity_type1 == entity_type2:
reasons.append("same_type")
confidence += 0.05
+87 -4
View File
@@ -34,6 +34,25 @@ from ..utils.progress_tracker import get_progress_tracker
from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri
def _is_jsonld_document(data: Dict[str, Any]) -> bool:
"""
Report whether a dictionary is already a JSON-LD document.
``export_knowledge_graph`` converts a knowledge graph to JSON-LD and then
hands the finished document to ``export()``, which converted it a second
time. The converted document no longer carries ``entities``/
``relationships`` keys, so the second pass treated it as an opaque value and
buried it inside ``@graph``.
Args:
data: Dictionary to test
Returns:
True when the dictionary declares a JSON-LD context
"""
return "@context" in data
class JSONExporter:
"""
JSON exporter for knowledge graphs and semantic data.
@@ -397,9 +416,29 @@ class JSONExporter:
# Convert data based on type
if isinstance(data, dict):
# A knowledge graph is converted even when it carries a context of
# its own: the specialized conversion is what mints entity ids and
# relationship endpoints, and skipping it leaves them raw keys.
if "entities" in data or "relationships" in data:
# Knowledge graph structure - use specialized conversion
jsonld.update(self._convert_kg_to_jsonld(data, **options))
elif _is_jsonld_document(data):
# Already JSON-LD: merge it rather than nesting it. Wrapping a
# converted document in @graph re-typed the payload as a named
# graph and doubled the @context, which is what happened when
# export_knowledge_graph handed its own output back to export().
context = data.get("@context")
if isinstance(context, dict):
jsonld["@context"].update(context)
elif context is not None:
# A context may also be a URL or an array of them, which
# cannot be merged key by key. Keeping both as an array
# preserves the caller's term expansion, which wins over
# ours, while still defining the semantica prefix. An
# explicit null is left alone: in an array it would reset
# the active context and take our own terms with it.
jsonld["@context"] = [jsonld["@context"], context]
jsonld.update({k: v for k, v in data.items() if k != "@context"})
else:
# Generic dictionary - wrap in @graph
jsonld["@graph"] = [data]
@@ -412,13 +451,57 @@ class JSONExporter:
# Add metadata and provenance if requested
if include_metadata:
jsonld["@id"] = f"https://semantica.dev/data/{utc_now_iso()}"
if include_provenance:
jsonld["semantica:exportedAt"] = utc_now_iso()
jsonld["semantica:format"] = "json-ld"
self._attach_document_metadata(jsonld, include_provenance)
return jsonld
@staticmethod
def _attach_document_metadata(
jsonld: Dict[str, Any], include_provenance: bool
) -> None:
"""
Attach the export's own metadata without naming the graph.
A top-level ``@id`` alongside a top-level ``@graph`` is a *named graph*:
the members of ``@graph`` become quads named by that ``@id`` and leave
the default graph empty. ``rdflib.Graph.parse()`` keeps only the default
graph, so every statement in the export was discarded without an error
(2 of 21 statements survived a two-entity knowledge graph). When the
payload lives in ``@graph``, the document node goes in beside it as one
more node; otherwise it is the document itself.
Args:
jsonld: Document being built, modified in place
include_provenance: Whether to record how and when it was exported
"""
# A caller may hand us a document that is deliberately a named graph.
# That name is theirs to keep, but our own statements must not end up
# inside it, where a default-graph reader would never see them.
payload_is_named_graph = "@id" in jsonld and "@graph" in jsonld
document: Dict[str, Any] = {}
# Do not overwrite an identifier the payload already carries: the
# knowledge-graph conversion names its own document node.
if "@id" not in jsonld or payload_is_named_graph:
document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}"
if include_provenance:
document["semantica:exportedAt"] = utc_now_iso()
document["semantica:format"] = "json-ld"
if payload_is_named_graph:
named = {key: value for key, value in jsonld.items() if key != "@context"}
for key in [key for key in jsonld if key != "@context"]:
del jsonld[key]
jsonld["@graph"] = [named, document]
elif "@graph" in jsonld:
# @graph may be a single node object as well as an array. list() on
# a dictionary yields its keys, which would discard the node.
members = jsonld["@graph"]
members = list(members) if isinstance(members, list) else [members]
jsonld["@graph"] = members + [document]
else:
jsonld.update(document)
def _convert_kg_to_json(self, kg: Dict[str, Any], **options) -> Dict[str, Any]:
"""
Convert knowledge graph to JSON format.
+44 -24
View File
@@ -31,6 +31,7 @@ from dataclasses import dataclass, field
from enum import Enum
import networkx as nx
from ..utils.entity_ids import get_entity_id
from ..utils.logging import get_logger
class ValidationSeverity(Enum):
@@ -147,29 +148,42 @@ class GraphValidator:
# 2. Entity Validation
entity_ids = set()
for entity in entities:
eid = get_entity_id(entity)
# Check required fields
missing = self.required_entity_fields - set(entity.keys())
missing.discard("id")
if eid is None:
missing.add("id or entity_id")
if missing:
issues.append(ValidationIssue(
code="MISSING_FIELD",
message=f"Entity missing required fields: {missing}",
severity=ValidationSeverity.ERROR,
element_id=entity.get("id", "unknown"),
element_id=eid or "unknown",
element_type="entity"
))
# Check ID uniqueness
eid = entity.get("id")
if eid:
if eid in entity_ids:
try:
if eid in entity_ids:
issues.append(ValidationIssue(
code="DUPLICATE_ID",
message=f"Duplicate entity ID found: {eid}",
severity=ValidationSeverity.CRITICAL,
element_id=eid,
element_type="entity"
))
entity_ids.add(eid)
except TypeError:
issues.append(ValidationIssue(
code="DUPLICATE_ID",
message=f"Duplicate entity ID found: {eid}",
severity=ValidationSeverity.CRITICAL,
element_id=eid,
code="INVALID_ID",
message=f"Entity ID is not hashable: {eid}",
severity=ValidationSeverity.ERROR,
element_id=str(eid),
element_type="entity"
))
entity_ids.add(eid)
# Schema Check (if schema provided)
if self.schema and "entity_types" in self.schema:
@@ -183,18 +197,29 @@ class GraphValidator:
element_type="entity"
))
def get_relationship_endpoint(rel, field, alias):
endpoint = rel.get(field)
if endpoint is None:
endpoint = rel.get(alias)
return endpoint
def is_valid_id(node_id):
if node_id is None:
return False
try:
return node_id in entity_ids
except TypeError:
# Not hashable, so it can't be in the set of string IDs
return False
# 3. Relationship Validation
for i, rel in enumerate(relationships):
# Endpoints may use either the legacy ``source``/``target`` keys or
# the canonical ``source_id``/``target_id`` keys emitted by
# ``ContextGraph.to_kg_dict()``. Accept either variant so both
# representations validate consistently.
src = rel.get("source")
if src is None:
src = rel.get("source_id")
tgt = rel.get("target")
if tgt is None:
tgt = rel.get("target_id")
src = get_relationship_endpoint(rel, "source", "source_id")
tgt = get_relationship_endpoint(rel, "target", "target_id")
# Check required fields: ``type`` plus a resolvable source/target.
missing = set()
@@ -215,15 +240,6 @@ class GraphValidator:
continue
# Check Dangling Edges
def is_valid_id(node_id):
if node_id is None:
return False
try:
return node_id in entity_ids
except TypeError:
# Not hashable, so it can't be in the set of string IDs
return False
if not is_valid_id(src):
issues.append(ValidationIssue(
code="DANGLING_EDGE",
@@ -259,7 +275,11 @@ class GraphValidator:
try:
nx_graph = nx.DiGraph()
nx_graph.add_nodes_from(entity_ids)
nx_graph.add_edges_from([(r["source"], r["target"]) for r in relationships if r.get("source") in entity_ids and r.get("target") in entity_ids])
for relationship in relationships:
src = get_relationship_endpoint(relationship, "source", "source_id")
tgt = get_relationship_endpoint(relationship, "target", "target_id")
if is_valid_id(src) and is_valid_id(tgt):
nx_graph.add_edge(src, tgt)
# Check for cycles
try:
+9 -3
View File
@@ -510,6 +510,9 @@ class TemporalGraphQuery:
- "count": Number of relationships
- "diversity": Number of unique relationship types
- "stability": Relationship duration/stability measure
(mean valid-time duration in seconds across
relationships that have both ``valid_from`` and
``valid_until`` set)
**options: Additional analysis options (unused)
Returns:
@@ -582,14 +585,17 @@ class TemporalGraphQuery:
result["diversity"] = len(rel_types)
if "stability" in metrics:
# Calculate stability based on relationship duration
# Stability is the mean duration (in seconds) that relationships
# remain valid. Relationships without a bounded validity interval
# (missing/open ``valid_from`` or ``valid_until``) are skipped, and
# non-positive intervals are clamped to zero.
durations = []
for rel in relationships:
valid_from = self._parse_time(rel.get("valid_from"))
valid_until = self._parse_time(rel.get("valid_until"))
if valid_from and valid_until:
# Simplified duration calculation
durations.append(1) # Placeholder
duration_seconds = (valid_until - valid_from).total_seconds()
durations.append(max(0.0, duration_seconds))
result["stability"] = sum(durations) / len(durations) if durations else 0
return result
+20 -4
View File
@@ -370,8 +370,12 @@ class UnitConverter:
Raises:
ValidationError: If units are incompatible or not in same category
"""
from_unit = from_unit.lower()
to_unit = to_unit.lower()
# Normalize aliases before validating categories and looking up factors.
# The public API documents abbreviations such as ``kg`` and ``km``;
# validating those raw aliases against the canonical category lists
# incorrectly rejected otherwise supported conversions.
from_unit = self.normalize_unit(from_unit)
to_unit = self.normalize_unit(to_unit)
# Validate units
if not self.validate_units(from_unit, to_unit):
@@ -401,8 +405,8 @@ class UnitConverter:
Returns:
bool: True if units are compatible (same category), False otherwise
"""
from_unit = from_unit.lower()
to_unit = to_unit.lower()
from_unit = self.normalize_unit(from_unit)
to_unit = self.normalize_unit(to_unit)
# Check if both units exist
if (
@@ -498,6 +502,18 @@ class UnitConverter:
"ml": "milliliter",
"milliliter": "milliliter",
"milliliters": "milliliter",
"ft": "foot",
"foot": "foot",
"feet": "foot",
"yd": "yard",
"yard": "yard",
"yards": "yard",
"mi": "mile",
"mile": "mile",
"miles": "mile",
"gal": "gallon",
"gallon": "gallon",
"gallons": "gallon",
}
return unit_map.get(unit_lower, unit_lower)
+21 -65
View File
@@ -84,6 +84,9 @@ class SPARQLReasoner:
self.triplet_store = self.config.get("triplet_store")
self.enable_inference = self.config.get("enable_inference", True)
# Reserved for query caching once a triplet-store execution path
# lands. execute_query() raises NotImplementedError until then, so
# the cache cannot be populated through any public path yet.
self.query_cache: Dict[str, Any] = {}
def expand_query(self, query: str, **options) -> str:
@@ -330,79 +333,32 @@ class SPARQLReasoner:
"""
Execute SPARQL query with reasoning.
Not implemented: no triplet-store execution path exists yet, so the
query is refused loudly instead of returning an empty result set
that callers would read as "no matches" (issue #1083).
Args:
query: SPARQL query string
**options: Additional options
Returns:
Query results
Raises:
NotImplementedError: always, until a triplet-store execution
path lands.
"""
tracking_id = self.progress_tracker.start_tracking(
module="reasoning",
submodule="SPARQLReasoner",
message="Executing SPARQL query with reasoning",
raise NotImplementedError(
"SPARQLReasoner.execute_query() is not implemented: no "
"triplet-store execution path exists yet. Returning an empty "
"result set would be misread as 'no matches', so the query "
"is refused instead."
)
try:
# Check cache
self.progress_tracker.update_tracking(
tracking_id, message="Checking query cache..."
)
if query in self.query_cache:
self.logger.debug("Returning cached query result")
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
message="Returned cached query result",
)
return self.query_cache[query]
# Expand query
self.progress_tracker.update_tracking(
tracking_id, message="Expanding query with inference rules..."
)
expanded_query = self.expand_query(query, **options)
# Execute query (if triplet store available)
self.progress_tracker.update_tracking(
tracking_id, message="Executing query..."
)
if self.triplet_store:
# This would call the triplet store's query method
# For now, return empty result
result = SPARQLQueryResult(bindings=[], variables=[])
else:
# Mock result for testing
result = SPARQLQueryResult(bindings=[], variables=[])
# Infer additional results
if self.enable_inference:
self.progress_tracker.update_tracking(
tracking_id, message="Inferring additional results..."
)
result = self.infer_results(result, **options)
# Cache result
self.progress_tracker.update_tracking(
tracking_id, message="Caching query result..."
)
self.query_cache[query] = result
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
message=f"Query executed: {len(result.bindings)} results",
)
return result
except Exception as e:
self.progress_tracker.stop_tracking(
tracking_id, status="failed", message=str(e)
)
raise
def clear_cache(self) -> None:
"""Clear query cache."""
"""Clear query cache.
Reserved for when a triplet-store execution path lands: until then,
``execute_query()`` raises ``NotImplementedError`` and nothing can
populate the cache.
"""
self.query_cache.clear()
def add_inference_rule(self, rule_definition: str, **options) -> Rule:
+31 -2
View File
@@ -680,6 +680,35 @@ _TRIPLET_KEYS = ("triplets",)
# 'metadata' and 'count'.
_CONTEXT_KEYS = ("metadata", "statistics", "count")
# Validation errors below interpolate caller-controlled keys. A pathological
# key (megabytes long) would otherwise size the exception string and, through
# the export wrappers that log the full exception, the log entry. The display
# keeps the offending key recognizable while bounding the message.
_MAX_KEY_DISPLAY = 64
def _truncate_key(key: Any) -> str:
"""Render a mapping key for an error message, bounded in length."""
value = str(key)
if len(value) > _MAX_KEY_DISPLAY:
return value[:_MAX_KEY_DISPLAY] + ""
return value
# Truncating each key bounds the per-key cost; capping the count of keys
# shown bounds the total, so a payload carrying many unknown keys cannot
# size the message (or the log entry that records it) either.
_MAX_KEYS_DISPLAY = 8
def _truncate_key_list(keys: Iterable[Any]) -> str:
"""Render keys for an error message, bounded in count and length."""
rendered = [_truncate_key(key) for key in keys]
if len(rendered) <= _MAX_KEYS_DISPLAY:
return ", ".join(f"'{key}'" for key in rendered)
shown = ", ".join(f"'{key}'" for key in rendered[:_MAX_KEYS_DISPLAY])
return f"{shown}, and {len(rendered) - _MAX_KEYS_DISPLAY} more"
def _require_recognized_keys(
payload: Mapping, recognized_keys: Sequence[str], *, what: str
@@ -702,7 +731,7 @@ def _require_recognized_keys(
if not payload or any(key in payload for key in recognized_keys):
return
supplied = ", ".join(f"'{key}'" for key in sorted(map(str, payload)))
supplied = _truncate_key_list(sorted(map(str, payload)))
expected = ", ".join(f"'{key}'" for key in recognized_keys)
raise ValidationError(
f"{what} has no recognized key. Supplied: {supplied}. "
@@ -753,7 +782,7 @@ def _require_nothing_dropped(
if not dropped:
return
named = ", ".join(f"'{key}'" for key in dropped)
named = _truncate_key_list(dropped)
expected = ", ".join(f"'{key}'" for key in recognized_keys)
raise ValidationError(
f"{what} resolved to nothing, but {named} still holds records. "
+80
View File
@@ -12,6 +12,7 @@ from semantica.deduplication.cluster_builder import ClusterBuilder
from semantica.deduplication.registry import MethodRegistry
from semantica.deduplication.config import DeduplicationConfig
from semantica.deduplication.methods import get_deduplication_method
from semantica.utils.types import Entity
from semantica.utils.progress_tracker import ConsoleProgressDisplay
class TestDeduplication(unittest.TestCase):
@@ -87,6 +88,85 @@ class TestDeduplication(unittest.TestCase):
# One group should have at least 2 entities (the Apple ones)
apple_group = next((g for g in groups if len(g.entities) >= 2), None)
self.assertIsNotNone(apple_group)
def test_different_types_are_never_duplicates(self):
"""Entities with different non-empty types must not merge (issue #1137)."""
detector = DuplicateDetector(
similarity_threshold=0.4, confidence_threshold=0.4
)
entities = [
{"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"},
{"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"},
]
duplicates = detector.detect_duplicates(entities)
self.assertEqual(
duplicates, [],
"Person 'Alice' and Organization 'Acme' must not be duplicate candidates",
)
# GraphBuilder with merge_entities=True must keep both entities
from semantica.kg import GraphBuilder
graph = GraphBuilder(merge_entities=True).build(
{"entities": entities, "relationships": []}
)
self.assertEqual(len(graph["entities"]), 2)
def test_same_type_same_name_still_merges(self):
"""Type guard must not break legitimate dedup of same-type entities."""
detector = DuplicateDetector(
similarity_threshold=0.4, confidence_threshold=0.4
)
entities = [
{"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"},
{"id": "e2", "type": "Person", "name": "Alice", "text": "Alice"},
]
duplicates = detector.detect_duplicates(entities)
self.assertTrue(
duplicates, "Same-type same-name entities must still be detected as duplicates"
)
def test_untyped_same_name_still_merges(self):
"""Entities with no type must retain previous behavior (merge on similarity)."""
detector = DuplicateDetector(
similarity_threshold=0.4, confidence_threshold=0.4
)
entities = [
{"id": "x1", "name": "Apple"},
{"id": "x2", "name": "Apple"},
]
duplicates = detector.detect_duplicates(entities)
self.assertTrue(
duplicates, "Untyped same-name entities must still be detected as duplicates"
)
def test_entity_objects_different_types_not_duplicates(self):
"""Entity objects expose their type via .type, not .label (issue #1137)."""
detector = DuplicateDetector(
similarity_threshold=0.4, confidence_threshold=0.0
)
entities = [
Entity(id="e1", text="Alice", type="Person"),
Entity(id="e2", text="Acme", type="Organization"),
]
duplicates = detector.detect_duplicates(entities)
self.assertEqual(
duplicates, [],
"Entity objects with different types must never be detected as duplicates",
)
def test_zero_threshold_still_excludes_type_mismatch(self):
"""Type mismatch must be excluded structurally, not just by confidence 0."""
detector = DuplicateDetector(
similarity_threshold=0.4, confidence_threshold=0.0
)
entities = [
{"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"},
{"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"},
]
duplicates = detector.detect_duplicates(entities)
self.assertEqual(
duplicates, [],
"Different-type candidates must be excluded even with confidence_threshold=0.0",
)
def test_entity_merger(self):
"""Test entity merging."""
+218
View File
@@ -0,0 +1,218 @@
"""Every JSON-LD export must put its payload in the default graph.
A JSON-LD document carrying a top-level ``@id`` *and* a top-level ``@graph`` is
a **named graph**: the contents of ``@graph`` are quads named by that ``@id``,
not triples in the default graph. ``rdflib.Graph.parse()`` the ordinary way a
Python consumer loads RDF keeps the default graph and discards the rest,
without an error. ``JSONExporter`` emitted exactly that shape:
* ``_convert_to_jsonld`` wrote the payload into ``@graph`` and then stamped a
document ``@id`` beside it, so every list export and every generic-dict
export was named;
* ``export_knowledge_graph`` converted the graph to JSON-LD and handed the
finished document back to ``export()``, which converted it a *second* time.
The converted document no longer has ``entities``/``relationships`` keys, so
the second pass treated it as opaque and wrapped it in ``@graph`` burying
a whole knowledge graph, entities, relationships and all, inside a named
graph whose name is a wall-clock timestamp.
Measured on v0.6.6: a two-entity, one-relationship graph exported to JSON-LD
parsed as **2 triples** with ``Graph()`` and 21 quads with ``Dataset()``. The 19
missing triples were the entire knowledge graph, and nothing reported a
problem. Semantica's own reader has the mirror of this bug (#1129), so the
export could not even be read back by Semantica.
"""
import json
import pytest
from rdflib import Dataset, Graph
from semantica.export.json_exporter import JSONExporter
KG = {
"entities": [
{"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"},
{"id": "https://example.org/e2", "text": "Jane Roe", "type": "PERSON"},
],
"relationships": [
{
"source_id": "https://example.org/e1",
"target_id": "https://example.org/e2",
"type": "employs",
}
],
"metadata": {"source_document": "contract.pdf"},
}
RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"
PAYLOADS = {
"knowledge_graph": KG,
"list": [
{"@id": "https://example.org/a", RDFS_LABEL: "A"},
{"@id": "https://example.org/b", RDFS_LABEL: "B"},
],
"generic_dict": {"@id": "https://example.org/x", RDFS_LABEL: "X"},
}
def _write(payload, tmp_path, name="out.jsonld", **options):
path = tmp_path / name
JSONExporter().export(payload, path, format="json-ld", **options)
return path
def _counts(path):
"""Triples a default-graph reader sees, and quads a quad reader sees."""
graph = Graph()
graph.parse(str(path), format="json-ld")
dataset = Dataset()
dataset.parse(str(path), format="json-ld")
return len(graph), sum(1 for _ in dataset.quads((None, None, None, None)))
@pytest.mark.parametrize("name", sorted(PAYLOADS))
def test_no_export_hides_its_payload_in_a_named_graph(name, tmp_path):
"""A top-level @id beside a top-level @graph names the graph."""
path = _write(PAYLOADS[name], tmp_path, f"{name}.jsonld")
document = json.loads(path.read_text())
assert not ("@id" in document and "@graph" in document), (
f"{name}: @id + @graph at the top level makes a named graph, "
"which a default-graph reader discards in full"
)
@pytest.mark.parametrize("name", sorted(PAYLOADS))
def test_a_plain_graph_reader_loses_nothing(name, tmp_path):
"""Graph() and Dataset() must agree: no triple may live outside the default graph."""
path = _write(PAYLOADS[name], tmp_path, f"{name}.jsonld")
triples, quads = _counts(path)
assert triples == quads, (
f"{name}: Graph() read {triples} of {quads} statements; "
f"{quads - triples} were dropped silently"
)
def test_exported_knowledge_graph_survives_a_default_graph_read(tmp_path):
"""The entities and the relationship must be there after a plain parse."""
path = tmp_path / "kg.jsonld"
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
graph = Graph()
graph.parse(str(path), format="json-ld")
subjects = {str(s) for s in graph.subjects()}
objects = {str(o) for o in graph.objects()}
assert "https://example.org/e1" in subjects
assert "https://example.org/e2" in subjects
assert "Acme Corp" in objects
assert "Jane Roe" in objects
assert "employs" in objects
def test_knowledge_graph_is_not_converted_twice(tmp_path):
"""A nested @context is the signature of the document being re-converted."""
path = tmp_path / "kg.jsonld"
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
document = json.loads(path.read_text())
nested = [
node
for node in document.get("@graph", [])
if isinstance(node, dict) and "@context" in node
]
assert nested == [], "the knowledge graph was converted, then converted again"
def test_document_provenance_still_reaches_the_default_graph(tmp_path):
"""Keeping the payload readable must not cost the export its own metadata."""
path = _write(PAYLOADS["list"], tmp_path)
graph = Graph()
graph.parse(str(path), format="json-ld")
predicates = {str(p) for p in graph.predicates()}
assert "https://semantica.dev/ns#exportedAt" in predicates
assert "https://semantica.dev/ns#format" in predicates
assert {"https://example.org/a", "https://example.org/b"} <= {
str(s) for s in graph.subjects()
}
assert {"A", "B"} <= {str(o) for o in graph.objects()}
# The document a caller hands to export() need not be one Semantica built, and
# the branch that recognises an already-converted document has to survive every
# shape JSON-LD allows. Each of the four cases below regressed when that branch
# was first written.
def test_a_url_valued_context_is_not_thrown_away(tmp_path):
"""@context may be a URL or an array, not only an object."""
payload = {
"@context": "https://schema.org/",
"@id": "https://example.org/thing",
"name": "Acme Corp",
}
path = _write(payload, tmp_path)
context = json.loads(path.read_text())["@context"]
flattened = context if isinstance(context, list) else [context]
assert any(entry == "https://schema.org/" for entry in flattened), (
"the caller's context was replaced by Semantica's defaults, "
"which silently changes how every term expands"
)
def test_a_graph_given_as_one_node_object_survives(tmp_path):
"""@graph may be a single node object; list() on it yields its keys."""
payload = {
"@context": {"rdfs": "http://www.w3.org/2000/01/rdf-schema#"},
"@graph": {"@id": "https://example.org/only", "rdfs:label": "Only"},
}
path = _write(payload, tmp_path)
graph = Graph()
graph.parse(str(path), format="json-ld")
assert "Only" in {str(o) for o in graph.objects()}
def test_a_caller_supplied_named_graph_keeps_our_provenance_readable(tmp_path):
"""A deliberate named graph stays named, but must not swallow the export's own metadata."""
payload = {
"@context": {"rdfs": "http://www.w3.org/2000/01/rdf-schema#"},
"@id": "https://example.org/named",
"@graph": [{"@id": "https://example.org/n1", "rdfs:label": "N1"}],
}
path = _write(payload, tmp_path)
document = json.loads(path.read_text())
assert "@id" not in document or "@graph" not in document
graph = Graph()
graph.parse(str(path), format="json-ld")
predicates = {str(p) for p in graph.predicates()}
assert "https://semantica.dev/ns#exportedAt" in predicates, (
"the export's provenance was written inside the caller's named graph, "
"where a default-graph reader cannot see it"
)
dataset = Dataset()
dataset.parse(str(path), format="json-ld")
names = {str(c.identifier) for c in dataset.graphs()}
assert "https://example.org/named" in names, "the caller's graph lost its name"
def test_a_knowledge_graph_carrying_a_context_is_still_converted(tmp_path):
"""entities/relationships must win over the already-JSON-LD branch."""
payload = dict(KG, **{"@context": {"ex": "https://example.org/ns#"}})
path = _write(payload, tmp_path)
document = json.loads(path.read_text())
assert "semantica:entities" in document, (
"the knowledge graph skipped its own conversion, so entity ids, "
"endpoints, types and confidences were left as raw keys"
)
assert "entities" not in document
+74
View File
@@ -0,0 +1,74 @@
from semantica.kg.graph_builder import GraphBuilder
from semantica.kg.graph_validator import GraphValidator
def test_entity_id_aliases_are_validated_across_graph_structure():
"""Entity aliases should work for schema and structural validation."""
graph = GraphBuilder(
merge_entities=True,
entity_resolution_strategy="exact",
resolve_conflicts=False,
).build(
{
"entities": [
{"entity_id": "alice:1", "name": "Alice", "type": "Person"},
{"entity_id": "alice:2", "name": " Alice ", "type": "Person"},
{"entity_id": "org:1", "name": "Acme", "type": "Organization"},
],
"relationships": [
{
"source_id": "alice:2",
"target_id": "org:1",
"type": "WORKS_FOR",
}
],
}
)
result = GraphValidator().validate(graph)
assert result.is_valid
assert not any(
issue.code in {"MISSING_FIELD", "DANGLING_EDGE", "ORPHAN_NODES"}
for issue in result.issues
)
def test_entity_id_and_relationship_aliases_validate_without_builder():
"""Validator endpoint fallbacks should be tested without normalization."""
result = GraphValidator().validate(
{
"entities": [
{"entity_id": "alice:1", "name": "Alice", "type": "Person"},
{"entity_id": "org:1", "name": "Acme", "type": "Organization"},
],
"relationships": [
{
"source_id": "alice:1",
"target_id": "org:1",
"type": "WORKS_FOR",
}
],
}
)
assert result.is_valid
assert not any(
issue.code in {"MISSING_FIELD", "DANGLING_EDGE", "ORPHAN_NODES"}
for issue in result.issues
)
def test_unhashable_entity_id_returns_validation_issue():
"""Invalid unhashable IDs should produce an issue instead of crashing."""
result = GraphValidator().validate(
{
"entities": [
{"entity_id": ["alice:1"], "name": "Alice", "type": "Person"}
],
"relationships": [],
}
)
assert not result.is_valid
assert any(issue.code == "INVALID_ID" for issue in result.issues)
+63
View File
@@ -407,6 +407,69 @@ class TestTemporalGraphQuery(unittest.TestCase):
self.assertEqual(result["num_relationships"], 1)
def test_analyze_evolution_stability_is_mean_duration_seconds(self):
day = 86400.0
graph = {
"relationships": [
{
"source": "1",
"target": "2",
"type": "a",
"valid_from": "2024-01-01",
"valid_until": "2024-01-02", # 1 day
},
{
"source": "2",
"target": "3",
"type": "b",
"valid_from": "2024-01-01",
"valid_until": "2024-01-04", # 3 days
},
]
}
result = self.query_engine.analyze_evolution(graph, metrics=["stability"])
# Mean of 1-day and 3-day durations == 2 days in seconds.
self.assertAlmostEqual(result["stability"], 2 * day)
def test_analyze_evolution_stability_skips_unbounded_intervals(self):
graph = {
"relationships": [
{
"source": "1",
"target": "2",
"type": "bounded",
"valid_from": "2024-01-01",
"valid_until": "2024-01-02",
},
{
"source": "2",
"target": "3",
"type": "open",
"valid_from": "2024-01-01",
"valid_until": TemporalBound.OPEN,
},
{
"source": "3",
"target": "4",
"type": "no-start",
"valid_until": "2024-06-01",
},
]
}
result = self.query_engine.analyze_evolution(graph, metrics=["stability"])
# Only the fully bounded relationship contributes (1 day).
self.assertAlmostEqual(result["stability"], 86400.0)
def test_analyze_evolution_stability_empty_is_zero(self):
result = self.query_engine.analyze_evolution(
{"relationships": []}, metrics=["stability"]
)
self.assertEqual(result["stability"], 0)
def test_query_at_time_legacy_transaction_axis_uses_valid_from_when_recorded_missing(self):
graph = {
"relationships": [
+15
View File
@@ -1,4 +1,6 @@
import unittest
from semantica.utils.exceptions import ValidationError
from semantica.normalize.number_normalizer import (
NumberNormalizer,
UnitConverter,
@@ -32,6 +34,19 @@ class TestUnitConverter(unittest.TestCase):
# 1 kg = 1000 g
self.assertEqual(self.converter.convert_units(1, "kg", "g"), 1000.0)
def test_convert_accepts_aliases_for_category_validation(self):
# Aliases are part of the documented API, not just parsing syntax.
self.assertEqual(self.converter.convert_units(1, "feet", "m"), 0.3048)
self.assertEqual(self.converter.convert_units(1, "gal", "liter"), 3.78541)
def test_convert_rejects_mismatched_categories_even_for_aliases(self):
# Both units normalize to canonical names first, so the category
# check sees real categories and rejects cross-category conversions.
with self.assertRaises(ValidationError):
self.converter.convert_units(1, "kg", "ft")
with self.assertRaises(ValidationError):
self.converter.convert_units(1, "gal", "lb")
def test_normalize_unit(self):
self.assertEqual(self.converter.normalize_unit("km"), "kilometer")
self.assertEqual(self.converter.normalize_unit("kgs"), "kilogram")
@@ -30,6 +30,29 @@ class TestSpecializedReasoners(unittest.TestCase):
binding_types = [b.get("x_type") for b in inferred.bindings]
self.assertIn("Human", binding_types)
def test_execute_query_raises_not_implemented(self):
"""Empty results must not pass as a valid answer (issue #1083).
Both branches returned ``SPARQLQueryResult(bindings=[], variables=[])``
-- with or without a triplet store -- so callers that trust an empty
result as "no matches" silently drew wrong conclusions. Until a real
execution path lands, refusing loudly is safer.
"""
reasoner = SPARQLReasoner()
with self.assertRaises(NotImplementedError):
reasoner.execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }")
def test_execute_query_with_triplet_store_raises_not_implemented(self):
reasoner = SPARQLReasoner(triplet_store=object())
with self.assertRaises(NotImplementedError):
reasoner.execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }")
def test_execute_query_error_explains_why_the_query_is_refused(self):
reasoner = SPARQLReasoner()
with self.assertRaises(NotImplementedError) as ctx:
reasoner.execute_query("SELECT ?s WHERE { ?s ?p ?o }")
self.assertIn("not implemented", str(ctx.exception))
def test_abductive_reasoner_generate_hypotheses(self):
reasoner = AbductiveReasoner()
reasoner.reasoner.add_rule("IF Disease(Flu) THEN Symptom(Fever)")
@@ -474,6 +474,77 @@ class TestIsRecordBoundary(unittest.TestCase):
)
class TestKeyDisplayBounds(unittest.TestCase):
"""Exception messages must not scale with caller-controlled keys (#1001).
The validation boundary interpolates supplied keys straight into error
messages, so an extremely large key produced an equally large exception
string -- and, through the export wrappers that log the full exception,
an equally large log entry. The displayed key is truncated to a bounded
length while the supplied payload itself is never modified.
"""
def test_unrecognized_key_display_is_bounded(self):
with self.assertRaises(ValidationError) as ctx:
normalize_graph_payload({"x" * 1_000_000: [ENTITY]})
message = str(ctx.exception)
self.assertLess(len(message), 300)
self.assertIn("x" * 64 + "", message)
# Truncating the supplied key must not cost the actionable part.
self.assertIn("no recognized key", message)
self.assertIn("entities", message)
def test_dropped_record_key_display_is_bounded(self):
with self.assertRaises(ValidationError) as ctx:
normalize_graph_payload({"entities": [], "y" * 1_000_000: [ENTITY]})
message = str(ctx.exception)
self.assertLess(len(message), 300)
self.assertIn("y" * 64 + "", message)
self.assertIn("holds records", message)
def test_short_keys_are_displayed_in_full(self):
with self.assertRaises(ValidationError) as ctx:
normalize_graph_payload({"short_key": [ENTITY]})
self.assertIn("'short_key'", str(ctx.exception))
def test_bounded_display_does_not_mutate_the_payload(self):
big_key = "z" * 1_000_000
payload = {big_key: [ENTITY]}
with self.assertRaises(ValidationError):
normalize_graph_payload(payload)
self.assertEqual(list(payload), [big_key])
self.assertEqual(payload[big_key], [ENTITY])
def test_many_unrecognized_keys_are_summarized(self):
"""Per-key truncation does not bound the number of keys shown.
A payload carrying many short unrecognized keys would still size the
message (and the log entry that records it), so the count of
displayed keys is bounded too.
"""
payload = {f"key_{i}": [ENTITY] for i in range(100)}
with self.assertRaises(ValidationError) as ctx:
normalize_graph_payload(payload)
message = str(ctx.exception)
self.assertLess(len(message), 500)
self.assertIn("and 92 more", message)
def test_many_dropped_record_keys_are_summarized(self):
payload = {"entities": [], **{f"data_{i}": [ENTITY] for i in range(100)}}
with self.assertRaises(ValidationError) as ctx:
normalize_graph_payload(payload)
message = str(ctx.exception)
self.assertLess(len(message), 500)
self.assertIn("and 92 more", message)
@dataclass
class _DataclassNode:
id: str