mengzhx/semantica
1
0
Fork 0
mirror of https://github.com/semantica-agi/semantica.git synced 2026-08-29 04:26:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge upstream/main into metadata-passthrough

Browse Source 
#1123 through #1127 landed while this was open, and #1125 rewrote the same
four entity loops this branch extends. Confidence is now normalised through
normalize_confidence, which returns None for a value that has no xsd:decimal
form, so the clause can be absent.

Resolved by folding that into the clause list this branch already builds:
the Turtle path assembles its predicate-object clauses and then terminates
the last one, which is what makes a variable-length list work at all, and
an omitted confidence is simply one clause fewer. RDF/XML and JSON-LD take
the upstream conditional as written, with the metadata call after it.
This commit is contained in:
Fabio Rovai
2026-08-21 14:40:54 +01:00
parent eb7427d12c 46451ae2e1
commit d06434ae31
16 changed files with 1750 additions and 393 deletions
Download Patch File Download Diff File Expand all files Collapse all files
semantica/embeddings/methods.py
+25 -24
View File
@@ -80,6 +80,7 @@ import numpy as np
from ..utils.exceptions import ConfigurationError, ProcessingError
from ..utils.logging import get_logger
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
from .config import embeddings_config
from .embedding_generator import EmbeddingGenerator
from .pooling_strategies import PoolingStrategyFactory
@@ -119,12 +120,12 @@ def generate_embeddings(
# Check for custom method in registry
custom_method = method_registry.get("generation", method)
if custom_method:
try:
return custom_method(data, data_type=data_type, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, data_type=data_type, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
if method == "default":
@@ -167,12 +168,12 @@ def embed_text(
# Check for custom method in registry
custom_method = method_registry.get("text", method)
if custom_method:
try:
return custom_method(text, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -227,12 +228,12 @@ def calculate_similarity(
# Check for custom method in registry
custom_method = method_registry.get("similarity", method)
if custom_method:
try:
return custom_method(embedding1, embedding2, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, embedding1, embedding2, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
generator = EmbeddingGenerator(**kwargs)
@@ -274,12 +275,12 @@ def pool_embeddings(
# Check for custom method in registry
custom_method = method_registry.get("pooling", method)
if custom_method:
try:
return custom_method(embeddings, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, embeddings, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
strategy = PoolingStrategyFactory.create(method, **kwargs)
semantica/export/methods.py
+79 -78
View File
@@ -164,6 +164,7 @@ from typing import Any, Callable, Dict, List, Optional, Union
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
from .arango_aql_exporter import ArangoAQLExporter
from .arrow_exporter import ArrowExporter
from .config import export_config
@@ -221,12 +222,12 @@ def export_rdf(
# Check for custom method in registry
custom_method = method_registry.get("rdf", method)
if custom_method and custom_method is not export_rdf:
try:
return custom_method(data, file_path, format=format, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -270,12 +271,12 @@ def export_json(
# Check for custom method in registry
custom_method = method_registry.get("json", method)
if custom_method and custom_method is not export_json:
try:
return custom_method(data, file_path, format=format, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -316,12 +317,12 @@ def export_csv(
# Check for custom method in registry
custom_method = method_registry.get("csv", method)
if custom_method and custom_method is not export_csv:
try:
return custom_method(data, file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -361,12 +362,12 @@ def export_arrow(
# Check for custom method in registry
custom_method = method_registry.get("arrow", method)
if custom_method and custom_method is not export_arrow:
try:
return custom_method(data, file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -421,12 +422,12 @@ def export_parquet(
# Check for custom method in registry
custom_method = method_registry.get("parquet", method)
if custom_method and custom_method is not export_parquet:
try:
return custom_method(data, file_path, compression=compression, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, file_path, compression=compression, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -472,12 +473,12 @@ def export_graph(
# Check for custom method in registry
custom_method = method_registry.get("graph", method)
if custom_method and custom_method is not export_graph:
try:
return custom_method(graph_data, file_path, format=format, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, graph_data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -538,12 +539,12 @@ def export_yaml(
# Check for custom method in registry
custom_method = method_registry.get("yaml", method)
if custom_method and custom_method is not export_yaml:
try:
return custom_method(data, file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -595,12 +596,12 @@ def export_owl(
# Check for custom method in registry
custom_method = method_registry.get("owl", method)
if custom_method and custom_method is not export_owl:
try:
return custom_method(ontology, file_path, format=format, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, ontology, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -647,12 +648,12 @@ def export_vector(
# Check for custom method in registry
custom_method = method_registry.get("vector", method)
if custom_method and custom_method is not export_vector:
try:
return custom_method(vectors, file_path, format=format, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, vectors, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -695,12 +696,12 @@ def export_lpg(
# Check for custom method in registry
custom_method = method_registry.get("lpg", method)
if custom_method and custom_method is not export_lpg:
try:
return custom_method(knowledge_graph, file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, knowledge_graph, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -738,12 +739,12 @@ def export_neo4j_csv(
"""
custom_method = method_registry.get("neo4j_csv", method)
if custom_method and custom_method is not export_neo4j_csv:
try:
return custom_method(knowledge_graph, output_dir, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, knowledge_graph, output_dir, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = export_config.get_method_config("neo4j_csv")
@@ -808,12 +809,12 @@ def export_arango(
# Check for custom method in registry
custom_method = method_registry.get("arango", method)
if custom_method and custom_method is not export_arango:
try:
return custom_method(knowledge_graph, file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, knowledge_graph, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -859,12 +860,12 @@ def generate_report(
# Check for custom method in registry
custom_method = method_registry.get("report", method)
if custom_method and custom_method is not generate_report:
try:
return custom_method(data, file_path, format=format, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
semantica/export/rdf_exporter.py
+166 -11
View File
@@ -30,6 +30,7 @@ License: MIT
"""
from pathlib import Path
from decimal import Decimal, InvalidOperation
from typing import Any, Dict, List, Optional, Set, Union
from ..utils.exceptions import ProcessingError, ValidationError
@@ -49,6 +50,75 @@ DEFAULT_ENTITY_TYPE = f"{SEMANTICA_NS}Entity"
#: Written when a relationship carries no type of its own. Same reasoning.
DEFAULT_RELATION_TYPE = f"{SEMANTICA_NS}related_to"
#: The one datatype every serializer writes confidence in.
#:
#: The four paths used to disagree: Turtle wrote the value bare, which the
#: Turtle grammar reads as xsd:decimal, N-Triples typed it xsd:float, RDF/XML
#: emitted a plain literal with no datatype, and JSON-LD emitted a native
#: number, which becomes xsd:double. Those are four distinct RDF terms for one
#: value (issue #1100).
#:
#: xsd:decimal is the choice because it is what the Turtle path already
#: produced, so the most used output is unchanged, and because it is exact:
#: xsd:float is 32 bit binary, and cannot represent 0.9 or 0.95 at all.
CONFIDENCE_DATATYPE = "http://www.w3.org/2001/XMLSchema#decimal"
#: Largest power of ten a confidence may carry. xsd:decimal has no exponent
#: notation, so a value has to be written out in full, and a compact literal
#: such as "1e100000000" would expand to a hundred million digits.
MAX_CONFIDENCE_EXPONENT = 100
def normalize_confidence(value: Any) -> Optional[str]:
"""
Return the canonical xsd:decimal lexical form of a confidence value.
Returns None when the value cannot be a decimal, so callers omit the triple
rather than writing something the vocabulary contradicts. The Turtle path
used to interpolate the raw value, so a confidence of "high" produced
`semantica:confidence high .` and made the whole document unparseable
(issue #1102).
Booleans are rejected. `bool` is a subclass of `int` in Python, so True
would otherwise silently become a confidence of 1.
"""
if value is None or isinstance(value, bool):
return None
if isinstance(value, str):
value = value.strip()
if not value:
return None
if not isinstance(value, (int, float, str, Decimal)):
return None
try:
decimal_value = Decimal(str(value))
except (InvalidOperation, ValueError, TypeError):
return None
# NaN and the infinities are Decimal values with no xsd:decimal form.
if not decimal_value.is_finite():
return None
# xsd:decimal has no exponent notation, so writing one means expanding it.
# "1e100000000" is eleven characters that expand to a hundred million, and
# the export path continues past validation errors, so a single malformed
# field could exhaust memory. Nothing near this magnitude is a confidence.
if not -MAX_CONFIDENCE_EXPONENT <= decimal_value.adjusted() <= MAX_CONFIDENCE_EXPONENT:
return None
# `str(Decimal("0.00001"))` gives "0.00001", but a float that has already
# been through repr can arrive as "1e-05", which xsd:decimal does not allow.
formatted = format(decimal_value, "f")
if "." in formatted:
formatted = formatted.rstrip("0").rstrip(".") or "0"
# Decimal keeps the sign of zero, so 0.0 and -0.0 would serialise as two
# distinct RDF terms and defeat the point of a canonical form.
if formatted.lstrip("-").strip("0.") == "":
formatted = "0"
return formatted
def mint_entity_iri(text: str) -> str:
"""Mint a stable IRI for an entity that arrived without an id.
@@ -578,13 +648,21 @@ class RDFSerializer:
entity_type = entity.get("type", DEFAULT_ENTITY_TYPE)
text = entity.get("text") or entity.get("label", "")
confidence = entity.get("confidence", 1.0)
confidence = normalize_confidence(entity.get("confidence", 1.0))
clauses = [
f"a <{entity_type}>",
f'semantica:text "{text}"',
f"semantica:confidence {confidence}",
]
if confidence is None:
self.logger.warning(
f"Entity {entity_id} has a confidence that is not a number "
f"({entity.get('confidence')!r}), so no confidence is written"
)
else:
clauses.append(
f'semantica:confidence "{confidence}"^^<{CONFIDENCE_DATATYPE}>'
)
clauses.extend(
_turtle_metadata_clauses(
_metadata_statements(
@@ -611,6 +689,17 @@ class RDFSerializer:
if include_temporal:
owl_lines = self._owl_time_triples_for_rel(rel, idx, time_axis)
if owl_lines:
# The interval hangs off the relationship's own IRI, and a
# relationship written as a single triple has no such node
# in the graph. Without this the timestamps are well formed
# and unreachable: no query can get from the edge to its
# validity interval (#1106). The shape matches the JSON-LD
# export, and every term is declared in the vocabulary.
lines.extend(
self._reified_relationship_triples(
rel, idx, source_id, target_id, rel_type
)
)
lines.extend(owl_lines)
# Graph-level metadata needs a subject, and this serializer has never
@@ -639,6 +728,44 @@ class RDFSerializer:
return "\n".join(lines)
def _reified_relationship_triples(
self,
rel: Dict[str, Any],
idx: int,
source_id: str,
target_id: str,
rel_type: str,
) -> List[str]:
"""
Emit the reified relationship node that OWL-Time triples hang off.
The direct triple stays. This adds a subject the interval can attach
to, using the same sem:Relationship shape the JSON-LD export already
writes, so the two serializations describe relationships the same way.
"""
rel_id = rel.get("id") or mint_relationship_iri(idx, source_id or "", target_id or "")
# The full predicate, not its local name. Truncating to the fragment
# made https://a.example/ns#employs and https://b.example/ns#employs the
# same literal, so the temporal node no longer said which predicate it
# described, and it disagreed with the direct triple beside it.
escaped = (
str(rel_type)
.replace("\\", "\\\\")
.replace('"', '\\"')
.replace("\n", "\\n")
.replace("\r", "\\r")
)
predicates = [f"a semantica:Relationship"]
if source_id:
predicates.append(f"semantica:source <{source_id}>")
if target_id:
predicates.append(f"semantica:target <{target_id}>")
predicates.append(f'semantica:type "{escaped}"')
return ["", f"<{rel_id}> " + " ;\n ".join(predicates) + " ."]
def _owl_time_triples_for_rel(
self, rel: Dict[str, Any], idx: int, time_axis: str
) -> List[str]:
@@ -751,15 +878,22 @@ class RDFSerializer:
entity_type = entity.get("type", DEFAULT_ENTITY_TYPE)
text = entity.get("text") or entity.get("label", "")
confidence = entity.get("confidence", 1.0)
confidence = normalize_confidence(entity.get("confidence", 1.0))
# RDF/XML syntax: rdf:Description with rdf:about
lines.append(f' <rdf:Description rdf:about="{entity_id}">')
lines.append(f' <rdf:type rdf:resource="{entity_type}"/>')
lines.append(f" <semantica:text>{text}</semantica:text>")
lines.append(
f" <semantica:confidence>{confidence}</semantica:confidence>"
)
if confidence is None:
self.logger.warning(
f"Entity {entity_id} has a confidence that is not a number "
f"({entity.get('confidence')!r}), so no confidence is written"
)
else:
lines.append(
f' <semantica:confidence rdf:datatype="{CONFIDENCE_DATATYPE}">'
f"{confidence}</semantica:confidence>"
)
lines.extend(
_rdfxml_metadata_lines(
_metadata_statements(
@@ -863,8 +997,20 @@ class RDFSerializer:
"@id": entity_id,
"@type": entity.get("type", "semantica:Entity"),
"semantica:text": entity.get("text") or entity.get("label", ""),
"semantica:confidence": entity.get("confidence", 1.0),
}
confidence = normalize_confidence(entity.get("confidence", 1.0))
if confidence is None:
self.logger.warning(
f"Entity {entity_id} has a confidence that is not a number "
f"({entity.get('confidence')!r}), so no confidence is written"
)
else:
# A native JSON number becomes xsd:double once expanded, so the
# value is written as a typed literal instead.
node["semantica:confidence"] = {
"@value": confidence,
"@type": CONFIDENCE_DATATYPE,
}
node.update(
_jsonld_metadata_entries(
_metadata_statements(
@@ -969,11 +1115,20 @@ class RDFSerializer:
f'{subject} {expand_uri("semantica:text")} "{safe_text}" .'
)
# Confidence property
confidence = entity.get("confidence")
if confidence is not None:
# Confidence property. The default matches the other serializers,
# which have always written one; omitting it here was half of why
# Turtle and N-Triples of one KG were different graphs (#1100).
raw_confidence = entity.get("confidence", 1.0)
confidence = normalize_confidence(raw_confidence)
if confidence is None:
self.logger.warning(
f"Entity {entity.get('id')} has a confidence that is not a "
f"number ({raw_confidence!r}), so no confidence is written"
)
else:
lines.append(
f'{subject} {expand_uri("semantica:confidence")} "{confidence}"^^<http://www.w3.org/2001/XMLSchema#float> .'
f'{subject} {expand_uri("semantica:confidence")} '
f'"{confidence}"^^<{CONFIDENCE_DATATYPE}> .'
)
lines.extend(
semantica/ingest/methods.py
+79 -78
View File
@@ -180,6 +180,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
from ..utils.exceptions import ConfigurationError, ProcessingError
from ..utils.logging import get_logger
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
from .config import ingest_config
from .file_ingestor import FileIngestor, FileObject
from .registry import method_registry
@@ -248,12 +249,12 @@ def ingest_file(
# Check for custom method in registry
custom_method = method_registry.get("file", method)
if custom_method and custom_method != ingest_file:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -314,12 +315,12 @@ def ingest_parquet(
"""
custom_method = method_registry.get("parquet", method)
if custom_method and custom_method != ingest_parquet:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
try:
@@ -393,12 +394,12 @@ def ingest_arrow(
"""
custom_method = method_registry.get("arrow", method)
if custom_method and custom_method != ingest_arrow:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
try:
@@ -477,12 +478,12 @@ def ingest_xml(
"""
custom_method = method_registry.get("xml", method)
if custom_method and custom_method != ingest_xml:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
from .xml_ingestor import XMLIngestor
@@ -541,12 +542,12 @@ def ingest_web(
# Check for custom method in registry
custom_method = method_registry.get("web", method)
if custom_method and custom_method != ingest_web:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
try:
@@ -631,12 +632,12 @@ def ingest_public_api(
"""
custom_method = method_registry.get("public_api", method)
if custom_method and custom_method != ingest_public_api:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
from .public_api_ingestor import PublicAPIExamples, PublicAPIIngestor
@@ -718,12 +719,12 @@ def ingest_feed(
# Check for custom method in registry
custom_method = method_registry.get("feed", method)
if custom_method and custom_method != ingest_feed:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
try:
@@ -787,12 +788,12 @@ def ingest_stream(
# Check for custom method in registry
custom_method = method_registry.get("stream", method)
if custom_method and custom_method != ingest_stream:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
from .stream_ingestor import StreamIngestor
@@ -864,12 +865,12 @@ def ingest_repository(
# Check for custom method in registry
custom_method = method_registry.get("repo", method)
if custom_method and custom_method != ingest_repository:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
try:
@@ -936,12 +937,12 @@ def ingest_email(
# Check for custom method in registry
custom_method = method_registry.get("email", method)
if custom_method and custom_method != ingest_email:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
try:
@@ -1015,12 +1016,12 @@ def ingest_ontology(
# Check for custom method in registry
custom_method = method_registry.get("ontology", method)
if custom_method and custom_method != ingest_ontology:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
from .ontology_ingestor import OntologyIngestor
@@ -1081,12 +1082,12 @@ def ingest_database(
if method:
custom_method = method_registry.get("db", method)
if custom_method and custom_method != ingest_database:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
from .db_ingestor import DBIngestor
@@ -1188,12 +1189,12 @@ def ingest_mcp(
# Check for custom method in registry
custom_method = method_registry.get("mcp", method)
if custom_method and custom_method != ingest_mcp:
try:
return custom_method(source, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
from .mcp_ingestor import MCPIngestor
semantica/kg/methods.py
+19 -18
View File
@@ -142,6 +142,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from ..utils.exceptions import ConfigurationError, ProcessingError
from ..utils.logging import get_logger
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
from .centrality_calculator import CentralityCalculator
from .community_detector import CommunityDetector
from .config import kg_config
@@ -189,12 +190,12 @@ def build_kg(
# Check for custom method in registry
custom_method = method_registry.get("build", method)
if custom_method:
try:
return custom_method(sources, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, sources, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -237,12 +238,12 @@ def analyze_graph(
# Check for custom method in registry
custom_method = method_registry.get("analyze", method)
if custom_method:
try:
return custom_method(graph, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, graph, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
@@ -441,12 +442,12 @@ def analyze_connectivity(
# Check for custom method in registry
custom_method = method_registry.get("connectivity", method)
if custom_method:
try:
return custom_method(graph, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, graph, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
# Get config
semantica/normalize/methods.py
+79 -80
View File
@@ -125,6 +125,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from ..utils.exceptions import ConfigurationError, ProcessingError
from ..utils.logging import get_logger
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
from .config import normalize_config
from .data_cleaner import DataCleaner
from .date_normalizer import DateNormalizer
@@ -168,12 +169,12 @@ def normalize_text(text: str, method: str = "default", **kwargs) -> str:
"""
custom_method = method_registry.get("text", method)
if custom_method:
try:
return custom_method(text, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("text")
@@ -212,12 +213,12 @@ def clean_text(text: str, method: str = "default", **kwargs) -> str:
"""
custom_method = method_registry.get("clean", method)
if custom_method:
try:
return custom_method(text, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("clean")
@@ -262,12 +263,12 @@ def normalize_entity(
"""
custom_method = method_registry.get("entity", method)
if custom_method:
try:
return custom_method(entity_name, entity_type, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, entity_name, entity_type, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("entity")
@@ -309,12 +310,12 @@ def resolve_aliases(
"""
custom_method = method_registry.get("entity", method)
if custom_method:
try:
return custom_method(entity_name, entity_type, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, entity_name, entity_type, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("entity")
@@ -356,12 +357,12 @@ def disambiguate_entity(
"""
custom_method = method_registry.get("entity", method)
if custom_method:
try:
return custom_method(entity_name, **context)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = context.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, entity_name, fallback_on_custom_error=fallback, **context
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("entity")
@@ -411,12 +412,12 @@ def normalize_date(
"""
custom_method = method_registry.get("date", method)
if custom_method:
try:
return custom_method(date_input, format, timezone, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, date_input, format, timezone, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("date")
@@ -452,12 +453,12 @@ def normalize_time(time_input: Any, method: str = "default", **kwargs) -> str:
"""
custom_method = method_registry.get("date", method)
if custom_method:
try:
return custom_method(time_input, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, time_input, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("date")
@@ -498,12 +499,12 @@ def normalize_number(
"""
custom_method = method_registry.get("number", method)
if custom_method:
try:
return custom_method(number_input, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, number_input, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("number")
@@ -542,12 +543,12 @@ def normalize_quantity(
"""
custom_method = method_registry.get("number", method)
if custom_method:
try:
return custom_method(quantity_input, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, quantity_input, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("number")
@@ -598,14 +599,12 @@ def clean_data(
"""
custom_method = method_registry.get("clean", method)
if custom_method:
try:
return custom_method(
dataset, remove_duplicates, validate, handle_missing, **kwargs
)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, dataset, remove_duplicates, validate, handle_missing, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("clean")
@@ -653,12 +652,12 @@ def detect_duplicates(
"""
custom_method = method_registry.get("clean", method)
if custom_method:
try:
return custom_method(dataset, threshold, key_fields, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, dataset, threshold, key_fields, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("clean")
@@ -700,12 +699,12 @@ def detect_language(
"""
custom_method = method_registry.get("language", method)
if custom_method:
try:
return custom_method(text, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("language")
@@ -761,12 +760,12 @@ def handle_encoding(
"""
custom_method = method_registry.get("encoding", method)
if custom_method:
try:
return custom_method(data, operation, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, operation, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = normalize_config.get_method_config("encoding")
semantica/ontology/engine.py
+12
View File
@@ -203,6 +203,8 @@ class OntologyEngine:
include_inherited: bool = True,
severity: str = "Violation",
quality_tier: str = "standard",
target_namespace: Optional[str] = None,
attach_domainless_properties: bool = False,
validate_output: bool = False,
**options,
) -> str:
@@ -217,6 +219,12 @@ class OntologyEngine:
include_inherited: Propagate parent class property shapes to child shapes.
severity: Default severity — "Violation", "Warning", or "Info".
quality_tier: Constraint completeness — "basic", "standard" (default), "strict".
target_namespace: Namespace sh:targetClass and sh:path are expanded in,
when the ontology supplies no absolute IRIs. Separate from base_uri,
which says where the shape resources themselves live.
attach_domainless_properties: Attach properties with no declared domain
to every node shape. Off by default: it states a constraint the
ontology does not.
validate_output: Syntax-check output via rdflib before returning.
Returns:
@@ -236,12 +244,16 @@ class OntologyEngine:
or (ns.get("base_uri") if isinstance(ns, dict) else None)
or "https://semantica.dev/shapes/"
)
# These two are constructor-only on SHACLGenerator, so forwarding
# them through generate(**options) silently dropped them.
generator = SHACLGenerator(
base_uri=resolved_base,
shapes_uri=shapes_uri,
include_inherited=include_inherited,
severity=severity,
quality_tier=quality_tier,
target_namespace=target_namespace,
attach_domainless_properties=attach_domainless_properties,
)
graph = generator.generate(ontology, **options)
self.progress.update_tracking(tracking_id, message="Serializing SHACL graph")
semantica/ontology/ontology_generator.py
+157 -16
View File
@@ -30,6 +30,7 @@ Author: Semantica Contributors
License: MIT
"""
import re
from dataclasses import dataclass, field, replace as dataclass_replace
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -781,6 +782,13 @@ class SHACLGraph:
shapes_uri: str
node_shapes: List[NodeShape] = field(default_factory=list)
prefixes: Dict[str, str] = field(default_factory=dict)
# Bare names mapped to the absolute IRI the data uses for them. Shapes are
# indexed internally by name; this is what those names expand to at
# serialisation time (#1104). Classes and properties are kept apart because
# a property may legitimately share a class's name, and a single map would
# silently give it the class's IRI.
class_iris: Dict[str, str] = field(default_factory=dict)
property_iris: Dict[str, str] = field(default_factory=dict)
class SHACLGenerator:
@@ -813,8 +821,23 @@ class SHACLGenerator:
include_inherited: bool = True,
severity: str = "Violation",
quality_tier: str = "standard",
target_namespace: Optional[str] = None,
attach_domainless_properties: bool = False,
config: Optional[Dict[str, Any]] = None,
):
"""
Args:
base_uri: Namespace the shape resources themselves live in.
target_namespace: Namespace the terms being validated live in, used
to expand sh:targetClass and sh:path when the ontology does not
supply absolute IRIs. This is deliberately separate from
base_uri: shapes that target their own namespace match nothing,
and pySHACL reports that as conforming (#1104).
attach_domainless_properties: When True, a property with no declared
domain is attached to every node shape, which is the pre-0.6.6
behaviour. It invents a constraint the ontology never stated, so
it is off by default (#1105).
"""
self.logger = get_logger("ontology_shacl")
self.progress_tracker = get_progress_tracker()
self.base_uri = base_uri.rstrip("/") + "/"
@@ -822,6 +845,8 @@ class SHACLGenerator:
self.include_inherited = include_inherited
self.severity = severity
self.quality_tier = quality_tier
self.target_namespace = target_namespace
self.attach_domainless_properties = attach_domainless_properties
self.config = config or {}
# ── Public API ────────────────────────────────────────────────────────────
@@ -859,10 +884,15 @@ class SHACLGenerator:
"ex": base_uri,
}
target_ns = self._resolve_target_namespace(ontology, base_uri)
prefixes["ex"] = target_ns
graph = SHACLGraph(
base_uri=base_uri,
shapes_uri=self.shapes_uri,
prefixes=prefixes,
class_iris=self._build_term_index(classes, target_ns),
property_iris=self._build_term_index(properties, target_ns),
)
self.progress_tracker.update_tracking(tracking_id, message="Generating node shapes")
@@ -931,6 +961,80 @@ class SHACLGenerator:
# ── Internal pipeline stages ──────────────────────────────────────────────
_DEFAULT_TARGET_NAMESPACE = "https://semantica.dev/ns#"
@staticmethod
def _is_absolute_iri(value: Any) -> bool:
return isinstance(value, str) and bool(
re.match(r"^[A-Za-z][A-Za-z0-9+.\-]*:", value.strip())
)
@staticmethod
def _join_iri(base: str, local: str) -> str:
separator = "" if base.endswith(("#", "/", ":")) else "#"
return f"{base}{separator}{local}"
def _resolve_target_namespace(self, ontology: Dict[str, Any], base_uri: str) -> str:
"""
Decide which namespace sh:targetClass and sh:path are expanded in.
base_uri says where the shapes live. It is only the right answer here
when the ontology declared it, meaning shapes and terms deliberately
share a namespace. Falling back to the shapes namespace produces shapes
that target terms no data graph uses.
"""
if self.target_namespace:
return self.target_namespace
namespace = ontology.get("namespace")
if isinstance(namespace, dict) and namespace.get("base_uri"):
return str(namespace["base_uri"])
# An IRI already carried by a term is the most reliable evidence of
# where the data lives, so prefer it over any configured default.
for terms in (ontology.get("classes"), ontology.get("properties")):
for term in terms or []:
if not isinstance(term, dict):
continue
for key in ("uri", "iri", "id"):
value = term.get(key)
if self._is_absolute_iri(value):
value = value.strip()
cut = max(value.rfind("#"), value.rfind("/"))
if cut != -1:
return value[: cut + 1]
if ontology.get("uri") and self._is_absolute_iri(ontology["uri"]):
return str(ontology["uri"])
if base_uri != self.base_uri:
return base_uri
return self._DEFAULT_TARGET_NAMESPACE
def _build_term_index(
self, terms: List[Dict[str, Any]], target_ns: str
) -> Dict[str, str]:
"""Map each term's name to the absolute IRI it expands to."""
index: Dict[str, str] = {}
for term in terms or []:
if not isinstance(term, dict):
continue
name = term.get("name")
if not isinstance(name, str) or not name.strip():
continue
name = name.strip()
iri = ""
for key in ("uri", "iri", "id"):
value = term.get(key)
if isinstance(value, str) and value.strip():
value = value.strip()
iri = value if self._is_absolute_iri(value) else self._join_iri(target_ns, value)
break
index.setdefault(name, iri or self._join_iri(target_ns, name))
return index
def _build_class_index(
self, classes: List[Dict[str, Any]]
) -> Dict[str, Dict[str, Any]]:
@@ -979,13 +1083,23 @@ class SHACLGenerator:
self.logger.debug(
f"Property '{pname}' domain '{d}' has no matching node shape — skipped"
)
else:
# No domain declared → attach to all shapes
self.logger.debug(
f"Property '{pname}' has no domain — attaching to all node shapes"
elif self.attach_domainless_properties:
self.logger.warning(
f"Property '{pname}' declares no domain and is being attached "
"to every node shape because attach_domainless_properties is "
"set. This states a constraint the ontology does not."
)
for node_shape in graph.node_shapes:
node_shape.property_shapes.append(self._build_property_shape(prop))
else:
# Attaching here would state a constraint the ontology does not.
# With minCount 1 that invalidates every instance of every
# class, so the property is left unattached (#1105).
self.logger.warning(
f"Property '{pname}' declares no domain, so it is not attached "
"to any node shape. Declare a domain, or pass "
"attach_domainless_properties=True to restore the old behaviour."
)
def _build_property_shape(self, prop: Dict[str, Any]) -> PropertyShape:
ptype = prop.get("type", "")
@@ -1065,13 +1179,40 @@ class SHACLGenerator:
def _prefix_decls(self, graph: SHACLGraph) -> str:
return "\n".join(f"@prefix {p}: <{u}> ." for p, u in sorted(graph.prefixes.items()))
def _uri(self, graph: SHACLGraph, local: str) -> str:
"""Return a compact URI reference; fall back to ex:local for bare names."""
def _term_iri(self, graph: SHACLGraph, local: str, kind: str = "class") -> str:
"""
Resolve a class or property name to the absolute IRI the data uses.
Every serializer goes through here. Turtle alone was corrected at first,
which left JSON-LD and N-Triples still pasting names onto the shapes
namespace, so the shapes they produced went on matching nothing (#1104).
`kind` selects the index: a property may share a class's name, and the
two can carry different IRIs.
"""
if local.startswith("http://") or local.startswith("https://"):
return f"<{local}>"
return local
index = graph.property_iris if kind == "property" else graph.class_iris
resolved = index.get(local)
if resolved:
return resolved
# Fall back to the other index before giving up: sh:class names a class,
# but a caller may pass a term only registered on the other side.
other = graph.class_iris if kind == "property" else graph.property_iris
resolved = other.get(local)
if resolved:
return resolved
if ":" in local:
return local
return f"ex:{local}"
separator = "" if graph.base_uri.endswith(("#", "/", ":")) else "#"
return f"{graph.base_uri}{separator}{local}"
def _uri(self, graph: SHACLGraph, local: str, kind: str = "class") -> str:
"""Turtle-facing wrapper: the resolved IRI, in angle brackets."""
resolved = self._term_iri(graph, local, kind)
if resolved.startswith(("http://", "https://", "urn:")):
return f"<{resolved}>"
return resolved
def _serialize_turtle(self, graph: SHACLGraph) -> str:
lines = [self._prefix_decls(graph), ""]
@@ -1098,11 +1239,11 @@ class SHACLGenerator:
is_last = i == len(node_shape.property_shapes) - 1
terminator = " ." if is_last else " ;"
parts = [" sh:property ["]
parts.append(f" sh:path {self._uri(graph, ps.path)} ;")
parts.append(f' sh:path {self._uri(graph, ps.path, "property")} ;')
if ps.datatype:
parts.append(f" sh:datatype {ps.datatype} ;")
if ps.class_:
parts.append(f" sh:class {self._uri(graph, ps.class_)} ;")
parts.append(f' sh:class {self._uri(graph, ps.class_, "class")} ;')
if ps.min_count is not None:
parts.append(f" sh:minCount {ps.min_count} ;")
if ps.max_count is not None:
@@ -1143,7 +1284,7 @@ class SHACLGenerator:
node: Dict[str, Any] = {
"@id": shape_id,
"@type": "sh:NodeShape",
"sh:targetClass": {"@id": f"{graph.base_uri}{node_shape.target_class}"},
"sh:targetClass": {"@id": self._term_iri(graph, node_shape.target_class, "class")},
}
if node_shape.name:
node["sh:name"] = node_shape.name
@@ -1156,7 +1297,7 @@ class SHACLGenerator:
props = []
for ps in node_shape.property_shapes:
p: Dict[str, Any] = {
"sh:path": {"@id": f"{graph.base_uri}{ps.path}"}
"sh:path": {"@id": self._term_iri(graph, ps.path, "property")}
}
if ps.datatype:
dt = ps.datatype.replace(
@@ -1164,7 +1305,7 @@ class SHACLGenerator:
)
p["sh:datatype"] = {"@id": dt}
if ps.class_:
p["sh:class"] = {"@id": f"{graph.base_uri}{ps.class_}"}
p["sh:class"] = {"@id": self._term_iri(graph, ps.class_, "class")}
if ps.min_count is not None:
p["sh:minCount"] = ps.min_count
if ps.max_count is not None:
@@ -1197,7 +1338,7 @@ class SHACLGenerator:
for i, node_shape in enumerate(graph.node_shapes):
shape_uri = f"<{graph.base_uri}{node_shape.target_class}Shape>"
class_uri = f"<{graph.base_uri}{node_shape.target_class}>"
class_uri = f'<{self._term_iri(graph, node_shape.target_class, "class")}>'
t(shape_uri, f"<{RDF}type>", f"<{SHACL}NodeShape>")
t(shape_uri, f"<{SHACL}targetClass>", class_uri)
if node_shape.name:
@@ -1212,13 +1353,13 @@ class SHACLGenerator:
for j, ps in enumerate(node_shape.property_shapes):
bnode = f"_:ps{i}_{j}"
t(shape_uri, f"<{SHACL}property>", bnode)
prop_uri = f"<{graph.base_uri}{ps.path}>"
prop_uri = f'<{self._term_iri(graph, ps.path, "property")}>'
t(bnode, f"<{SHACL}path>", prop_uri)
if ps.datatype:
dt_uri = ps.datatype.replace("xsd:", XSD)
t(bnode, f"<{SHACL}datatype>", f"<{dt_uri}>")
if ps.class_:
t(bnode, f"<{SHACL}class>", f"<{graph.base_uri}{ps.class_}>")
t(bnode, f"<{SHACL}class>", f'<{self._term_iri(graph, ps.class_, "class")}>')
if ps.min_count is not None:
t(bnode, f"<{SHACL}minCount>", f'"{ps.min_count}"^^<{XSD}integer>')
if ps.max_count is not None:
semantica/ontology/vocabulary/semantica-ns.ttl
+7 -6
View File
@@ -53,16 +53,17 @@ this IRI.""" ;
sem:confidence a owl:DatatypeProperty ;
rdfs:label "confidence" ;
rdfs:range xsd:decimal ;
rdfs:comment """Extractor confidence in the assertion, on the unit interval.
Emitted for both entities and relationships, so the domain is left open rather
than tied to sem:Entity.
No rdfs:range is declared, deliberately. The Turtle serializer writes the value
bare, which the Turtle grammar reads as xsd:decimal, while the N-Triples
serializer types it xsd:float explicitly, and those two datatypes are disjoint.
Declaring either one would make the vocabulary contradict one of the exporters.
Issue #1100 tracks the disagreement; a range belongs here once the serializers
agree on one.""" ;
The range was left undeclared when this vocabulary first shipped, because the
four serializers disagreed: Turtle wrote the value bare, which the Turtle
grammar reads as xsd:decimal, N-Triples typed it xsd:float, RDF/XML wrote a
plain literal, and JSON-LD wrote a native number, which expands to xsd:double.
Declaring any one of them would have contradicted three exporters. Issue #1100
settled that on xsd:decimal, which every serializer now writes.""" ;
rdfs:isDefinedBy <https://semantica.dev/ns> .
sem:metadata a owl:AnnotationProperty ;
semantica/parse/methods.py
+73 -72
View File
@@ -125,6 +125,7 @@ from typing import Any, Callable, Dict, List, Optional, Union
from ..utils.exceptions import ConfigurationError, ProcessingError
from ..utils.logging import get_logger
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
from .code_parser import CodeParser
from .config import parse_config
from .csv_parser import CSVParser
@@ -178,12 +179,12 @@ def parse_document(
"""
custom_method = method_registry.get("document", method)
if custom_method:
try:
return custom_method(file_path, file_type, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, file_type, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("document")
@@ -289,12 +290,12 @@ def parse_web_content(
"""
custom_method = method_registry.get("web", method)
if custom_method:
try:
return custom_method(content, content_type, base_url, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, content, content_type, base_url, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("web")
@@ -345,12 +346,12 @@ def parse_structured_data(
"""
custom_method = method_registry.get("structured", method)
if custom_method:
try:
return custom_method(data, data_format, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, data_format, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("structured")
@@ -392,12 +393,12 @@ def parse_email(
"""
custom_method = method_registry.get("email", method)
if custom_method:
try:
return custom_method(email_content, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, email_content, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("email")
@@ -442,12 +443,12 @@ def parse_code(
"""
custom_method = method_registry.get("code", method)
if custom_method:
try:
return custom_method(file_path, language, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, language, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("code")
@@ -495,12 +496,12 @@ def parse_media(
"""
custom_method = method_registry.get("media", method)
if custom_method:
try:
return custom_method(file_path, media_type, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, media_type, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("media")
@@ -541,12 +542,12 @@ def parse_pdf(
"""
custom_method = method_registry.get("document", method)
if custom_method:
try:
return custom_method(file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("document")
@@ -585,12 +586,12 @@ def parse_docx(
"""
custom_method = method_registry.get("document", method)
if custom_method:
try:
return custom_method(file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("document")
@@ -628,12 +629,12 @@ def parse_json(file_path: Union[str, Path], method: str = "default", **kwargs) -
"""
custom_method = method_registry.get("structured", method)
if custom_method:
try:
return custom_method(file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("structured")
@@ -675,12 +676,12 @@ def parse_csv(
"""
custom_method = method_registry.get("structured", method)
if custom_method:
try:
return custom_method(file_path, delimiter, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, delimiter, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("structured")
@@ -714,12 +715,12 @@ def parse_xml(file_path: Union[str, Path], method: str = "default", **kwargs) ->
"""
custom_method = method_registry.get("structured", method)
if custom_method:
try:
return custom_method(file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("structured")
@@ -762,12 +763,12 @@ def parse_image(
"""
custom_method = method_registry.get("media", method)
if custom_method:
try:
return custom_method(file_path, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
config = parse_config.get_method_config("media")
semantica/utils/custom_methods.py
+75
View File
@@ -0,0 +1,75 @@
"""
Invocation policy for methods registered through a MethodRegistry.
Every module that supports custom methods used to wrap the registered callable
in a bare `except Exception`, log a warning, and carry on into the built-in
implementation. That makes a registered method advisory: it can add behaviour,
but it cannot decline.
For a gate, a validator or a policy check that is the whole point. Raising is
how such a method says "do not produce this output". Catching the exception and
running the default produces exactly the output the caller registered the method
to prevent, and the only trace is a warning (issue #1108).
Exceptions from a registered method therefore propagate by default. Callers who
relied on the old behaviour can pass `fallback_on_custom_error=True`, which
restores the warn-and-continue path for that call.
"""
from typing import Any, Callable
class _FellBack:
"""Sentinel: the custom method failed and the caller should use the default."""
__slots__ = ()
def __repr__(self) -> str: # pragma: no cover - debugging aid
return "CUSTOM_METHOD_FELL_BACK"
#: Returned by :func:`call_custom_method` when a custom method raised and
#: ``fallback_on_custom_error=True`` was passed. Compare with ``is``.
CUSTOM_METHOD_FELL_BACK = _FellBack()
def call_custom_method(
logger: Any,
method: Any,
custom_method: Callable[..., Any],
/,
*args: Any,
**kwargs: Any,
) -> Any:
"""
Invoke a registered custom method.
Args:
logger: Module logger, used only on the opt-in fallback path.
method: The registered name, for the warning message.
custom_method: The registered callable.
*args: Positional arguments for the custom method.
**kwargs: Keyword arguments for the custom method. The reserved key
``fallback_on_custom_error`` is consumed here and never forwarded.
Returns:
Whatever the custom method returns, or :data:`CUSTOM_METHOD_FELL_BACK`
when it raised and the caller opted into falling back.
Raises:
Exception: Whatever the custom method raised, unless the caller passed
``fallback_on_custom_error=True``.
"""
fallback = bool(kwargs.pop("fallback_on_custom_error", False))
if not fallback:
return custom_method(*args, **kwargs)
try:
return custom_method(*args, **kwargs)
except Exception as exc:
logger.warning(
f"Custom method {method} failed: {exc}, falling back to default "
"because fallback_on_custom_error was set"
)
return CUSTOM_METHOD_FELL_BACK
tests/export/test_confidence_literal_typing.py
+212
View File
@@ -0,0 +1,212 @@
"""
Regression tests for #1100 and #1102.
#1100: the four RDF serializers rendered the same confidence value four
different ways. Turtle wrote it bare, which the Turtle grammar reads as
xsd:decimal. N-Triples typed it xsd:float. RDF/XML wrote a plain literal with
no datatype at all. JSON-LD emitted a native JSON number, which becomes
xsd:double. Those are four distinct RDF terms, so a FILTER matches at most one
of them and merging two exports of one graph yields two confidence values for
the same entity.
#1102: the Turtle path interpolated the value with no type check, so a
non-numeric confidence produced `semantica:confidence high .`, which is not
parseable Turtle. One bad field made the whole export unreadable.
"""
import json
import pytest
rdflib = pytest.importorskip("rdflib")
from rdflib import Graph, URIRef # noqa: E402
from rdflib.compare import isomorphic # noqa: E402
from semantica.export.rdf_exporter import RDFSerializer # noqa: E402
NS = "https://semantica.dev/ns#"
CONFIDENCE = URIRef(NS + "confidence")
XSD_DECIMAL = URIRef("http://www.w3.org/2001/XMLSchema#decimal")
def _kg(confidence):
entity = {"id": NS + "e1", "text": "Acme", "type": NS + "ORG"}
if confidence is not _ABSENT:
entity["confidence"] = confidence
return {"entities": [entity], "relationships": []}
_ABSENT = object()
def _graphs(kg):
"""Parse every serialization of one KG into a graph, keyed by format."""
serializer = RDFSerializer()
out = {}
out["turtle"] = Graph()
out["turtle"].parse(data=serializer.serialize_to_turtle(json.loads(json.dumps(kg))),
format="turtle")
out["ntriples"] = Graph()
out["ntriples"].parse(data=serializer.serialize_to_ntriples(json.loads(json.dumps(kg))),
format="nt")
out["rdfxml"] = Graph()
out["rdfxml"].parse(data=serializer.serialize_to_rdfxml(json.loads(json.dumps(kg))),
format="xml")
jsonld = serializer.serialize_to_jsonld(json.loads(json.dumps(kg)))
out["jsonld"] = Graph()
out["jsonld"].parse(
data=jsonld if isinstance(jsonld, str) else json.dumps(jsonld), format="json-ld"
)
return out
def _confidence_terms(graph):
return [o for _, p, o in graph if p == CONFIDENCE]
def test_every_serializer_agrees_on_the_confidence_term():
"""The heart of #1100. One value, one RDF term, whatever the format."""
terms = {}
for name, graph in _graphs(_kg(0.9)).items():
values = _confidence_terms(graph)
assert len(values) == 1, f"{name} emitted {len(values)} confidence triples"
terms[name] = values[0]
distinct = set(terms.values())
assert len(distinct) == 1, (
"the same confidence serialised as different RDF terms: "
+ ", ".join(f"{k}={v!r} ({v.datatype})" for k, v in terms.items())
)
def test_the_agreed_term_is_a_typed_decimal():
for name, graph in _graphs(_kg(0.9)).items():
term = _confidence_terms(graph)[0]
assert term.datatype == XSD_DECIMAL, f"{name} typed it {term.datatype}"
assert str(term) == "0.9", f"{name} wrote the lexical form {str(term)!r}"
def test_turtle_and_ntriples_are_the_same_graph():
"""#1100 as filed: two serializations of one KG must not be two graphs."""
graphs = _graphs(_kg(0.9))
assert isomorphic(graphs["turtle"], graphs["ntriples"]), (
"Turtle only:\n"
+ "\n".join(str(t) for t in set(graphs["turtle"]) - set(graphs["ntriples"]))
+ "\nN-Triples only:\n"
+ "\n".join(str(t) for t in set(graphs["ntriples"]) - set(graphs["turtle"]))
)
def test_a_numeric_string_is_accepted():
for name, graph in _graphs(_kg("0.85")).items():
terms = _confidence_terms(graph)
assert terms, f"{name} dropped a usable numeric string"
assert terms[0].datatype == XSD_DECIMAL
assert str(terms[0]) == "0.85"
def test_an_integer_confidence_is_accepted():
for name, graph in _graphs(_kg(1)).items():
terms = _confidence_terms(graph)
assert terms, f"{name} dropped an integer confidence"
assert terms[0].datatype == XSD_DECIMAL
def test_a_small_value_is_not_written_in_exponent_notation():
"""1e-05 is a valid Python repr and an invalid xsd:decimal lexical form."""
for name, graph in _graphs(_kg(0.00001)).items():
term = _confidence_terms(graph)[0]
assert "e" not in str(term).lower(), f"{name} wrote {str(term)!r}"
assert term.value is not None, f"{name} produced an ill-typed literal"
# ── #1102 ────────────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"bad", ["high", "", "not a number", None, True, False, [0.9], {"v": 0.9},
float("nan"), float("inf")],
)
def test_an_unusable_confidence_never_breaks_the_export(bad):
"""`semantica:confidence high .` made the whole Turtle document unparseable."""
serializer = RDFSerializer()
kg = _kg(bad)
turtle = serializer.serialize_to_turtle(json.loads(json.dumps(kg, default=str)))
graph = Graph()
graph.parse(data=turtle, format="turtle") # must not raise
assert _confidence_terms(graph) == [], (
f"{bad!r} was emitted as a confidence value: {_confidence_terms(graph)}"
)
# The rest of the entity must survive.
assert (URIRef(NS + "e1"), URIRef(NS + "text"), rdflib.Literal("Acme")) in graph
def test_an_unusable_confidence_is_dropped_consistently_everywhere():
for name, graph in _graphs(_kg("high")).items():
assert _confidence_terms(graph) == [], f"{name} still emitted it"
def test_all_serializers_stay_parseable_with_an_unusable_confidence():
graphs = _graphs(_kg("high")) # each parse would raise on malformed output
assert isomorphic(graphs["turtle"], graphs["ntriples"])
def test_the_emitted_datatype_matches_the_shipped_vocabulary():
"""
Drift guard. The vocabulary shipped with no rdfs:range on sem:confidence
precisely because the serializers disagreed. Now that they agree, the range
is declared, and the two must not drift apart again.
"""
from rdflib import RDFS
from semantica.export.rdf_exporter import CONFIDENCE_DATATYPE
from semantica.ontology.vocabulary import vocabulary_turtle
vocabulary = Graph()
vocabulary.parse(data=vocabulary_turtle(), format="turtle")
declared = list(vocabulary.objects(URIRef(NS + "confidence"), RDFS.range))
assert declared, "sem:confidence declares no rdfs:range"
assert str(declared[0]) == CONFIDENCE_DATATYPE, (
f"vocabulary says {declared[0]}, serializers write {CONFIDENCE_DATATYPE}"
)
# ── Review findings on the first revision of this fix ────────────────────────
@pytest.mark.parametrize("huge", ["1e100000000", "1E1000000", "-1e999999", 10**400])
def test_an_absurd_magnitude_is_rejected_not_expanded(huge):
"""
xsd:decimal has no exponent notation, so the value has to be written out.
"1e100000000" is eleven characters that expand to a hundred million digits,
and the export path continues past validation errors.
"""
from semantica.export.rdf_exporter import normalize_confidence
assert normalize_confidence(huge) is None
def test_a_legitimately_small_confidence_is_still_accepted():
from semantica.export.rdf_exporter import normalize_confidence
assert normalize_confidence(1e-9) == "0.000000001"
def test_signed_zero_is_normalized():
"""0.0 and -0.0 would otherwise be two distinct RDF terms."""
from semantica.export.rdf_exporter import normalize_confidence
assert normalize_confidence(0.0) == normalize_confidence(-0.0) == "0"
def test_signed_zero_gives_one_term_across_serializers():
positive = {n: _confidence_terms(g)[0] for n, g in _graphs(_kg(0.0)).items()}
negative = {n: _confidence_terms(g)[0] for n, g in _graphs(_kg(-0.0)).items()}
assert set(positive.values()) | set(negative.values()) == set(positive.values())
assert len(set(positive.values())) == 1
tests/export/test_owl_time_reachability.py
+189
View File
@@ -0,0 +1,189 @@
"""
Regression test for #1106.
`include_temporal=True` emitted a well formed OWL-Time interval hanging off a
relationship IRI that appears nowhere else in the graph. The relationship
itself is written as a single triple, `<e1> <employs> <e2>`, so there is no
node to carry the time and no path from the edge to its validity interval.
The timestamps parsed, validated and meant nothing: no query could reach them
from the relationship they describe.
The JSON-LD path already reifies relationships as sem:Relationship with
sem:source, sem:target and sem:type, and the shipped vocabulary declares all
four terms. Turtle now emits the same shape when it has temporal data to
attach, so the interval hangs off a node the graph can actually reach.
"""
import pytest
rdflib = pytest.importorskip("rdflib")
from rdflib import Graph, RDF, URIRef # noqa: E402
from semantica.export.rdf_exporter import RDFSerializer # noqa: E402
NS = "https://semantica.dev/ns#"
TIME = "http://www.w3.org/2006/time#"
E1, E2 = NS + "e1", NS + "e2"
EMPLOYS = NS + "employs"
KG = {
"entities": [
{"id": E1, "text": "Acme", "type": NS + "ORG"},
{"id": E2, "text": "Globex", "type": NS + "ORG"},
],
"relationships": [
{
"source_id": E1,
"target_id": E2,
"type": EMPLOYS,
"valid_from": "2024-01-01T00:00:00Z",
"valid_until": "2025-01-01T00:00:00Z",
}
],
}
def _graph(**options):
turtle = RDFSerializer().serialize_to_turtle(
{k: [dict(v) for v in vs] for k, vs in KG.items()}, **options
)
graph = Graph()
graph.parse(data=turtle, format="turtle")
return graph
def test_the_interval_holder_is_reachable_from_the_graph():
"""The heart of #1106: the node carrying time had no inbound arc at all."""
graph = _graph(include_temporal=True)
holders = [s for s, p, _ in graph if str(p) == TIME + "hasTime"]
assert holders, "no OWL-Time interval was emitted"
for holder in holders:
inbound = [(s, p) for s, p, o in graph if o == holder]
outbound = [(p, o) for s, p, o in graph if s == holder and str(p) != TIME + "hasTime"]
assert inbound or outbound, (
f"{holder} carries an interval but nothing else in the graph mentions it"
)
def test_the_relationship_is_reified_so_time_has_a_subject():
graph = _graph(include_temporal=True)
relationships = list(graph.subjects(RDF.type, URIRef(NS + "Relationship")))
assert len(relationships) == 1, f"expected one reified relationship, got {relationships}"
node = relationships[0]
assert (node, URIRef(NS + "source"), URIRef(E1)) in graph
assert (node, URIRef(NS + "target"), URIRef(E2)) in graph
assert list(graph.objects(node, URIRef(TIME + "hasTime"))), (
"the reified relationship does not carry the interval"
)
def test_a_query_can_walk_from_the_edge_to_its_interval():
"""What the dangling node made impossible."""
graph = _graph(include_temporal=True)
rows = list(graph.query(
"""
PREFIX sem: <https://semantica.dev/ns#>
PREFIX time: <http://www.w3.org/2006/time#>
SELECT ?begin WHERE {
?s ?p ?o .
?rel sem:source ?s ;
sem:target ?o ;
time:hasTime/time:hasBeginning/time:inXSDDateTimeStamp ?begin .
}
"""
))
assert rows, "no path from the relationship to its validity interval"
assert str(rows[0][0]) == "2024-01-01T00:00:00Z"
def test_the_direct_triple_is_still_written():
"""Reification is added alongside the edge, it does not replace it."""
graph = _graph(include_temporal=True)
assert (URIRef(E1), URIRef(EMPLOYS), URIRef(E2)) in graph
def test_default_output_is_unchanged():
"""Nothing is reified when there is no temporal data to attach."""
graph = _graph()
assert list(graph.subjects(RDF.type, URIRef(NS + "Relationship"))) == []
assert (URIRef(E1), URIRef(EMPLOYS), URIRef(E2)) in graph
def test_a_relationship_without_temporal_data_is_not_reified():
kg = {
"entities": KG["entities"],
"relationships": [
{"source_id": E1, "target_id": E2, "type": EMPLOYS},
dict(KG["relationships"][0]),
],
}
turtle = RDFSerializer().serialize_to_turtle(kg, include_temporal=True)
graph = Graph()
graph.parse(data=turtle, format="turtle")
assert len(list(graph.subjects(RDF.type, URIRef(NS + "Relationship")))) == 1
def test_the_reified_terms_are_declared_in_the_shipped_vocabulary():
"""A reification nobody declared would just move the problem."""
from semantica.ontology.vocabulary import vocabulary_turtle
vocabulary = Graph()
vocabulary.parse(data=vocabulary_turtle(), format="turtle")
declared = {str(s) for s in vocabulary.subjects()}
graph = _graph(include_temporal=True)
node = next(iter(graph.subjects(RDF.type, URIRef(NS + "Relationship"))))
emitted = {str(p) for p in graph.predicates(node, None)} | {NS + "Relationship"}
undeclared = {t for t in emitted if t.startswith(NS) and t not in declared}
assert not undeclared, f"emitted but not declared in the vocabulary: {undeclared}"
# ── Review finding on the first revision of this fix ─────────────────────────
def test_the_reified_type_keeps_the_full_predicate():
"""
Truncating to the local name made two predicates from different namespaces
indistinguishable, and disagreed with the direct triple beside it.
"""
from rdflib import Literal
def reified_type(rel_type):
kg = {
"entities": KG["entities"],
"relationships": [
{
"source_id": E1,
"target_id": E2,
"type": rel_type,
"valid_from": "2024-01-01T00:00:00Z",
}
],
}
graph = Graph()
graph.parse(
data=RDFSerializer().serialize_to_turtle(kg, include_temporal=True),
format="turtle",
)
node = next(iter(graph.subjects(RDF.type, URIRef(NS + "Relationship"))))
return set(graph.objects(node, URIRef(NS + "type")))
a = reified_type("https://a.example/ns#employs")
b = reified_type("https://b.example/ns#employs")
assert a == {Literal("https://a.example/ns#employs")}, a
assert a != b, "two distinct predicates produced the same reified type"
def test_the_reified_type_matches_the_direct_triples_predicate():
from rdflib import Literal
graph = _graph(include_temporal=True)
node = next(iter(graph.subjects(RDF.type, URIRef(NS + "Relationship"))))
assert set(graph.objects(node, URIRef(NS + "type"))) == {Literal(EMPLOYS)}
assert (URIRef(E1), URIRef(EMPLOYS), URIRef(E2)) in graph
tests/ontology/test_ontology_advanced.py
+23 -10
View File
@@ -293,20 +293,33 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase):
f"Duplicate paths in {node_shape.target_class}: {paths}")
# 21
def test_no_domain_property_attaches_to_all_shapes(self):
onto = {
"classes": [{"name": "A"}, {"name": "B"}],
"properties": [
{"name": "globalProp", "type": "datatype", "range": "string"}
# no domain
],
}
gen = self._make_gen()
graph = gen.generate(onto)
_NO_DOMAIN_ONTOLOGY = {
"classes": [{"name": "A"}, {"name": "B"}],
"properties": [
{"name": "globalProp", "type": "datatype", "range": "string"}
# no domain
],
}
def test_no_domain_property_attaches_to_all_shapes_when_opted_in(self):
"""attach_domainless_properties=True keeps the pre-0.6.6 behaviour."""
gen = self._make_gen(attach_domainless_properties=True)
graph = gen.generate(self._NO_DOMAIN_ONTOLOGY)
for node_shape in graph.node_shapes:
paths = {ps.path for ps in node_shape.property_shapes}
self.assertIn("globalProp", paths)
def test_no_domain_property_is_not_attached_by_default(self):
"""
Attaching states a constraint the ontology does not (#1105). This test
previously asserted the opposite, which pinned the defect in place.
"""
gen = self._make_gen()
graph = gen.generate(self._NO_DOMAIN_ONTOLOGY)
for node_shape in graph.node_shapes:
paths = {ps.path for ps in node_shape.property_shapes}
self.assertNotIn("globalProp", paths)
# 22
def test_empty_classes_produces_no_shapes(self):
gen = self._make_gen()
tests/ontology/test_shacl_target_namespace.py
+330
View File
@@ -0,0 +1,330 @@
"""
Regression tests for #1104 and #1105.
#1104: SHACLGenerator used one namespace for two different jobs. `base_uri`
names where the shape resources live, and it was also used to expand every
sh:targetClass and sh:path. With the default `https://semantica.dev/shapes/`
that made shapes target `.../shapes/Person`, while data carries
`.../ns#Person` or the ontology's own class IRI. The shapes matched nothing.
pySHACL then reported conforms=True, because a shape with no focus nodes is
vacuously satisfied, so the mismatch was invisible to the validator the
package ships with.
#1105: a property with no declared domain was attached to every node shape,
which invents a constraint the ontology never stated. With minCount 1 that
makes every instance of every class invalid.
These tests validate real data through pySHACL rather than reading the shapes
text, so a shape that targets nothing cannot pass by being ignored.
"""
import pytest
rdflib = pytest.importorskip("rdflib")
pyshacl = pytest.importorskip("pyshacl")
from rdflib import Graph, RDF, Namespace # noqa: E402
from semantica.ontology.ontology_generator import SHACLGenerator # noqa: E402
SH = Namespace("http://www.w3.org/ns/shacl#")
ONTOLOGY_NS = "https://example.org/onto#"
SHAPES_NS = "https://semantica.dev/shapes/"
def _ontology(*, declare_namespace: bool, carry_class_uris: bool):
"""
Build the same ontology in the shapes the generator can hand over.
A generated ontology carries class URIs; a hand written one often carries
only names, and only sometimes declares a namespace. All of them have to
produce shapes that match the data.
"""
def class_def(name):
entry = {"name": name, "label": name}
if carry_class_uris:
entry["uri"] = ONTOLOGY_NS + name
return entry
def prop_def(name, **extra):
entry = {"name": name, "type": "datatype", "range": "string", **extra}
if carry_class_uris:
entry["uri"] = ONTOLOGY_NS + name
return entry
ontology = {
"classes": [class_def("Person"), class_def("Organization")],
"properties": [
prop_def("fullName", domain="Person", required=True),
# No domain. The ontology never says which class this belongs to.
prop_def("sourceDocument", required=True),
],
}
if declare_namespace:
ontology["namespace"] = {"base_uri": ONTOLOGY_NS}
return ontology
SHAPES = [
pytest.param(True, True, id="namespace+uris"),
pytest.param(True, False, id="namespace-only"),
pytest.param(False, True, id="uris-only"),
]
# A Person with no fullName. This violates the shape the ontology does state.
VIOLATING_DATA = f"""
@prefix ex: <{ONTOLOGY_NS}> .
ex:alice a ex:Person .
"""
# A Person that satisfies every constraint the ontology actually declares.
CONFORMING_DATA = f"""
@prefix ex: <{ONTOLOGY_NS}> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
ex:bob a ex:Person ;
ex:fullName "Bob Smith"^^xsd:string .
"""
def _shapes_graph(ontology, **kwargs):
generator = SHACLGenerator(**kwargs)
graph = generator.generate(ontology)
shapes = Graph()
shapes.parse(data=generator.serialize(graph, "turtle"), format="turtle")
return shapes
def _validate(data_ttl, shapes_graph):
data = Graph()
data.parse(data=data_ttl, format="turtle")
conforms, _, text = pyshacl.validate(
data, shacl_graph=shapes_graph, inference="none", advanced=True
)
return conforms, text
@pytest.mark.parametrize("declare_namespace,carry_class_uris", SHAPES)
def test_target_class_names_a_class_the_data_can_instantiate(declare_namespace, carry_class_uris):
shapes = _shapes_graph(_ontology(
declare_namespace=declare_namespace, carry_class_uris=carry_class_uris
))
targets = {str(o) for o in shapes.objects(None, SH.targetClass)}
assert targets == {ONTOLOGY_NS + "Person", ONTOLOGY_NS + "Organization"}, targets
assert not any(t.startswith(SHAPES_NS) for t in targets), (
f"shapes still target their own shapes namespace: {targets}"
)
@pytest.mark.parametrize("declare_namespace,carry_class_uris", SHAPES)
def test_property_path_names_a_predicate_the_data_uses(declare_namespace, carry_class_uris):
shapes = _shapes_graph(_ontology(
declare_namespace=declare_namespace, carry_class_uris=carry_class_uris
))
paths = {str(o) for o in shapes.objects(None, SH.path)}
assert paths, "no sh:path emitted at all"
for path in paths:
assert path.startswith(ONTOLOGY_NS), path
@pytest.mark.parametrize("declare_namespace,carry_class_uris", SHAPES)
def test_a_real_violation_is_actually_reported(declare_namespace, carry_class_uris):
"""The killer case. Shapes that match nothing make pySHACL return conforms=True."""
shapes = _shapes_graph(_ontology(
declare_namespace=declare_namespace, carry_class_uris=carry_class_uris
))
conforms, text = _validate(VIOLATING_DATA, shapes)
assert not conforms, (
"a Person with no fullName was reported as conforming, which means the "
f"shapes matched no focus nodes:\n{text}"
)
assert "fullName" in text
@pytest.mark.parametrize("declare_namespace,carry_class_uris", SHAPES)
def test_conforming_data_still_conforms(declare_namespace, carry_class_uris):
shapes = _shapes_graph(_ontology(
declare_namespace=declare_namespace, carry_class_uris=carry_class_uris
))
conforms, text = _validate(CONFORMING_DATA, shapes)
assert conforms, text
def test_default_target_namespace_is_the_vocabulary_not_the_shapes_namespace():
"""With nothing declared anywhere, targets must not land in the shapes namespace."""
ontology = {
"classes": [{"name": "Person", "label": "Person"}],
"properties": [{"name": "fullName", "type": "datatype", "range": "string",
"domain": "Person", "required": True}],
}
shapes = _shapes_graph(ontology)
targets = {str(o) for o in shapes.objects(None, SH.targetClass)}
assert targets, "no sh:targetClass emitted at all"
for target in targets:
assert not target.startswith(SHAPES_NS), target
def test_shape_resources_are_distinct_from_the_classes_they_target():
shapes = _shapes_graph(_ontology(declare_namespace=True, carry_class_uris=True))
node_shapes = {str(s) for s in shapes.subjects(RDF.type, SH.NodeShape)}
targets = {str(o) for o in shapes.objects(None, SH.targetClass)}
assert node_shapes, "no node shapes emitted"
assert node_shapes.isdisjoint(targets), (
f"a shape and the class it targets are the same resource: {node_shapes & targets}"
)
# ── #1105 ────────────────────────────────────────────────────────────────────
def test_domainless_property_is_not_asserted_on_every_class():
"""minCount 1 on a domain-less property invalidates every instance of every class."""
generator = SHACLGenerator()
graph = generator.generate(_ontology(declare_namespace=True, carry_class_uris=True))
carriers = [
shape.target_class
for shape in graph.node_shapes
for prop in shape.property_shapes
if prop.path.endswith("sourceDocument")
]
assert carriers == [], f"a property with no declared domain was attached to {carriers}"
def test_domainless_property_does_not_invalidate_conforming_data():
shapes = _shapes_graph(_ontology(declare_namespace=True, carry_class_uris=True))
conforms, text = _validate(CONFORMING_DATA, shapes)
assert conforms, f"invented a constraint the ontology never declared:\n{text}"
assert "sourceDocument" not in text
def test_domainless_attachment_is_available_as_an_explicit_opt_in():
"""The old behaviour stays reachable for anyone who relied on it."""
generator = SHACLGenerator(attach_domainless_properties=True)
graph = generator.generate(_ontology(declare_namespace=True, carry_class_uris=True))
carriers = {
shape.target_class
for shape in graph.node_shapes
for prop in shape.property_shapes
if prop.path.endswith("sourceDocument")
}
assert len(carriers) == len(graph.node_shapes)
# ── Review findings on the first revision of this fix ────────────────────────
@pytest.mark.parametrize("fmt,parse_as", [
("turtle", "turtle"), ("n-triples", "nt"), ("json-ld", "json-ld"),
])
def test_every_format_targets_the_ontology_namespace(fmt, parse_as):
"""
The first revision fixed Turtle alone. JSON-LD and N-Triples went on
pasting names onto the shapes namespace, so two of the three formats still
produced shapes that matched nothing.
"""
generator = SHACLGenerator()
graph = generator.generate(_ontology(declare_namespace=False, carry_class_uris=True))
shapes = Graph()
shapes.parse(data=generator.serialize(graph, fmt), format=parse_as)
targets = {str(o) for o in shapes.objects(None, SH.targetClass)}
assert targets == {ONTOLOGY_NS + "Person", ONTOLOGY_NS + "Organization"}, targets
assert not any(t.startswith(SHAPES_NS) for t in targets), targets
@pytest.mark.parametrize("fmt,parse_as", [
("turtle", "turtle"), ("n-triples", "nt"), ("json-ld", "json-ld"),
])
def test_every_format_reports_a_real_violation(fmt, parse_as):
generator = SHACLGenerator()
graph = generator.generate(_ontology(declare_namespace=False, carry_class_uris=True))
shapes = Graph()
shapes.parse(data=generator.serialize(graph, fmt), format=parse_as)
conforms, text = _validate(VIOLATING_DATA, shapes)
assert not conforms, f"{fmt} shapes matched no focus nodes:\n{text}"
def test_a_property_sharing_a_class_name_keeps_its_own_iri():
"""One name-keyed map gave the property the class's IRI, so sh:path validated
the wrong predicate."""
ontology = {
"classes": [{"name": "Account", "uri": ONTOLOGY_NS + "Account"}],
"properties": [
{
"name": "Account",
"uri": ONTOLOGY_NS + "accountNumber",
"type": "datatype",
"range": "string",
"domain": "Account",
"required": True,
}
],
}
shapes = _shapes_graph(ontology)
paths = {str(o) for o in shapes.objects(None, SH.path)}
targets = {str(o) for o in shapes.objects(None, SH.targetClass)}
assert paths == {ONTOLOGY_NS + "accountNumber"}, paths
assert targets == {ONTOLOGY_NS + "Account"}, targets
def test_sh_class_resolves_to_the_class_namespace():
ontology = {
"classes": [
{"name": "Person", "uri": ONTOLOGY_NS + "Person"},
{"name": "Organization", "uri": ONTOLOGY_NS + "Organization"},
],
"properties": [
{
"name": "worksAt",
"uri": ONTOLOGY_NS + "worksAt",
"type": "object",
"range": "Organization",
"domain": "Person",
}
],
}
shapes = _shapes_graph(ontology)
classes = {str(o) for o in shapes.objects(None, SH["class"])}
assert classes == {ONTOLOGY_NS + "Organization"}, classes
def test_the_engine_forwards_the_new_options():
"""to_shacl passed them through generate(**options), which never reads them."""
from semantica.ontology.engine import OntologyEngine
ontology = _ontology(declare_namespace=False, carry_class_uris=False)
turtle = OntologyEngine().to_shacl(ontology, target_namespace="https://forwarded.example/ns#")
shapes = Graph()
shapes.parse(data=turtle, format="turtle")
targets = {str(o) for o in shapes.objects(None, SH.targetClass)}
assert all(t.startswith("https://forwarded.example/ns#") for t in targets), targets
attached = OntologyEngine().to_shacl(ontology, attach_domainless_properties=True)
assert "sourceDocument" in attached
default = OntologyEngine().to_shacl(ontology)
assert "sourceDocument" not in default
def test_the_opt_in_warns_rather_than_whispering(caplog):
import logging
with caplog.at_level(logging.WARNING):
SHACLGenerator(attach_domainless_properties=True).generate(
_ontology(declare_namespace=True, carry_class_uris=True)
)
messages = " ".join(record.getMessage() for record in caplog.records)
assert "sourceDocument" in messages
tests/utils/test_custom_method_can_refuse.py
+225
View File
@@ -0,0 +1,225 @@
"""
Regression tests for #1108.
Every module supporting custom methods wrapped the registered callable in a
bare `except Exception`, logged a warning, and continued into the built-in
implementation. That makes a registered method advisory: it can add behaviour,
but it cannot decline.
For a gate, a validator or a policy check, declining is the entire purpose.
The demonstration below is the one from the issue: a verifier rejects invalid
RDF and deletes the file, and the swallowed exception lets the default write it
straight back.
"""
import json
from pathlib import Path
import pytest
from semantica.export import methods as export_methods
from semantica.export.registry import method_registry
from semantica.utils.custom_methods import (
CUSTOM_METHOD_FELL_BACK,
call_custom_method,
)
class Refused(Exception):
"""Raised by a gate that declines to produce output."""
@pytest.fixture(autouse=True)
def _clean_registry():
method_registry.clear("rdf")
yield
method_registry.clear("rdf")
KG = {"entities": [{"id": "e1", "text": "Acme", "type": "ORG"}], "relationships": []}
def test_a_registered_gate_can_refuse(tmp_path):
"""The exception must reach the caller instead of being logged and dropped."""
def gate(data, file_path, **kwargs):
raise Refused("this graph does not pass validation")
method_registry.register("rdf", "gate", gate)
with pytest.raises(Refused):
export_methods.export_rdf(KG, str(tmp_path / "out.ttl"), method="gate")
def test_a_refusal_leaves_no_output_behind(tmp_path):
"""The issue's demonstration: the default used to write the file back."""
target = tmp_path / "out.ttl"
def gate(data, file_path, **kwargs):
Path(file_path).unlink(missing_ok=True)
raise Refused("rejected by the verifier")
method_registry.register("rdf", "gate", gate)
with pytest.raises(Refused):
export_methods.export_rdf(KG, str(target), method="gate")
assert not target.exists(), (
"the default implementation wrote the file the gate refused to produce"
)
def test_a_custom_method_that_succeeds_is_unaffected(tmp_path):
target = tmp_path / "out.ttl"
def writer(data, file_path, **kwargs):
Path(file_path).write_text("# written by the custom method\n")
return {"written_by": "custom"}
method_registry.register("rdf", "writer", writer)
result = export_methods.export_rdf(KG, str(target), method="writer")
assert result == {"written_by": "custom"}
assert target.read_text().startswith("# written by the custom method")
def test_the_old_behaviour_is_available_as_an_explicit_opt_in(tmp_path):
target = tmp_path / "out.ttl"
def gate(data, file_path, **kwargs):
raise Refused("rejected")
method_registry.register("rdf", "gate", gate)
export_methods.export_rdf(
KG, str(target), method="gate", fallback_on_custom_error=True
)
assert target.exists(), "opting in should still fall through to the default"
def test_the_reserved_keyword_is_never_forwarded():
"""`fallback_on_custom_error` is consumed by the policy, not by the method."""
seen = {}
def recorder(**kwargs):
seen.update(kwargs)
return "ok"
result = call_custom_method(
_NullLogger(), "recorder", recorder, alpha=1, fallback_on_custom_error=True
)
assert result == "ok"
assert seen == {"alpha": 1}
class _NullLogger:
def warning(self, *args, **kwargs):
self.last = args
def test_the_sentinel_is_returned_only_on_the_opt_in_path():
def boom():
raise Refused("no")
logger = _NullLogger()
assert call_custom_method(
logger, "boom", boom, fallback_on_custom_error=True
) is CUSTOM_METHOD_FELL_BACK
with pytest.raises(Refused):
call_custom_method(logger, "boom", boom)
def test_a_falsy_return_value_is_not_mistaken_for_a_failure():
"""`is not CUSTOM_METHOD_FELL_BACK` matters: None and 0 are real results."""
for value in (None, 0, "", False, []):
assert call_custom_method(_NullLogger(), "m", lambda: value) is value
@pytest.mark.parametrize(
"module_name",
["export", "ingest", "parse", "normalize", "embeddings", "kg"],
)
def test_no_module_still_swallows_custom_method_failures(module_name):
"""The swallow was repeated across six modules, not just the one filed."""
import semantica
# Read the file rather than import it: some of these modules pull in
# optional third-party dependencies that need not be installed to check
# that the swallow is gone.
source = (
Path(semantica.__file__).parent / module_name / "methods.py"
).read_text(encoding="utf-8")
assert "falling back to default" not in source, (
f"semantica/{module_name}/methods.py still swallows custom method failures"
)
# ── Review findings on the first revision of this fix ────────────────────────
def test_the_reserved_flag_never_reaches_the_default_implementation(monkeypatch, tmp_path):
"""
`**kwargs` unpacking builds a fresh dict inside the helper, so popping there
left the caller's own kwargs untouched and the flag was forwarded on to the
default path. The helper documents the flag as never forwarded, so that
promise was false for exactly the case the flag exists for.
"""
seen = {}
class Spy:
def __init__(self, **config):
seen.update(config)
def export(self, *args, **kwargs):
(tmp_path / "written").write_text("default ran")
monkeypatch.setattr(export_methods, "RDFExporter", Spy)
def gate(data, file_path, **kwargs):
raise Refused("rejected")
method_registry.register("rdf", "gate", gate)
export_methods.export_rdf(
KG, str(tmp_path / "out.ttl"), method="gate", fallback_on_custom_error=True
)
assert (tmp_path / "written").exists(), "the default path did not run"
assert "fallback_on_custom_error" not in seen, (
f"the reserved flag was forwarded to the default implementation: {seen}"
)
def test_the_reserved_flag_never_reaches_a_successful_custom_method(tmp_path):
seen = {}
def writer(data, file_path, **kwargs):
seen.update(kwargs)
return "ok"
method_registry.register("rdf", "writer", writer)
result = export_methods.export_rdf(
KG, str(tmp_path / "out.ttl"), method="writer", fallback_on_custom_error=True
)
assert result == "ok"
assert "fallback_on_custom_error" not in seen, seen
@pytest.mark.parametrize(
"module_name", ["export", "ingest", "parse", "normalize", "embeddings", "kg"],
)
def test_every_site_consumes_the_flag_before_forwarding(module_name):
"""A site that forgets the pop reintroduces the leak silently."""
import semantica
source = (
Path(semantica.__file__).parent / module_name / "methods.py"
).read_text(encoding="utf-8")
calls = source.count("result = call_custom_method(")
pops = source.count('.pop("fallback_on_custom_error", False)')
assert calls == pops, (
f"semantica/{module_name}/methods.py has {calls} call site(s) but "
f"{pops} consume the reserved flag"
)