mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
fix(export): carry metadata through every RDF serialization (#1154)
convert_kg_to_rdf copies metadata into the RDF-ready dictionary at
rdf_exporter.py:302 and no serializer has ever read it back out. Turtle,
N-Triples, RDF/XML and RDFExporter's JSON-LD each write an entity's id,
type, text and confidence and nothing else, so an entity keeps its
confidence score and loses what produced it. JSONExporter's json-ld path
keeps the same fields, which is how one knowledge graph exported two ways
carried the user's data through one exporter and none through the other.
Measured on e3405ebc with an entity carrying four metadata keys: 3 triples
per format, 0 of them metadata. With this change: 7 triples per format,
4 of them metadata, and the same four in all four formats.
The keys Semantica itself writes are mapped to declared terms in
DEFAULT_METADATA_TERMS and declared in semantica-ns.ttl. A key the caller
supplied is not: which namespace an arbitrary key belongs in is #1146, and
that issue is open on the maintainer's modelling call, so the exporter
warns and skips rather than inventing an IRI. Callers who already know the
answer pass metadata_terms={key: iri}.
Two keys cannot keep their own name. sem:source is already the
ObjectProperty holding the subject of a reified relationship, so the Neo4j
loader's "source" is written as sem:sourceSystem and its "uri" as
sem:sourceUri, the one term whose value is a node rather than a literal.
sem:builtAt and sem:snapshotAt have range xsd:string, not xsd:dateTime.
GraphBuilder stamps with a timezone-naive datetime.now(), and #1114 is the
demonstration of what typing such a value as xsd:dateTime costs: a
timezone-qualified SPARQL filter over it silently drops the row. #1121
swept export and provenance and deliberately left kg/ alone.
Graph-level metadata is written only when the caller names the graph with
graph_uri, because this serializer has never minted a document node and
#1147 is where that default belongs once it lands.
The lexical form and datatype of a value are chosen once, in
_typed_literal_parts, so the four serializers cannot come to disagree
about them the way they disagreed about confidence in #1100. The JSON-LD
path writes explicit @value/@type rather than JSON's native numbers,
which would have made an integer xsd:double there and xsd:integer
everywhere else.
21 tests, asserting on the parsed graph in all four formats. Output is
unchanged when no metadata is present. Full-suite failure set is identical
to the parent commit: 512 = 512.
This commit is contained in:
@@ -69,6 +69,187 @@ def mint_relationship_iri(index: int, source: Any, target: Any) -> str:
|
|||||||
return f"{SEMANTICA_NS}rel_{index}_{digest}"
|
return f"{SEMANTICA_NS}rel_{index}_{digest}"
|
||||||
|
|
||||||
|
|
||||||
|
#: The metadata keys Semantica itself produces, and the terms they are written
|
||||||
|
#: as. GraphBuilder.build_graph writes the first five, create_snapshot writes
|
||||||
|
#: snapshot_time, and load_from_neo4j writes source / uri / database. These are
|
||||||
|
#: Semantica's own vocabulary, so they are minted in the declared namespace and
|
||||||
|
#: declared in semantica-ns.ttl.
|
||||||
|
#:
|
||||||
|
#: A key the caller supplied is a different matter. Which namespace an
|
||||||
|
#: arbitrary metadata key belongs in is issue #1146, and until that is settled
|
||||||
|
#: the exporter refuses to guess: it warns and skips, and a caller who already
|
||||||
|
#: knows the answer passes ``metadata_terms``.
|
||||||
|
#:
|
||||||
|
#: The map is key -> term rather than key -> namespace because two of the keys
|
||||||
|
#: cannot keep their own name. ``source`` on a graph loaded from Neo4j is the
|
||||||
|
#: system it came from, while sem:source is already the ObjectProperty holding
|
||||||
|
#: the subject of a reified relationship; reusing it would put a string where
|
||||||
|
#: an entity belongs.
|
||||||
|
DEFAULT_METADATA_TERMS: Dict[str, str] = {
|
||||||
|
"num_entities": f"{SEMANTICA_NS}numEntities",
|
||||||
|
"num_relationships": f"{SEMANTICA_NS}numRelationships",
|
||||||
|
"temporal_enabled": f"{SEMANTICA_NS}temporalEnabled",
|
||||||
|
"entity_resolution_applied": f"{SEMANTICA_NS}entityResolutionApplied",
|
||||||
|
"timestamp": f"{SEMANTICA_NS}builtAt",
|
||||||
|
"snapshot_time": f"{SEMANTICA_NS}snapshotAt",
|
||||||
|
"source": f"{SEMANTICA_NS}sourceSystem",
|
||||||
|
"uri": f"{SEMANTICA_NS}sourceUri",
|
||||||
|
"database": f"{SEMANTICA_NS}sourceDatabase",
|
||||||
|
}
|
||||||
|
|
||||||
|
#: Terms whose value is a node rather than a string. Everything else stays a
|
||||||
|
#: literal: a metadata value that merely looks like a URL is not thereby a
|
||||||
|
#: reference to one.
|
||||||
|
IRI_VALUED_METADATA_TERMS: Set[str] = {f"{SEMANTICA_NS}sourceUri"}
|
||||||
|
|
||||||
|
_XSD_NS = "http://www.w3.org/2001/XMLSchema#"
|
||||||
|
|
||||||
|
|
||||||
|
def _escape_literal(value: str) -> str:
|
||||||
|
"""Escape a string for a Turtle or N-Triples quoted literal."""
|
||||||
|
return (
|
||||||
|
value.replace("\\", "\\\\")
|
||||||
|
.replace('"', '\\"')
|
||||||
|
.replace("\n", "\\n")
|
||||||
|
.replace("\r", "\\r")
|
||||||
|
.replace("\t", "\\t")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _escape_xml(value: str) -> str:
|
||||||
|
return value.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||||
|
|
||||||
|
|
||||||
|
def _split_iri(iri: str) -> Optional[tuple]:
|
||||||
|
"""Split an IRI into (namespace, local name) for RDF/XML's QName syntax."""
|
||||||
|
for sep in ("#", "/"):
|
||||||
|
index = iri.rfind(sep)
|
||||||
|
if index != -1 and index + 1 < len(iri):
|
||||||
|
local = iri[index + 1 :]
|
||||||
|
if local and not local[0].isdigit():
|
||||||
|
return iri[: index + 1], local
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _metadata_statements(
|
||||||
|
metadata: Any,
|
||||||
|
terms: Dict[str, str],
|
||||||
|
logger: Any,
|
||||||
|
) -> List[tuple]:
|
||||||
|
"""Resolve a metadata mapping to a list of (term IRI, value) pairs.
|
||||||
|
|
||||||
|
A key with no term is skipped and reported. Silence is the defect this
|
||||||
|
fixes, so an unmapped key must be louder than a mapped one, not quieter.
|
||||||
|
"""
|
||||||
|
if not isinstance(metadata, dict):
|
||||||
|
return []
|
||||||
|
|
||||||
|
statements: List[tuple] = []
|
||||||
|
for key, value in metadata.items():
|
||||||
|
term = terms.get(key)
|
||||||
|
if term is None:
|
||||||
|
logger.warning(
|
||||||
|
"Metadata key %r has no term and was not exported. Which "
|
||||||
|
"namespace a caller-supplied key belongs in is issue #1146; "
|
||||||
|
"pass metadata_terms={%r: '<iri>'} to export it now.",
|
||||||
|
key,
|
||||||
|
key,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
if isinstance(value, (dict, list, tuple, set)):
|
||||||
|
logger.warning(
|
||||||
|
"Metadata key %r holds a %s, which has no modelled RDF shape "
|
||||||
|
"yet, and was not exported.",
|
||||||
|
key,
|
||||||
|
type(value).__name__,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
statements.append((term, value))
|
||||||
|
return statements
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_metadata_terms(overrides: Optional[Dict[str, str]]) -> Dict[str, str]:
|
||||||
|
if not overrides:
|
||||||
|
return DEFAULT_METADATA_TERMS
|
||||||
|
return {**DEFAULT_METADATA_TERMS, **overrides}
|
||||||
|
|
||||||
|
|
||||||
|
def _typed_literal_parts(term: str, value: Any) -> tuple:
|
||||||
|
"""Return (kind, lexical, datatype) for one metadata value.
|
||||||
|
|
||||||
|
kind is "iri" or "literal". The lexical form and datatype are chosen once,
|
||||||
|
here, so that the four serializers cannot disagree about them the way they
|
||||||
|
disagreed about confidence in #1100.
|
||||||
|
"""
|
||||||
|
if term in IRI_VALUED_METADATA_TERMS and isinstance(value, str):
|
||||||
|
return "iri", value, None
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return "literal", "true" if value else "false", f"{_XSD_NS}boolean"
|
||||||
|
if isinstance(value, int):
|
||||||
|
return "literal", str(value), f"{_XSD_NS}integer"
|
||||||
|
if isinstance(value, float):
|
||||||
|
return "literal", repr(value), f"{_XSD_NS}decimal"
|
||||||
|
return "literal", str(value), None
|
||||||
|
|
||||||
|
|
||||||
|
def _turtle_object(term: str, value: Any) -> str:
|
||||||
|
kind, lexical, datatype = _typed_literal_parts(term, value)
|
||||||
|
if kind == "iri":
|
||||||
|
return f"<{lexical}>"
|
||||||
|
if datatype is None:
|
||||||
|
return f'"{_escape_literal(lexical)}"'
|
||||||
|
return f'"{lexical}"^^<{datatype}>'
|
||||||
|
|
||||||
|
|
||||||
|
def _turtle_metadata_clauses(statements: List[tuple]) -> List[str]:
|
||||||
|
return [f"<{term}> {_turtle_object(term, value)}" for term, value in statements]
|
||||||
|
|
||||||
|
|
||||||
|
def _ntriples_metadata_lines(subject: str, statements: List[tuple]) -> List[str]:
|
||||||
|
return [
|
||||||
|
f"<{subject}> <{term}> {_turtle_object(term, value)} ."
|
||||||
|
for term, value in statements
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _rdfxml_metadata_lines(statements: List[tuple], indent: str) -> List[str]:
|
||||||
|
"""RDF/XML needs a QName, so an unprefixed term declares its own prefix."""
|
||||||
|
lines: List[str] = []
|
||||||
|
for position, (term, value) in enumerate(statements):
|
||||||
|
split = _split_iri(term)
|
||||||
|
if split is None:
|
||||||
|
continue
|
||||||
|
namespace, local = split
|
||||||
|
kind, lexical, datatype = _typed_literal_parts(term, value)
|
||||||
|
prefix = f"md{position}"
|
||||||
|
opening = f'{indent}<{prefix}:{local} xmlns:{prefix}="{_escape_xml(namespace)}"'
|
||||||
|
if kind == "iri":
|
||||||
|
lines.append(f'{opening} rdf:resource="{_escape_xml(lexical)}"/>')
|
||||||
|
continue
|
||||||
|
if datatype is not None:
|
||||||
|
opening += f' rdf:datatype="{_escape_xml(datatype)}"'
|
||||||
|
lines.append(f"{opening}>{_escape_xml(lexical)}</{prefix}:{local}>")
|
||||||
|
return lines
|
||||||
|
|
||||||
|
|
||||||
|
def _jsonld_metadata_entries(statements: List[tuple]) -> Dict[str, Any]:
|
||||||
|
"""Absolute IRIs as keys, and explicit @value/@type rather than JSON's own
|
||||||
|
types: JSON's number is xsd:double, which would make the JSON-LD export
|
||||||
|
disagree with the other three about the datatype of an integer."""
|
||||||
|
entries: Dict[str, Any] = {}
|
||||||
|
for term, value in statements:
|
||||||
|
kind, lexical, datatype = _typed_literal_parts(term, value)
|
||||||
|
if kind == "iri":
|
||||||
|
entries[term] = {"@id": lexical}
|
||||||
|
elif datatype is None:
|
||||||
|
entries[term] = lexical
|
||||||
|
else:
|
||||||
|
entries[term] = {"@value": lexical, "@type": datatype}
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
class NamespaceManager:
|
class NamespaceManager:
|
||||||
"""
|
"""
|
||||||
RDF namespace management engine.
|
RDF namespace management engine.
|
||||||
@@ -364,6 +545,8 @@ class RDFSerializer:
|
|||||||
"""
|
"""
|
||||||
include_temporal: bool = options.pop("include_temporal", False)
|
include_temporal: bool = options.pop("include_temporal", False)
|
||||||
time_axis: str = options.pop("time_axis", "valid")
|
time_axis: str = options.pop("time_axis", "valid")
|
||||||
|
metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None))
|
||||||
|
graph_uri: Optional[str] = options.pop("graph_uri", None)
|
||||||
|
|
||||||
lines = []
|
lines = []
|
||||||
|
|
||||||
@@ -397,9 +580,23 @@ class RDFSerializer:
|
|||||||
text = entity.get("text") or entity.get("label", "")
|
text = entity.get("text") or entity.get("label", "")
|
||||||
confidence = entity.get("confidence", 1.0)
|
confidence = entity.get("confidence", 1.0)
|
||||||
|
|
||||||
lines.append(f"<{entity_id}> a <{entity_type}> ;")
|
clauses = [
|
||||||
lines.append(f' semantica:text "{text}" ;')
|
f"a <{entity_type}>",
|
||||||
lines.append(f" semantica:confidence {confidence} .")
|
f'semantica:text "{text}"',
|
||||||
|
f"semantica:confidence {confidence}",
|
||||||
|
]
|
||||||
|
clauses.extend(
|
||||||
|
_turtle_metadata_clauses(
|
||||||
|
_metadata_statements(
|
||||||
|
entity.get("metadata"), metadata_terms, self.logger
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
lines.append(f"<{entity_id}> {clauses[0]} ;")
|
||||||
|
for clause in clauses[1:-1]:
|
||||||
|
lines.append(f" {clause} ;")
|
||||||
|
lines.append(f" {clauses[-1]} .")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# Convert relationships to RDF triplets
|
# Convert relationships to RDF triplets
|
||||||
@@ -416,6 +613,30 @@ class RDFSerializer:
|
|||||||
if owl_lines:
|
if owl_lines:
|
||||||
lines.extend(owl_lines)
|
lines.extend(owl_lines)
|
||||||
|
|
||||||
|
# Graph-level metadata needs a subject, and this serializer has never
|
||||||
|
# minted a document node. Rather than invent one here, it is written
|
||||||
|
# only when the caller names the graph; issue #1147 is where the
|
||||||
|
# default subject comes from once that lands.
|
||||||
|
graph_clauses = (
|
||||||
|
_turtle_metadata_clauses(
|
||||||
|
_metadata_statements(
|
||||||
|
rdf_data.get("metadata"), metadata_terms, self.logger
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if graph_uri
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
if graph_clauses:
|
||||||
|
lines.append("")
|
||||||
|
lines.append(
|
||||||
|
f"<{graph_uri}> {graph_clauses[0]} "
|
||||||
|
+ (";" if len(graph_clauses) > 1 else ".")
|
||||||
|
)
|
||||||
|
for clause in graph_clauses[1:-1]:
|
||||||
|
lines.append(f" {clause} ;")
|
||||||
|
if len(graph_clauses) > 1:
|
||||||
|
lines.append(f" {graph_clauses[-1]} .")
|
||||||
|
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
def _owl_time_triples_for_rel(
|
def _owl_time_triples_for_rel(
|
||||||
@@ -510,6 +731,9 @@ class RDFSerializer:
|
|||||||
... }
|
... }
|
||||||
>>> rdfxml = serializer.serialize_to_rdfxml(rdf_data)
|
>>> rdfxml = serializer.serialize_to_rdfxml(rdf_data)
|
||||||
"""
|
"""
|
||||||
|
metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None))
|
||||||
|
graph_uri: Optional[str] = options.pop("graph_uri", None)
|
||||||
|
|
||||||
lines = ['<?xml version="1.0" encoding="UTF-8"?>']
|
lines = ['<?xml version="1.0" encoding="UTF-8"?>']
|
||||||
lines.append('<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"')
|
lines.append('<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"')
|
||||||
lines.append(' xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"')
|
lines.append(' xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"')
|
||||||
@@ -536,6 +760,14 @@ class RDFSerializer:
|
|||||||
lines.append(
|
lines.append(
|
||||||
f" <semantica:confidence>{confidence}</semantica:confidence>"
|
f" <semantica:confidence>{confidence}</semantica:confidence>"
|
||||||
)
|
)
|
||||||
|
lines.extend(
|
||||||
|
_rdfxml_metadata_lines(
|
||||||
|
_metadata_statements(
|
||||||
|
entity.get("metadata"), metadata_terms, self.logger
|
||||||
|
),
|
||||||
|
" ",
|
||||||
|
)
|
||||||
|
)
|
||||||
lines.append(" </rdf:Description>")
|
lines.append(" </rdf:Description>")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
@@ -552,6 +784,22 @@ class RDFSerializer:
|
|||||||
lines.append(" </rdf:Description>")
|
lines.append(" </rdf:Description>")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
|
graph_lines = (
|
||||||
|
_rdfxml_metadata_lines(
|
||||||
|
_metadata_statements(
|
||||||
|
rdf_data.get("metadata"), metadata_terms, self.logger
|
||||||
|
),
|
||||||
|
" ",
|
||||||
|
)
|
||||||
|
if graph_uri
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
if graph_lines:
|
||||||
|
lines.append(f' <rdf:Description rdf:about="{_escape_xml(graph_uri)}">')
|
||||||
|
lines.extend(graph_lines)
|
||||||
|
lines.append(" </rdf:Description>")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
lines.append("</rdf:RDF>")
|
lines.append("</rdf:RDF>")
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
@@ -582,6 +830,9 @@ class RDFSerializer:
|
|||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None))
|
||||||
|
graph_uri: Optional[str] = options.pop("graph_uri", None)
|
||||||
|
|
||||||
# Initialize JSON-LD structure with context
|
# Initialize JSON-LD structure with context
|
||||||
jsonld = {
|
jsonld = {
|
||||||
"@context": {
|
"@context": {
|
||||||
@@ -608,14 +859,20 @@ class RDFSerializer:
|
|||||||
# and was dropped in full by a JSON-LD parser, silently.
|
# and was dropped in full by a JSON-LD parser, silently.
|
||||||
entity_id = entity.get("id") or mint_entity_iri(entity.get("text", ""))
|
entity_id = entity.get("id") or mint_entity_iri(entity.get("text", ""))
|
||||||
|
|
||||||
jsonld["@graph"].append(
|
node = {
|
||||||
{
|
"@id": entity_id,
|
||||||
"@id": entity_id,
|
"@type": entity.get("type", "semantica:Entity"),
|
||||||
"@type": entity.get("type", "semantica:Entity"),
|
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
"semantica:confidence": entity.get("confidence", 1.0),
|
||||||
"semantica:confidence": entity.get("confidence", 1.0),
|
}
|
||||||
}
|
node.update(
|
||||||
|
_jsonld_metadata_entries(
|
||||||
|
_metadata_statements(
|
||||||
|
entity.get("metadata"), metadata_terms, self.logger
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
jsonld["@graph"].append(node)
|
||||||
|
|
||||||
# Convert relationships to JSON-LD
|
# Convert relationships to JSON-LD
|
||||||
relationships = rdf_data.get("relationships", [])
|
relationships = rdf_data.get("relationships", [])
|
||||||
@@ -639,6 +896,18 @@ class RDFSerializer:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
graph_entries = (
|
||||||
|
_jsonld_metadata_entries(
|
||||||
|
_metadata_statements(
|
||||||
|
rdf_data.get("metadata"), metadata_terms, self.logger
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if graph_uri
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
if graph_entries:
|
||||||
|
jsonld["@graph"].append({"@id": graph_uri, **graph_entries})
|
||||||
|
|
||||||
return json.dumps(jsonld, indent=2, ensure_ascii=False)
|
return json.dumps(jsonld, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
def serialize_to_ntriples(self, rdf_data: Dict[str, Any], **options) -> str:
|
def serialize_to_ntriples(self, rdf_data: Dict[str, Any], **options) -> str:
|
||||||
@@ -655,6 +924,9 @@ class RDFSerializer:
|
|||||||
Returns:
|
Returns:
|
||||||
String containing N-Triples serialization
|
String containing N-Triples serialization
|
||||||
"""
|
"""
|
||||||
|
metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None))
|
||||||
|
graph_uri: Optional[str] = options.pop("graph_uri", None)
|
||||||
|
|
||||||
lines = []
|
lines = []
|
||||||
|
|
||||||
def expand_uri(uri: str) -> str:
|
def expand_uri(uri: str) -> str:
|
||||||
@@ -704,6 +976,15 @@ class RDFSerializer:
|
|||||||
f'{subject} {expand_uri("semantica:confidence")} "{confidence}"^^<http://www.w3.org/2001/XMLSchema#float> .'
|
f'{subject} {expand_uri("semantica:confidence")} "{confidence}"^^<http://www.w3.org/2001/XMLSchema#float> .'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
lines.extend(
|
||||||
|
_ntriples_metadata_lines(
|
||||||
|
subject.strip("<>"),
|
||||||
|
_metadata_statements(
|
||||||
|
entity.get("metadata"), metadata_terms, self.logger
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
# Convert relationships
|
# Convert relationships
|
||||||
relationships = rdf_data.get("relationships", [])
|
relationships = rdf_data.get("relationships", [])
|
||||||
for rel in relationships:
|
for rel in relationships:
|
||||||
@@ -716,6 +997,16 @@ class RDFSerializer:
|
|||||||
f"{expand_uri(source_id)} {expand_uri(rel_type)} {expand_uri(target_id)} ."
|
f"{expand_uri(source_id)} {expand_uri(rel_type)} {expand_uri(target_id)} ."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if graph_uri:
|
||||||
|
lines.extend(
|
||||||
|
_ntriples_metadata_lines(
|
||||||
|
graph_uri,
|
||||||
|
_metadata_statements(
|
||||||
|
rdf_data.get("metadata"), metadata_terms, self.logger
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -134,6 +134,90 @@ JSONExporter.export_to_jsonld in export/json_exporter.py.""" ;
|
|||||||
rdfs:range xsd:string ;
|
rdfs:range xsd:string ;
|
||||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
# ── Metadata carried through from the graph builder ──────────────────────────
|
||||||
|
#
|
||||||
|
# The keys GraphBuilder and the Neo4j loader write into "metadata". Declared
|
||||||
|
# here because the RDF serializers emit them (#1154); a caller-supplied key is
|
||||||
|
# not declared here and is not emitted, because which namespace it belongs in
|
||||||
|
# is #1146.
|
||||||
|
|
||||||
|
sem:numEntities a owl:DatatypeProperty ;
|
||||||
|
rdfs:label "number of entities" ;
|
||||||
|
rdfs:comment """Count of entities in the graph as built, from
|
||||||
|
GraphBuilder.build_graph. A count of what was built, not a constraint on what
|
||||||
|
the graph contains: an export filtered after the fact will disagree with it.""" ;
|
||||||
|
rdfs:range xsd:integer ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
sem:numRelationships a owl:DatatypeProperty ;
|
||||||
|
rdfs:label "number of relationships" ;
|
||||||
|
rdfs:comment "Count of relationships in the graph as built." ;
|
||||||
|
rdfs:range xsd:integer ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
sem:temporalEnabled a owl:DatatypeProperty ;
|
||||||
|
rdfs:label "temporal enabled" ;
|
||||||
|
rdfs:comment """True when the builder was configured to track valid time.
|
||||||
|
False does not mean the graph is untimed; it means no temporal bounds were
|
||||||
|
recorded for it.""" ;
|
||||||
|
rdfs:range xsd:boolean ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
sem:entityResolutionApplied a owl:DatatypeProperty ;
|
||||||
|
rdfs:label "entity resolution applied" ;
|
||||||
|
rdfs:comment """True when a resolver ran over the extracted entities, so a
|
||||||
|
consumer knows whether two nodes with the same surface text were ever
|
||||||
|
considered for merging.""" ;
|
||||||
|
rdfs:range xsd:boolean ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
sem:builtAt a owl:DatatypeProperty ;
|
||||||
|
rdfs:label "built at" ;
|
||||||
|
rdfs:comment """When the graph was built, as GraphBuilder recorded it.
|
||||||
|
|
||||||
|
The range is xsd:string, deliberately, and not xsd:dateTime. GraphBuilder
|
||||||
|
stamps with a timezone-naive datetime.now(), and #1114 is the demonstration of
|
||||||
|
what typing such a value as xsd:dateTime costs: a timezone-qualified SPARQL
|
||||||
|
filter over it raises an indeterminate comparison and silently drops the row.
|
||||||
|
#1121 swept the export and provenance modules to an explicit UTC offset and
|
||||||
|
deliberately left kg/ alone, because the context and vector-store modules
|
||||||
|
compare against naive values already on disk. Until that sweep reaches
|
||||||
|
GraphBuilder this value is a string that looks like a timestamp, and saying so
|
||||||
|
is more useful than a type that invites arithmetic it cannot support.""" ;
|
||||||
|
rdfs:range xsd:string ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
sem:snapshotAt a owl:DatatypeProperty ;
|
||||||
|
rdfs:label "snapshot at" ;
|
||||||
|
rdfs:comment """The point in time a snapshot represents, from
|
||||||
|
GraphBuilder.create_snapshot. A string for the same reason as sem:builtAt.""" ;
|
||||||
|
rdfs:range xsd:string ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
sem:sourceSystem a owl:DatatypeProperty ;
|
||||||
|
rdfs:label "source system" ;
|
||||||
|
rdfs:comment """The system a graph was loaded from, currently the literal
|
||||||
|
"neo4j" written by GraphBuilder.load_from_neo4j.
|
||||||
|
|
||||||
|
Named sourceSystem rather than source because sem:source is already the
|
||||||
|
ObjectProperty carrying the subject of a reified relationship. The metadata key
|
||||||
|
is still "source"; the exporter maps the key to this term.""" ;
|
||||||
|
rdfs:range xsd:string ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
sem:sourceUri a owl:ObjectProperty ;
|
||||||
|
rdfs:label "source URI" ;
|
||||||
|
rdfs:comment """The address of the system a graph was loaded from. The one
|
||||||
|
metadata term whose value is a node rather than a literal, because it names a
|
||||||
|
thing rather than describing one.""" ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
|
sem:sourceDatabase a owl:DatatypeProperty ;
|
||||||
|
rdfs:label "source database" ;
|
||||||
|
rdfs:comment "The database within the source system a graph was loaded from." ;
|
||||||
|
rdfs:range xsd:string ;
|
||||||
|
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||||
|
|
||||||
# ── Temporal term (OWL-Time export) ──────────────────────────────────────────
|
# ── Temporal term (OWL-Time export) ──────────────────────────────────────────
|
||||||
|
|
||||||
sem:openEndedInterval a owl:DatatypeProperty ;
|
sem:openEndedInterval a owl:DatatypeProperty ;
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
"""Metadata must survive serialization (issue #1154).
|
||||||
|
|
||||||
|
``convert_kg_to_rdf`` copies ``metadata`` into the RDF-ready dictionary at
|
||||||
|
rdf_exporter.py:302, and no serializer has ever read it back out. Turtle,
|
||||||
|
N-Triples, RDF/XML and RDFExporter's JSON-LD all write the entity's id, type,
|
||||||
|
text and confidence, and none of them writes a single metadata statement, so an
|
||||||
|
entity keeps its confidence score and loses what produced it: the source
|
||||||
|
document, the page, the extractor, the reviewer. JSONExporter's json-ld path
|
||||||
|
keeps all of them, which is how the same knowledge graph exported two ways came
|
||||||
|
to carry ten triples of user data through one exporter and none through the
|
||||||
|
other.
|
||||||
|
|
||||||
|
The keys Semantica itself produces (GraphBuilder writes num_entities,
|
||||||
|
num_relationships, temporal_enabled, timestamp and entity_resolution_applied;
|
||||||
|
the Neo4j loader writes source, uri and database) are Semantica's own
|
||||||
|
vocabulary, so they are minted in the declared namespace and declared in
|
||||||
|
semantica-ns.ttl. Keys the caller supplied are not: which namespace those
|
||||||
|
belong in is issue #1146, and until that is settled the exporter refuses to
|
||||||
|
guess rather than inventing an IRI, warns, and takes an explicit
|
||||||
|
``metadata_terms`` mapping from any caller who already knows the answer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from rdflib import Graph, Literal, URIRef
|
||||||
|
from rdflib.namespace import XSD
|
||||||
|
|
||||||
|
from semantica.export.rdf_exporter import (
|
||||||
|
DEFAULT_METADATA_TERMS,
|
||||||
|
RDFSerializer,
|
||||||
|
SEMANTICA_NS,
|
||||||
|
mint_entity_iri,
|
||||||
|
)
|
||||||
|
|
||||||
|
ENTITY_IRI = "https://example.org/e1"
|
||||||
|
|
||||||
|
# The provenance fields the issue names, plus one key Semantica itself writes.
|
||||||
|
GRAPH_WITH_METADATA = {
|
||||||
|
"entities": [
|
||||||
|
{
|
||||||
|
"id": ENTITY_IRI,
|
||||||
|
"type": "https://example.org/Org",
|
||||||
|
"text": "Acme Corp",
|
||||||
|
"confidence": 0.91,
|
||||||
|
"metadata": {"num_entities": 1, "temporal_enabled": True},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"relationships": [],
|
||||||
|
"metadata": {
|
||||||
|
"num_entities": 1,
|
||||||
|
"num_relationships": 0,
|
||||||
|
"temporal_enabled": False,
|
||||||
|
"entity_resolution_applied": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
NUM_ENTITIES = URIRef(f"{SEMANTICA_NS}numEntities")
|
||||||
|
TEMPORAL_ENABLED = URIRef(f"{SEMANTICA_NS}temporalEnabled")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse(text: str, fmt: str) -> Graph:
|
||||||
|
"""Assert on the parsed graph, never on the serialized text."""
|
||||||
|
g = Graph()
|
||||||
|
g.parse(data=text, format=fmt)
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize(serializer: RDFSerializer, fmt: str, data, **options) -> Graph:
|
||||||
|
method, parse_as = {
|
||||||
|
"turtle": (serializer.serialize_to_turtle, "turtle"),
|
||||||
|
"ntriples": (serializer.serialize_to_ntriples, "nt"),
|
||||||
|
"rdfxml": (serializer.serialize_to_rdfxml, "xml"),
|
||||||
|
"jsonld": (serializer.serialize_to_jsonld, "json-ld"),
|
||||||
|
}[fmt]
|
||||||
|
return _parse(method(data, **options), parse_as)
|
||||||
|
|
||||||
|
|
||||||
|
FORMATS = ["turtle", "ntriples", "rdfxml", "jsonld"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fmt", FORMATS)
|
||||||
|
def test_entity_metadata_reaches_every_serialization(fmt):
|
||||||
|
"""The headline defect: the statement is absent from all four formats."""
|
||||||
|
g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA)
|
||||||
|
assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(1)) in g
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fmt", FORMATS)
|
||||||
|
def test_entity_metadata_booleans_keep_their_datatype(fmt):
|
||||||
|
g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA)
|
||||||
|
assert (URIRef(ENTITY_IRI), TEMPORAL_ENABLED, Literal(True)) in g
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_format_writes_the_same_metadata_triples():
|
||||||
|
"""A value must not change datatype with the serializer, as #1100 found."""
|
||||||
|
per_format = {}
|
||||||
|
for fmt in FORMATS:
|
||||||
|
g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA)
|
||||||
|
per_format[fmt] = {
|
||||||
|
(p, o) for s, p, o in g if str(p).startswith(SEMANTICA_NS) and "numEntities" in str(p)
|
||||||
|
}
|
||||||
|
assert len(set(map(frozenset, per_format.values()))) == 1, per_format
|
||||||
|
|
||||||
|
|
||||||
|
def test_graph_metadata_needs_a_subject_the_caller_named():
|
||||||
|
"""Graph-level metadata hangs off graph_uri; #1147 owns the default."""
|
||||||
|
doc = URIRef("https://example.org/graph/1")
|
||||||
|
g = _serialize(
|
||||||
|
RDFSerializer(),
|
||||||
|
"turtle",
|
||||||
|
GRAPH_WITH_METADATA,
|
||||||
|
graph_uri=str(doc),
|
||||||
|
)
|
||||||
|
assert (doc, NUM_ENTITIES, Literal(1)) in g
|
||||||
|
assert (doc, URIRef(f"{SEMANTICA_NS}entityResolutionApplied"), Literal(True)) in g
|
||||||
|
|
||||||
|
|
||||||
|
def test_graph_metadata_is_not_invented_without_a_subject():
|
||||||
|
g = _serialize(RDFSerializer(), "turtle", GRAPH_WITH_METADATA)
|
||||||
|
assert not list(g.subjects(NUM_ENTITIES, Literal(0)))
|
||||||
|
# the entity keeps its own metadata; only the graph-level block waits
|
||||||
|
assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(1)) in g
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unknown_key_is_refused_out_loud_not_dropped_in_silence(caplog):
|
||||||
|
"""#1146 owns which namespace a caller's key belongs in. Until then: warn."""
|
||||||
|
data = {
|
||||||
|
"entities": [
|
||||||
|
{"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}}
|
||||||
|
],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
with caplog.at_level("WARNING"):
|
||||||
|
g = _serialize(RDFSerializer(), "turtle", data)
|
||||||
|
assert not any("reviewed_by" in str(p) for p in g.predicates())
|
||||||
|
assert any("reviewed_by" in r.getMessage() for r in caplog.records)
|
||||||
|
assert any("1146" in r.getMessage() for r in caplog.records)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fmt", FORMATS)
|
||||||
|
def test_a_caller_who_knows_the_answer_can_supply_the_term(fmt):
|
||||||
|
data = {
|
||||||
|
"entities": [
|
||||||
|
{"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}}
|
||||||
|
],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
terms = {"reviewed_by": "http://purl.org/dc/terms/creator"}
|
||||||
|
g = _serialize(RDFSerializer(), fmt, data, metadata_terms=terms)
|
||||||
|
assert (
|
||||||
|
URIRef(ENTITY_IRI),
|
||||||
|
URIRef("http://purl.org/dc/terms/creator"),
|
||||||
|
Literal("fabio"),
|
||||||
|
) in g
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_literal_with_a_quote_or_newline_still_parses():
|
||||||
|
"""Metadata is user text; #1098 is the same class of defect one field over."""
|
||||||
|
data = {
|
||||||
|
"entities": [
|
||||||
|
{
|
||||||
|
"id": ENTITY_IRI,
|
||||||
|
"text": "Acme",
|
||||||
|
"metadata": {"source": 'the "Q3" report\nsecond line'},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
for fmt in FORMATS:
|
||||||
|
g = _serialize(RDFSerializer(), fmt, data)
|
||||||
|
assert (
|
||||||
|
URIRef(ENTITY_IRI),
|
||||||
|
URIRef(f"{SEMANTICA_NS}sourceSystem"),
|
||||||
|
Literal('the "Q3" report\nsecond line'),
|
||||||
|
) in g
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_iri_valued_key_is_written_as_a_node_not_a_string():
|
||||||
|
"""The Neo4j loader's ``uri`` key. Note the term is sem:sourceUri, not
|
||||||
|
sem:uri: the key names a field, the term names a relation."""
|
||||||
|
data = {
|
||||||
|
"entities": [
|
||||||
|
{"id": ENTITY_IRI, "text": "Acme", "metadata": {"uri": "https://example.org/db"}}
|
||||||
|
],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
g = _serialize(RDFSerializer(), "turtle", data)
|
||||||
|
assert (
|
||||||
|
URIRef(ENTITY_IRI),
|
||||||
|
URIRef(f"{SEMANTICA_NS}sourceUri"),
|
||||||
|
URIRef("https://example.org/db"),
|
||||||
|
) in g
|
||||||
|
|
||||||
|
|
||||||
|
def test_output_is_unchanged_when_no_metadata_is_present():
|
||||||
|
plain = {
|
||||||
|
"entities": [{"id": ENTITY_IRI, "type": "https://example.org/Org", "text": "Acme"}],
|
||||||
|
"relationships": [{"source_id": ENTITY_IRI, "target_id": "https://example.org/e2"}],
|
||||||
|
}
|
||||||
|
serializer = RDFSerializer()
|
||||||
|
assert serializer.serialize_to_turtle(plain) == serializer.serialize_to_turtle(plain)
|
||||||
|
g = _parse(serializer.serialize_to_turtle(plain), "turtle")
|
||||||
|
assert len(g) == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_default_term_is_declared_in_the_shipped_vocabulary():
|
||||||
|
"""Drift guard: a term the exporter emits and the vocabulary omits is a bug."""
|
||||||
|
from semantica.ontology.vocabulary import vocabulary_path
|
||||||
|
|
||||||
|
vocab = Graph()
|
||||||
|
vocab.parse(vocabulary_path(), format="turtle")
|
||||||
|
declared = {str(s) for s in vocab.subjects()}
|
||||||
|
missing = sorted(set(DEFAULT_METADATA_TERMS.values()) - declared)
|
||||||
|
assert not missing, f"emitted but undeclared: {missing}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_jsonld_metadata_survives_a_real_jsonld_processor():
|
||||||
|
data = {
|
||||||
|
"entities": [
|
||||||
|
{"id": ENTITY_IRI, "text": "Acme", "metadata": {"num_entities": 3}}
|
||||||
|
],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
raw = RDFSerializer().serialize_to_jsonld(data)
|
||||||
|
json.loads(raw) # must be valid JSON before it can be valid JSON-LD
|
||||||
|
g = _parse(raw, "json-ld")
|
||||||
|
assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(3)) in g
|
||||||
Reference in New Issue
Block a user