mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
feat(triplet-store): add embedded Oxigraph backend
This commit is contained in:
@@ -73,7 +73,7 @@ Semantica sits underneath your LLM, vector store, and agent framework as a deter
|
||||
- **Knowledge Pipeline:** Multi-source ingestion, entity-aware chunking, NER/relation/event extraction, and knowledge graph construction, with semantic deduplication and provenance-preserving merges throughout
|
||||
- **Enterprise Data Platforms:** Native connectors for Databricks (Unity Catalog + Delta Lake, PAT/OAuth M2M auth, catalog/schema/table/lineage introspection) and Snowflake (warehouse/database/schema, key-pair and OAuth auth), so tables already living in your lakehouse or warehouse become graph nodes with provenance, not another export/import hop
|
||||
- **Graph Analytics:** Centrality, community detection, link prediction, and shortest-path queries over the graph you just built
|
||||
- **Polyglot Graph Storage:** Native RDF (Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Polyglot Graph Storage:** Native RDF (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Visualization:** Explore any graph, ontology, or timeline in an interactive browser workbench
|
||||
- **Drop-in Integrations:** Native Agno support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
|
||||
@@ -160,7 +160,7 @@ Sources → Ingest → Parse → Normalize → Split → Extract → Conflict De
|
||||
- **Extract → Conflict Detection → Deduplication:** NER, relations, events, triplets; conflicting facts flagged and resolved before they merge
|
||||
- **Knowledge Graph:** `GraphBuilder` constructs the graph; bi-temporal facts and full graph analytics (centrality, communities, link prediction) run on top of it
|
||||
- **Ontology · Reasoning · Provenance · Decisions:** the intelligence layer sitting on the KG, with SHACL/OWL governance, Rete/Datalog/SPARQL inference, W3C PROV-O lineage, and first-class decision records
|
||||
- **Storage:** polyglot by design, with RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code
|
||||
- **Storage:** polyglot by design, with RDF triple stores (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code
|
||||
- **Outputs:** export (RDF, OWL, Parquet, Cypher, JSON-LD), interactive visualization, and access via REST API, MCP server, or CLI
|
||||
|
||||
**→ [Full Mermaid diagrams for the pipeline and the decision intelligence lifecycle](ARCHITECTURE.md)**
|
||||
@@ -1145,7 +1145,7 @@ if report.valid:
|
||||
| **Ontology Hub** | SHACL Studio · visual editor · cross-ontology alignments · health dashboard |
|
||||
| **Vector Store** | FAISS · Pinecone · Weaviate · Qdrant · Milvus · PgVector · hybrid + filtered search |
|
||||
| **Graph Databases (LPG)** | Neo4j · FalkorDB · Apache AGE · AWS Neptune |
|
||||
| **Triple Stores (RDF)** | Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load |
|
||||
| **Triple Stores (RDF)** | Oxigraph (embedded) · Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load |
|
||||
| **Enterprise Data Platforms** | Databricks (`DatabricksIngestor`: Unity Catalog + Delta Lake, PAT/OAuth M2M, table/query ingestion, catalog/schema/table/lineage introspection) · Snowflake (`SnowflakeIngestor`: warehouse/database/schema, password/key-pair/OAuth auth) |
|
||||
| **LLM Providers** | **All already supported today:** OpenAI (GPT-4o, o1, o3) · Anthropic (Claude) · Google Gemini · Mistral · Meta Llama · Groq · Cohere · Azure OpenAI · AWS Bedrock · Ollama · DeepSeek · Perplexity · Together AI · Fireworks AI · Replicate · HuggingFace · via `semantica.llms` and LiteLLM |
|
||||
|
||||
@@ -1511,6 +1511,7 @@ pip install semantica[graph-neo4j] # Neo4j graph store (LPG)
|
||||
pip install semantica[graph-falkordb] # FalkorDB graph store (LPG)
|
||||
pip install semantica[graph-apache-age] # Apache AGE graph store (LPG)
|
||||
pip install semantica[graph-amazon-neptune] # AWS Neptune graph store (LPG)
|
||||
pip install semantica[tripletstore-oxigraph] # Embedded in-memory/on-disk RDF store
|
||||
# RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J) need no extra:
|
||||
# semantica.triplet_store talks SPARQL over HTTP using the core `requests` dependency
|
||||
pip install semantica[vectorstore-qdrant] # Qdrant vector store
|
||||
|
||||
+1
-1
@@ -149,7 +149,7 @@ A database optimized for storing and querying graph-structured data using node a
|
||||
A retrieval strategy combining vector similarity search with keyword or metadata filtering: higher accuracy than either approach alone.
|
||||
|
||||
**Triplet Store**
|
||||
A database designed specifically for storing and querying RDF `(subject, predicate, object)` triples. Semantica supports Blazegraph, Apache Jena, and RDF4J.
|
||||
A database designed specifically for storing and querying RDF `(subject, predicate, object)` triples. Semantica supports embedded Oxigraph as well as Blazegraph, Apache Jena, and RDF4J.
|
||||
|
||||
**Vector Store**
|
||||
A database optimized for storing and searching high-dimensional embedding vectors by similarity. Semantica supports FAISS, Pinecone, Weaviate, Qdrant, Milvus, and PgVector.
|
||||
|
||||
+1
-1
@@ -251,7 +251,7 @@ store.add_triplets(subject, predicate, obj)
|
||||
results = store.sparql("SELECT ?s ?p ?o WHERE { ?s ?p ?o }")
|
||||
```
|
||||
|
||||
**Backends:** Blazegraph, Apache Jena, RDF4J
|
||||
**Backends:** Oxigraph (embedded), Blazegraph, Apache Jena, RDF4J
|
||||
|
||||
|
||||
## Quality Assurance
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: "Triplet Store Module"
|
||||
description: "RDF triple storage with SPARQL queries and bulk loading: Blazegraph, Apache Jena, and RDF4J."
|
||||
description: "Embedded and server-backed RDF storage with SPARQL queries and bulk loading."
|
||||
icon: "table"
|
||||
---
|
||||
|
||||
@@ -16,14 +16,15 @@ icon: "table"
|
||||
| `BlazegraphStore` | Blazegraph REST API: SPARQL 1.1 Update, namespace management |
|
||||
| `JenaStore` | Apache Jena: rdflib-backed, SPARQL read support via remote endpoint |
|
||||
| `RDF4JStore` | Eclipse RDF4J: REST API, transaction support |
|
||||
| `OxigraphStore` | Embedded SPARQL 1.1 store with in-memory and on-disk modes |
|
||||
|
||||
## What You Get
|
||||
|
||||
- **TripletStore** — Unified interface across Blazegraph, Apache Jena, and RDF4J: swap backends with one parameter.
|
||||
- **TripletStore** — Unified interface across embedded Oxigraph, Blazegraph, Apache Jena, and RDF4J: swap backends with one parameter.
|
||||
- **SPARQL** — Full SPARQL SELECT, ASK, CONSTRUCT, and UPDATE query support via `execute_query()`.
|
||||
- **Bulk Loading** — `add_triplets()` batches writes with configurable batch size, retry logic, and progress tracking.
|
||||
- **SKOS Vocabulary** — Built-in helpers: `add_skos_concept()` and `get_skos_concepts()` for controlled vocabulary management.
|
||||
- **Named Graphs** — Blazegraph and RDF4J support named graph scoping via `graph=` on `execute_query()`.
|
||||
- **Named Graphs** — Oxigraph, Blazegraph, and RDF4J support named graph scoping via `graph=` on `execute_query()`.
|
||||
- **Delta Computation** — `compute_delta(old_graph_uri, new_graph_uri)` returns added and removed triples between two named graph snapshots.
|
||||
|
||||
## Getting Started
|
||||
@@ -117,6 +118,25 @@ for row in result.bindings:
|
||||
## Backends
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Oxigraph">
|
||||
```bash
|
||||
pip install "semantica[tripletstore-oxigraph]"
|
||||
```
|
||||
|
||||
```python
|
||||
# In-memory: no server process or files required
|
||||
store = TripletStore(backend="oxigraph")
|
||||
|
||||
# Persistent: reopen the same directory to reuse the data
|
||||
persistent_store = TripletStore(
|
||||
backend="oxigraph",
|
||||
path="./data/knowledge-graph",
|
||||
)
|
||||
```
|
||||
|
||||
**Best for:** local development, CI, desktop applications, and persistent
|
||||
single-process workloads without external infrastructure.
|
||||
</Tab>
|
||||
<Tab title="Blazegraph">
|
||||
```bash
|
||||
pip install requests
|
||||
@@ -172,6 +192,7 @@ for row in result.bindings:
|
||||
|
||||
| Backend | License | Named Graphs | Write via | Best For |
|
||||
| :------- | :------- | :------------ | :--------- | :-------- |
|
||||
| Oxigraph | Apache 2.0 / MIT | Yes | Embedded native API | Local, CI, on-disk |
|
||||
| Blazegraph | Open source | Yes | SPARQL Update REST | High triple count, SPARQL 1.1 |
|
||||
| Apache Jena | Apache 2.0 | No (rdflib backend) | rdflib in-process | Local dev, read queries |
|
||||
| RDF4J | Eclipse 1.0 | Yes | REST API N-Triples | Enterprise Java, transactions |
|
||||
@@ -180,7 +201,9 @@ for row in result.bindings:
|
||||
</Tabs>
|
||||
|
||||
<Tip>
|
||||
**Use Apache Jena for development, Blazegraph for production.** Jena initializes with rdflib in-memory: no server required for local testing. Switch to Blazegraph for high-throughput persistent workloads by changing `backend=`.
|
||||
**Use Oxigraph for zero-infrastructure development and local persistence.**
|
||||
Switch to a server-backed store for distributed production deployments by
|
||||
changing `backend=`.
|
||||
</Tip>
|
||||
|
||||
## Triplet Object
|
||||
@@ -364,10 +387,10 @@ while True:
|
||||
|
||||
## Named Graph Scoping
|
||||
|
||||
Blazegraph and RDF4J support named graphs. Scope `execute_query()` to a named graph with the `graph=` parameter:
|
||||
Oxigraph, Blazegraph, and RDF4J support named graphs. Scope `execute_query()` to a named graph with the `graph=` parameter:
|
||||
|
||||
```python
|
||||
# Add a triplet: named graph stored in metadata or backend-specific API
|
||||
# Add a triplet to a named graph
|
||||
from semantica.semantic_extract.types import Triplet
|
||||
|
||||
t = Triplet(
|
||||
@@ -375,7 +398,7 @@ t = Triplet(
|
||||
predicate="http://example.org/p",
|
||||
object="http://example.org/b",
|
||||
)
|
||||
store.add_triplet(t) # named graph targeting requires backend-specific API
|
||||
store.add_triplet(t, graph="http://example.org/graph1")
|
||||
|
||||
# Query a named graph via FROM clause in SPARQL
|
||||
result = store.execute_query("""
|
||||
@@ -393,11 +416,14 @@ result = store.execute_query("""
|
||||
```
|
||||
|
||||
<Note>
|
||||
Named graph support is only available for Blazegraph and RDF4J backends. The `graph=` parameter is silently ignored for the Jena backend.
|
||||
Named graph query scoping is available for Oxigraph, Blazegraph, and RDF4J.
|
||||
The `graph=` query parameter is silently ignored for the Jena backend.
|
||||
</Note>
|
||||
|
||||
<Tip>
|
||||
**Use named graphs to isolate sources.** Pass `graph="http://example.org/source_A"` to `execute_query()` to scope a query to a specific named graph. Blazegraph and RDF4J support named graphs; Jena (rdflib backend) does not.
|
||||
**Use named graphs to isolate sources.** Pass `graph="http://example.org/source_A"`
|
||||
to writes and `execute_query()` to scope both storage and retrieval. Oxigraph,
|
||||
Blazegraph, and RDF4J support named graph query scoping.
|
||||
</Tip>
|
||||
|
||||
## Bulk Loading
|
||||
|
||||
+5
-2
@@ -146,6 +146,9 @@ graph-all = [
|
||||
"semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune,graph-apache-age]"
|
||||
]
|
||||
|
||||
# ---- Triplet Store Backends ----
|
||||
tripletstore-oxigraph = ["pyoxigraph>=0.5.0"]
|
||||
|
||||
# ---- Vector Store Backends ----
|
||||
vectorstore-qdrant = ["qdrant-client>=1.0.0"]
|
||||
vectorstore-weaviate = ["weaviate-client>=4.0.0"]
|
||||
@@ -239,8 +242,8 @@ explorer-lite = [
|
||||
|
||||
# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux)
|
||||
all = [
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]"
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]"
|
||||
]
|
||||
|
||||
# ---------------- ENTRYPOINTS ----------------
|
||||
|
||||
@@ -3,11 +3,11 @@ Triplet Store Module
|
||||
|
||||
This module provides comprehensive triplet store integration and management
|
||||
for RDF data storage and querying, supporting multiple triplet store backends
|
||||
(Blazegraph, Jena, RDF4J, Anzo) with unified interfaces.
|
||||
(Blazegraph, Jena, RDF4J, Anzo, Oxigraph) with unified interfaces.
|
||||
|
||||
Key Features:
|
||||
- Unified triplet store interface
|
||||
- Multi-backend support (Blazegraph, Jena, RDF4J, Anzo)
|
||||
- Multi-backend support (Blazegraph, Jena, RDF4J, Anzo, Oxigraph)
|
||||
- CRUD operations for RDF triplets
|
||||
- SPARQL query execution and optimization
|
||||
- Bulk data loading with progress tracking
|
||||
@@ -21,6 +21,7 @@ Main Classes:
|
||||
- JenaStore: Apache Jena integration store
|
||||
- RDF4JStore: Eclipse RDF4J integration store
|
||||
- AnzoStore: Altair Anzo integration store
|
||||
- OxigraphStore: Embedded in-memory or on-disk RDF store
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triplet_store import TripletStore
|
||||
@@ -37,6 +38,7 @@ from .blazegraph_store import BlazegraphStore
|
||||
from .jena_store import JenaStore
|
||||
from .rdf4j_store import RDF4JStore
|
||||
from .anzo_store import AnzoStore
|
||||
from .oxigraph_store import OxigraphStore
|
||||
from .methods import (
|
||||
register_store,
|
||||
add_triplet,
|
||||
@@ -62,6 +64,7 @@ __all__ = [
|
||||
"JenaStore",
|
||||
"RDF4JStore",
|
||||
"AnzoStore",
|
||||
"OxigraphStore",
|
||||
"register_store",
|
||||
"add_triplet",
|
||||
"add_triplets",
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Embedded RDF storage backed by Oxigraph.
|
||||
|
||||
This backend provides an in-process SPARQL 1.1 store for development, tests,
|
||||
and workloads that do not need a separately managed graph database server.
|
||||
It can run entirely in memory or persist its data to a local directory.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..semantic_extract.triplet_extractor import Triplet
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from . import sparql_escaping
|
||||
|
||||
|
||||
class OxigraphStore:
|
||||
"""Embedded, optionally persistent RDF store powered by PyOxigraph."""
|
||||
|
||||
supports_named_graphs = True
|
||||
_XSD_STRING = "http://www.w3.org/2001/XMLSchema#string"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: Optional[Union[str, os.PathLike]] = None,
|
||||
**config,
|
||||
):
|
||||
"""Initialize an in-memory or on-disk Oxigraph store.
|
||||
|
||||
Args:
|
||||
path: Directory used for persistent storage. If omitted, the store
|
||||
is kept in memory.
|
||||
**config: Backend configuration. ``path`` may also be supplied in
|
||||
this mapping.
|
||||
|
||||
Raises:
|
||||
ImportError: If the optional ``pyoxigraph`` package is missing.
|
||||
ProcessingError: If the store cannot be opened.
|
||||
"""
|
||||
self.logger = get_logger("oxigraph_store")
|
||||
self.config = config
|
||||
self.path = path if path is not None else config.get("path")
|
||||
|
||||
try:
|
||||
self._oxigraph = importlib.import_module("pyoxigraph")
|
||||
except (ImportError, OSError) as exc:
|
||||
raise ImportError(
|
||||
"PyOxigraph is required for the 'oxigraph' backend. "
|
||||
'Install it with: pip install "semantica[tripletstore-oxigraph]"'
|
||||
) from exc
|
||||
|
||||
store_path = None
|
||||
if self.path is not None:
|
||||
path_obj = Path(self.path).expanduser()
|
||||
try:
|
||||
path_obj.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as exc:
|
||||
raise ProcessingError(
|
||||
f"Failed to create Oxigraph store directory {path_obj}: {exc}"
|
||||
) from exc
|
||||
store_path = str(path_obj)
|
||||
self.path = store_path
|
||||
|
||||
try:
|
||||
self.store = self._oxigraph.Store(store_path)
|
||||
except Exception as exc:
|
||||
location = store_path or "memory"
|
||||
raise ProcessingError(
|
||||
f"Failed to initialize Oxigraph store at {location}: {exc}"
|
||||
) from exc
|
||||
|
||||
def add_triplet(self, triplet: Triplet, **options) -> Dict[str, Any]:
|
||||
"""Add one triplet to the default graph or ``options['graph']``."""
|
||||
return self.add_triplets([triplet], **options)
|
||||
|
||||
def add_triplets(self, triplets: List[Triplet], **options) -> Dict[str, Any]:
|
||||
"""Add triplets in one native Oxigraph batch."""
|
||||
try:
|
||||
graph_name = self._graph_name(options.get("graph"))
|
||||
quads = [self._to_quad(triplet, graph_name) for triplet in triplets]
|
||||
self.store.extend(quads)
|
||||
return {
|
||||
"success": True,
|
||||
"triplets_loaded": len(triplets),
|
||||
"graph": options.get("graph"),
|
||||
}
|
||||
except Exception as exc:
|
||||
self.logger.error(f"Oxigraph load failed: {exc}")
|
||||
raise ProcessingError(f"Oxigraph load failed: {exc}") from exc
|
||||
|
||||
def bulk_load(self, triplets: List[Triplet], **options) -> Dict[str, Any]:
|
||||
"""Load a batch of triplets using Oxigraph's native bulk operation."""
|
||||
return self.add_triplets(triplets, **options)
|
||||
|
||||
def get_triplets(
|
||||
self,
|
||||
subject: Optional[str] = None,
|
||||
predicate: Optional[str] = None,
|
||||
object: Optional[str] = None,
|
||||
**options,
|
||||
) -> List[Triplet]:
|
||||
"""Return triplets matching a pattern in one graph.
|
||||
|
||||
The default graph is queried unless ``graph`` is provided, matching
|
||||
the behavior of the other graph-aware triplet-store backends.
|
||||
"""
|
||||
try:
|
||||
ox = self._oxigraph
|
||||
subject_term = ox.NamedNode(subject) if subject is not None else None
|
||||
predicate_term = ox.NamedNode(predicate) if predicate is not None else None
|
||||
object_term = (
|
||||
self._object_from_value(object) if object is not None else None
|
||||
)
|
||||
graph_name = self._graph_name(options.get("graph"))
|
||||
quads = self.store.quads_for_pattern(
|
||||
subject_term, predicate_term, object_term, graph_name
|
||||
)
|
||||
return [self._quad_to_triplet(quad) for quad in quads]
|
||||
except Exception as exc:
|
||||
self.logger.error(f"Failed to get Oxigraph triplets: {exc}")
|
||||
raise ProcessingError(f"Failed to get Oxigraph triplets: {exc}") from exc
|
||||
|
||||
def delete_triplet(self, triplet: Triplet, **options) -> Dict[str, Any]:
|
||||
"""Delete one triplet from the default graph or ``options['graph']``."""
|
||||
try:
|
||||
graph_name = self._graph_name(options.get("graph"))
|
||||
self.store.remove(self._to_quad(triplet, graph_name))
|
||||
return {"success": True}
|
||||
except Exception as exc:
|
||||
self.logger.error(f"Failed to delete Oxigraph triplet: {exc}")
|
||||
raise ProcessingError(f"Failed to delete Oxigraph triplet: {exc}") from exc
|
||||
|
||||
def execute_sparql(self, query: str, **options) -> Dict[str, Any]:
|
||||
"""Execute a SPARQL query and return the shared backend result shape."""
|
||||
result_format = options.get("result_format")
|
||||
if result_format not in (None, "bindings", "construct"):
|
||||
raise ValidationError(f"Invalid result_format: {result_format!r}")
|
||||
|
||||
try:
|
||||
result = self.store.query(query)
|
||||
|
||||
if isinstance(result, self._oxigraph.QuerySolutions):
|
||||
if result_format == "construct":
|
||||
raise ValidationError(
|
||||
"result_format='construct' requires a CONSTRUCT or "
|
||||
"DESCRIBE query"
|
||||
)
|
||||
variables = [variable.value for variable in result.variables]
|
||||
bindings = []
|
||||
for solution in result:
|
||||
binding = {}
|
||||
for variable in variables:
|
||||
term = solution[variable]
|
||||
if term is not None:
|
||||
binding[variable] = self._term_to_binding(term)
|
||||
bindings.append(binding)
|
||||
return {
|
||||
"success": True,
|
||||
"bindings": bindings,
|
||||
"variables": variables,
|
||||
"metadata": {"query": query},
|
||||
}
|
||||
|
||||
if isinstance(result, self._oxigraph.QueryBoolean):
|
||||
if result_format == "construct":
|
||||
raise ValidationError(
|
||||
"result_format='construct' requires a CONSTRUCT or "
|
||||
"DESCRIBE query"
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"bindings": [],
|
||||
"variables": [],
|
||||
"metadata": {"query": query, "boolean": bool(result)},
|
||||
}
|
||||
|
||||
if isinstance(result, self._oxigraph.QueryTriples):
|
||||
if result_format == "bindings":
|
||||
raise ValidationError(
|
||||
"result_format='bindings' cannot be used with a "
|
||||
"CONSTRUCT or DESCRIBE query"
|
||||
)
|
||||
triples = []
|
||||
for triple in result:
|
||||
metadata = self._literal_metadata(triple.object)
|
||||
triples.append(
|
||||
(
|
||||
self._term_value(triple.subject),
|
||||
self._term_value(triple.predicate),
|
||||
self._term_value(triple.object),
|
||||
metadata,
|
||||
)
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"bindings": [],
|
||||
"variables": [],
|
||||
"triples": triples,
|
||||
"metadata": {
|
||||
"query": query,
|
||||
"result_format": "construct",
|
||||
},
|
||||
}
|
||||
|
||||
raise ProcessingError(
|
||||
f"Unsupported Oxigraph query result: {type(result).__name__}"
|
||||
)
|
||||
except (ValidationError, ProcessingError):
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.logger.error(f"Oxigraph SPARQL query failed: {exc}")
|
||||
raise ProcessingError(f"Oxigraph SPARQL query failed: {exc}") from exc
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Flush pending writes to disk for persistent stores."""
|
||||
self.store.flush()
|
||||
|
||||
def _to_quad(self, triplet: Triplet, graph_name: Any) -> Any:
|
||||
ox = self._oxigraph
|
||||
return ox.Quad(
|
||||
ox.NamedNode(triplet.subject),
|
||||
ox.NamedNode(triplet.predicate),
|
||||
self._object_from_triplet(triplet),
|
||||
graph_name,
|
||||
)
|
||||
|
||||
def _object_from_triplet(self, triplet: Triplet) -> Any:
|
||||
if self._is_uri_value(triplet.object):
|
||||
return self._oxigraph.NamedNode(self._unwrap_iri(triplet.object))
|
||||
|
||||
metadata = triplet.metadata or {}
|
||||
language = metadata.get("lang") or metadata.get("language")
|
||||
datatype = metadata.get("datatype") or metadata.get("literal_datatype")
|
||||
if language:
|
||||
return self._oxigraph.Literal(str(triplet.object), language=str(language))
|
||||
if datatype:
|
||||
datatype_iri = self._unwrap_iri(
|
||||
sparql_escaping.resolve_datatype_iri(str(datatype))
|
||||
)
|
||||
return self._oxigraph.Literal(
|
||||
str(triplet.object),
|
||||
datatype=self._oxigraph.NamedNode(datatype_iri),
|
||||
)
|
||||
return self._oxigraph.Literal(str(triplet.object))
|
||||
|
||||
def _object_from_value(self, value: str) -> Any:
|
||||
if self._is_uri_value(value):
|
||||
return self._oxigraph.NamedNode(self._unwrap_iri(value))
|
||||
return self._oxigraph.Literal(str(value))
|
||||
|
||||
def _graph_name(self, graph: Optional[str]) -> Any:
|
||||
if graph is None:
|
||||
return self._oxigraph.DefaultGraph()
|
||||
return self._oxigraph.NamedNode(str(graph))
|
||||
|
||||
def _quad_to_triplet(self, quad: Any) -> Triplet:
|
||||
metadata = {"source": "oxigraph"}
|
||||
metadata.update(self._literal_metadata(quad.object))
|
||||
return Triplet(
|
||||
subject=self._term_value(quad.subject),
|
||||
predicate=self._term_value(quad.predicate),
|
||||
object=self._term_value(quad.object),
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
def _term_to_binding(self, term: Any) -> Dict[str, str]:
|
||||
if isinstance(term, self._oxigraph.NamedNode):
|
||||
return {"type": "uri", "value": term.value}
|
||||
if isinstance(term, self._oxigraph.BlankNode):
|
||||
return {"type": "bnode", "value": term.value}
|
||||
if isinstance(term, self._oxigraph.Literal):
|
||||
binding = {"type": "literal", "value": term.value}
|
||||
if term.language is not None:
|
||||
binding["xml:lang"] = term.language
|
||||
elif term.datatype.value != self._XSD_STRING:
|
||||
binding["datatype"] = term.datatype.value
|
||||
return binding
|
||||
return {"type": "literal", "value": str(term)}
|
||||
|
||||
def _literal_metadata(self, term: Any) -> Dict[str, str]:
|
||||
if not isinstance(term, self._oxigraph.Literal):
|
||||
return {}
|
||||
metadata = {}
|
||||
if term.language is not None:
|
||||
metadata["language"] = term.language
|
||||
elif term.datatype.value != self._XSD_STRING:
|
||||
metadata["datatype"] = term.datatype.value
|
||||
return metadata
|
||||
|
||||
def _term_value(self, term: Any) -> str:
|
||||
if isinstance(
|
||||
term,
|
||||
(
|
||||
self._oxigraph.NamedNode,
|
||||
self._oxigraph.BlankNode,
|
||||
self._oxigraph.Literal,
|
||||
),
|
||||
):
|
||||
return term.value
|
||||
return str(term)
|
||||
|
||||
@staticmethod
|
||||
def _unwrap_iri(value: str) -> str:
|
||||
if value.startswith("<") and value.endswith(">"):
|
||||
return value[1:-1]
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _is_uri_value(value: str) -> bool:
|
||||
if not isinstance(value, str) or not value:
|
||||
return False
|
||||
if value.startswith("<") and value.endswith(">"):
|
||||
return True
|
||||
parsed = urlparse(value)
|
||||
return parsed.scheme in {"http", "https", "urn"} and not re.search(r"\s", value)
|
||||
@@ -3,7 +3,7 @@ Triplet Store Core Module
|
||||
|
||||
This module provides the core triplet store interface and management classes,
|
||||
providing a unified interface across multiple RDF store backends
|
||||
(Blazegraph, Jena, RDF4J).
|
||||
(Blazegraph, Jena, RDF4J, Anzo, and Oxigraph).
|
||||
|
||||
Key Features:
|
||||
- Unified triplet store interface
|
||||
@@ -42,11 +42,16 @@ class TripletStore:
|
||||
Main triplet store interface.
|
||||
|
||||
Provides a unified interface for working with RDF triple stores,
|
||||
supporting Blazegraph, Jena, and RDF4J backends.
|
||||
supporting server-backed stores and an embedded Oxigraph backend.
|
||||
"""
|
||||
|
||||
SUPPORTED_BACKENDS = {"blazegraph", "jena", "rdf4j", "anzo"}
|
||||
NAMED_GRAPH_CAPABLE_BACKENDS = {"blazegraph", "rdf4j", "anzo"}
|
||||
SUPPORTED_BACKENDS = {"blazegraph", "jena", "rdf4j", "anzo", "oxigraph"}
|
||||
NAMED_GRAPH_CAPABLE_BACKENDS = {
|
||||
"blazegraph",
|
||||
"rdf4j",
|
||||
"anzo",
|
||||
"oxigraph",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -58,7 +63,8 @@ class TripletStore:
|
||||
Initialize triplet store.
|
||||
|
||||
Args:
|
||||
backend: Backend type ("blazegraph", "jena", "rdf4j")
|
||||
backend: Backend type ("blazegraph", "jena", "rdf4j", "anzo",
|
||||
or "oxigraph")
|
||||
endpoint: Store endpoint URL
|
||||
**config: Backend-specific configuration
|
||||
"""
|
||||
@@ -143,6 +149,11 @@ class TripletStore:
|
||||
|
||||
self._store_backend = AnzoStore(**backend_config)
|
||||
|
||||
elif self.backend_type == "oxigraph":
|
||||
from .oxigraph_store import OxigraphStore
|
||||
|
||||
self._store_backend = OxigraphStore(**self.config)
|
||||
|
||||
self.logger.info(f"Initialized {self.backend_type} backend")
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Triplet Store Module Usage Guide
|
||||
|
||||
This comprehensive guide demonstrates how to use the triplet store module for RDF data storage and querying, supporting multiple triplet store backends (Blazegraph, Jena, RDF4J) with unified interfaces, SPARQL query execution, bulk loading, and query optimization.
|
||||
This comprehensive guide demonstrates how to use the triplet store module for RDF data storage and querying, supporting embedded Oxigraph and server-backed Blazegraph, Jena, RDF4J, and Anzo stores with unified interfaces, SPARQL query execution, bulk loading, and query optimization.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
@@ -148,6 +148,36 @@ print(f"Failed: {progress.failed_triplets}")
|
||||
|
||||
## Store Backends
|
||||
|
||||
### Oxigraph (embedded)
|
||||
|
||||
Oxigraph runs in the Python process and does not require a separate server.
|
||||
Install the optional backend first:
|
||||
|
||||
```bash
|
||||
pip install "semantica[tripletstore-oxigraph]"
|
||||
```
|
||||
|
||||
Omit `path` for an in-memory store, which is useful for tests and temporary
|
||||
workloads:
|
||||
|
||||
```python
|
||||
store = TripletStore(backend="oxigraph")
|
||||
```
|
||||
|
||||
Set `path` to persist the database in a local directory:
|
||||
|
||||
```python
|
||||
store = TripletStore(
|
||||
backend="oxigraph",
|
||||
path="./data/knowledge-graph",
|
||||
)
|
||||
```
|
||||
|
||||
The Oxigraph backend supports the same CRUD and SPARQL query methods as the
|
||||
other backends, including named graphs through the `graph` argument. For
|
||||
distributed production deployments, use one of the server-backed stores
|
||||
below.
|
||||
|
||||
### Blazegraph
|
||||
High-performance graph database supporting RDF/SPARQL.
|
||||
```python
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
import gc
|
||||
import importlib
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.semantic_extract.triplet_extractor import Triplet
|
||||
from semantica.triplet_store import OxigraphStore, TripletStore
|
||||
from semantica.utils.exceptions import ProcessingError, ValidationError
|
||||
|
||||
pytest.importorskip("pyoxigraph")
|
||||
|
||||
|
||||
EX = "http://example.org/"
|
||||
|
||||
|
||||
def _store(**config):
|
||||
return TripletStore(
|
||||
backend="oxigraph",
|
||||
enable_caching=False,
|
||||
enable_optimization=False,
|
||||
**config,
|
||||
)
|
||||
|
||||
|
||||
def test_memory_store_crud_preserves_literal_metadata():
|
||||
store = _store()
|
||||
triplets = [
|
||||
Triplet(EX + "alice", EX + "knows", EX + "bob"),
|
||||
Triplet(
|
||||
EX + "alice",
|
||||
EX + "name",
|
||||
"Alice",
|
||||
metadata={"language": "en"},
|
||||
),
|
||||
Triplet(
|
||||
EX + "alice",
|
||||
EX + "age",
|
||||
"42",
|
||||
metadata={"datatype": "xsd:integer"},
|
||||
),
|
||||
]
|
||||
|
||||
result = store.add_triplets(triplets)
|
||||
|
||||
assert result == {
|
||||
"success": True,
|
||||
"total": 3,
|
||||
"processed": 3,
|
||||
"failed": 0,
|
||||
"batches": 1,
|
||||
}
|
||||
saved = store.get_triplets(subject=EX + "alice")
|
||||
assert len(saved) == 3
|
||||
assert (
|
||||
next(t for t in saved if t.predicate == EX + "name").metadata["language"]
|
||||
== "en"
|
||||
)
|
||||
assert (
|
||||
next(t for t in saved if t.predicate == EX + "age").metadata["datatype"]
|
||||
== "http://www.w3.org/2001/XMLSchema#integer"
|
||||
)
|
||||
|
||||
store.delete_triplet(triplets[0])
|
||||
|
||||
assert store.get_triplets(predicate=EX + "knows") == []
|
||||
|
||||
|
||||
def test_named_graphs_are_isolated_for_crud_and_queries():
|
||||
store = _store()
|
||||
default_triplet = Triplet(EX + "default", EX + "label", "default")
|
||||
named_triplet = Triplet(EX + "named", EX + "label", "named")
|
||||
graph = EX + "graphs/agents"
|
||||
|
||||
store.add_triplet(default_triplet)
|
||||
store.add_triplet(named_triplet, graph=graph)
|
||||
|
||||
assert [t.subject for t in store.get_triplets()] == [default_triplet.subject]
|
||||
assert [t.subject for t in store.get_triplets(graph=graph)] == [
|
||||
named_triplet.subject
|
||||
]
|
||||
|
||||
result = store.execute_query(
|
||||
"SELECT ?s WHERE { ?s <http://example.org/label> ?label }",
|
||||
graph=graph,
|
||||
)
|
||||
assert [row["s"]["value"] for row in result.bindings] == [named_triplet.subject]
|
||||
|
||||
|
||||
def test_select_ask_and_construct_result_shapes():
|
||||
backend = OxigraphStore()
|
||||
backend.add_triplet(
|
||||
Triplet(
|
||||
EX + "alice",
|
||||
EX + "name",
|
||||
"Alice",
|
||||
metadata={"language": "en"},
|
||||
)
|
||||
)
|
||||
|
||||
selected = backend.execute_sparql(
|
||||
"SELECT ?name WHERE { <http://example.org/alice> "
|
||||
"<http://example.org/name> ?name }"
|
||||
)
|
||||
assert selected["variables"] == ["name"]
|
||||
assert selected["bindings"] == [
|
||||
{"name": {"type": "literal", "value": "Alice", "xml:lang": "en"}}
|
||||
]
|
||||
|
||||
asked = backend.execute_sparql("ASK { ?s ?p ?o }")
|
||||
assert asked["metadata"]["boolean"] is True
|
||||
|
||||
constructed = backend.execute_sparql("CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }")
|
||||
assert constructed["variables"] == []
|
||||
assert constructed["triples"] == [
|
||||
(EX + "alice", EX + "name", "Alice", {"language": "en"})
|
||||
]
|
||||
|
||||
|
||||
def test_on_disk_store_survives_reopen(tmp_path):
|
||||
path = tmp_path / "oxigraph"
|
||||
first = OxigraphStore(path=path)
|
||||
first.add_triplet(Triplet(EX + "alice", EX + "knows", EX + "bob"))
|
||||
first.flush()
|
||||
del first
|
||||
gc.collect()
|
||||
|
||||
reopened = OxigraphStore(path=path)
|
||||
|
||||
assert len(reopened.get_triplets()) == 1
|
||||
assert reopened.get_triplets()[0].object == EX + "bob"
|
||||
|
||||
|
||||
def test_invalid_result_format_is_rejected():
|
||||
backend = OxigraphStore()
|
||||
|
||||
with pytest.raises(ValidationError, match="Invalid result_format"):
|
||||
backend.execute_sparql("SELECT * WHERE { ?s ?p ?o }", result_format="csv")
|
||||
|
||||
|
||||
def test_invalid_rdf_term_is_reported_as_processing_error():
|
||||
backend = OxigraphStore()
|
||||
|
||||
with pytest.raises(ProcessingError, match="Oxigraph load failed"):
|
||||
backend.add_triplet(Triplet("not an iri", EX + "predicate", "value"))
|
||||
|
||||
|
||||
def test_missing_optional_dependency_has_install_hint():
|
||||
real_import_module = importlib.import_module
|
||||
|
||||
def import_without_oxigraph(name, package=None):
|
||||
if name == "pyoxigraph":
|
||||
raise ModuleNotFoundError(name)
|
||||
return real_import_module(name, package)
|
||||
|
||||
with patch(
|
||||
"semantica.triplet_store.oxigraph_store.importlib.import_module",
|
||||
side_effect=import_without_oxigraph,
|
||||
):
|
||||
with pytest.raises(ImportError, match="tripletstore-oxigraph"):
|
||||
OxigraphStore()
|
||||
Reference in New Issue
Block a user