From 20755e69e29b16dee50e4ccca2240b05a126baa5 Mon Sep 17 00:00:00 2001 From: Sameer Kadam Date: Sun, 15 Feb 2026 15:57:07 +0530 Subject: [PATCH] feat(graph): add Apache AGE backend integration with configuration, registration, tests and documentation (#311) --- README.md | 17 +- docs/graph_stores/apache_age.md | 243 +++++ semantica/graph_store/__init__.py | 3 + semantica/graph_store/age_store.py | 1312 ++++++++++++++++++++++++++ semantica/graph_store/config.py | 18 + semantica/graph_store/graph_store.py | 7 + tests/graph_store/__init__.py | 0 tests/graph_store/test_age_store.py | 772 +++++++++++++++ 8 files changed, 2369 insertions(+), 3 deletions(-) create mode 100644 docs/graph_stores/apache_age.md create mode 100644 semantica/graph_store/age_store.py create mode 100644 tests/graph_store/__init__.py create mode 100644 tests/graph_store/test_age_store.py diff --git a/README.md b/README.md index 6fce0ca0..ca507b48 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ print(f"Found {len(precedents)} precedents") - **Docling Support** — Document parsing with table extraction (PDF, DOCX, PPTX, XLSX) - **AWS Neptune** — Amazon Neptune graph database support with IAM authentication +- **Apache AGE** — PostgreSQL graph extension backend (openCypher via SQL) - **Custom Ontology Import** — Import existing ontologies (OWL, RDF, Turtle, JSON-LD) > **Built for environments where every answer must be explainable and governed.** @@ -284,7 +285,8 @@ The **semantic gap** is the fundamental disconnect between what AI systems can p - 📐 **OWL Ontologies** — HermiT/Pellet validated, custom ontology import support - 🔢 **Vector Embeddings** — FastEmbed by default - ☁️ **AWS Neptune** — Amazon Neptune graph database support -- 🔍 **Provenance** — Every AI response links back to: +- � **Apache AGE** — PostgreSQL graph extension with openCypher support +- �🔍 **Provenance** — Every AI response links back to: - 📄 Source documents - 🏷️ Extracted entities & relations - 📐 Ontology rules applied @@ -510,13 +512,13 @@ results = vector_store.search(query="supply chain", top_k=5) ### Graph Store & Triplet Store -> **Neo4j, FalkorDB, Amazon Neptune** • **SPARQL queries** • **RDF triplets** +> **Neo4j, FalkorDB, Amazon Neptune, Apache AGE** • **SPARQL queries** • **RDF triplets** ```python from semantica.graph_store import GraphStore from semantica.triplet_store import TripletStore -# Graph Store (Neo4j, FalkorDB) +# Graph Store (Neo4j, FalkorDB, Apache AGE) graph_store = GraphStore(backend="neo4j", uri="bolt://localhost:7687", user="neo4j", password="password") graph_store.add_nodes([{"id": "n1", "labels": ["Person"], "properties": {"name": "Alice"}}]) @@ -538,6 +540,15 @@ neptune_store.add_nodes([ # Query Operations result = neptune_store.execute_query("MATCH (p:Person) RETURN p.name, p.age") +# Apache AGE Graph Store (PostgreSQL + openCypher) +age_store = GraphStore( + backend="age", + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="semantica", +) +age_store.connect() +age_store.create_node(labels=["Person"], properties={"name": "Alice", "age": 30}) + # Triplet Store (Blazegraph, Jena, RDF4J) triplet_store = TripletStore(backend="blazegraph", endpoint="http://localhost:9999/blazegraph") triplet_store.add_triplet({"subject": "Alice", "predicate": "knows", "object": "Bob"}) diff --git a/docs/graph_stores/apache_age.md b/docs/graph_stores/apache_age.md new file mode 100644 index 00000000..27c16a9d --- /dev/null +++ b/docs/graph_stores/apache_age.md @@ -0,0 +1,243 @@ +# Apache AGE Graph Store + +**Backend**: PostgreSQL + [Apache AGE](https://age.apache.org/) +**Driver**: `psycopg2` + +Apache AGE is a PostgreSQL extension that adds graph database functionality, enabling you to run openCypher queries alongside traditional SQL. This backend lets Semantica use AGE as a property graph store with the same interface as Neo4j and FalkorDB. + +--- + +## Prerequisites + +| Component | Version | +|-----------|---------| +| PostgreSQL | 12+ | +| Apache AGE | 1.4+ (compiled and installed) | +| psycopg2 | 2.9+ | + +```bash +pip install psycopg2-binary +``` + +> **Note**: Apache AGE must be compiled and installed into your PostgreSQL instance. See the [AGE installation guide](https://age.apache.org/age-manual/master/intro/setup.html). + +--- + +## Quick Start + +```python +from semantica.graph_store import GraphStore + +# Using the unified GraphStore facade +store = GraphStore( + backend="age", + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="semantica", +) +store.connect() + +# Create nodes +alice = store.create_node(labels=["Person"], properties={"name": "Alice", "age": 30}) +bob = store.create_node(labels=["Person"], properties={"name": "Bob", "age": 25}) + +# Create relationship +rel = store.create_relationship(alice["id"], bob["id"], "KNOWS", {"since": 2023}) + +# Query +result = store.execute_query("MATCH (p:Person) RETURN p", cols="p agtype") +print(result["records"]) + +store.close() +``` + +### Direct Usage (without facade) + +```python +from semantica.graph_store.age_store import ApacheAgeStore + +store = ApacheAgeStore( + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="my_graph", +) +store.connect() + +node = store.create_node(["Entity"], {"semantica_id": "ent-001", "value": "test"}) +print(node) +# {"id": 844424930131969, "labels": ["Entity"], "properties": {"semantica_id": "ent-001", "value": "test"}} + +store.close() +``` + +--- + +## Configuration + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `GRAPH_STORE_AGE_CONNECTION_STRING` | PostgreSQL connection string | `host=localhost dbname=agedb user=postgres password=postgres` | +| `GRAPH_STORE_AGE_GRAPH_NAME` | AGE graph name | `semantica` | + +### Programmatic Configuration + +```python +from semantica.graph_store.config import graph_store_config + +graph_store_config.set("age_connection_string", "host=db.example.com dbname=prod_age user=app") +graph_store_config.set("age_graph_name", "production") +``` + +--- + +## Connection & Initialization + +On `connect()`, the store performs idempotent setup: + +1. `CREATE EXTENSION IF NOT EXISTS age;` +2. `LOAD 'age';` +3. `SET search_path = ag_catalog, "$user", public;` +4. Creates the named graph if it does not already exist. + +This is safe to call repeatedly. + +--- + +## ID Handling + +Apache AGE auto-generates internal vertex/edge IDs (large integers). These are **not** the same as any semantic or application-level ID you may want to assign. + +| Concept | Description | +|---------|-------------| +| **AGE internal ID** | Auto-generated by AGE. Exposed as `"id"` in all returned dicts. Used in `delete_node()`, `get_node()`, etc. | +| **Semantic ID** | Application-level identifier. Store it in the `semantica_id` property. | + +```python +node = store.create_node( + labels=["Document"], + properties={"semantica_id": "doc-abc-123", "title": "My Doc"}, +) +# node["id"] → AGE internal ID (e.g., 844424930131969) +# node["properties"]["semantica_id"] → "doc-abc-123" +``` + +> **Important**: Never mix AGE internal IDs with semantic IDs. Use `node["id"]` for graph operations (delete, update, traverse) and `node["properties"]["semantica_id"]` for application-level lookups. + +--- + +## Label Handling + +AGE supports exactly **one label per vertex**. Semantica handles this transparently: + +- `labels[0]` → used as the primary AGE vertex label. +- `labels[1:]` → stored in a `labels` property array on the vertex. + +When reading nodes, the store reconstructs the full label list automatically. + +```python +node = store.create_node( + labels=["Person", "Employee", "Admin"], + properties={"name": "Alice"}, +) +# In AGE: vertex with label "Person" and property labels=["Employee", "Admin"] +# Returned: {"id": ..., "labels": ["Person", "Employee", "Admin"], "properties": {"name": "Alice"}} +``` + +--- + +## Cypher Query Execution + +All Cypher queries are executed via AGE's SQL wrapper: + +```sql +SELECT * FROM cypher('graph_name', $$ $$) AS (col1 agtype, ...); +``` + +### Parameter Substitution + +AGE does not support `$param` style binding inside `cypher()` calls. The store safely converts parameters to Cypher literals with proper escaping: + +```python +result = store.execute_query( + "MATCH (p:Person) WHERE p.age > $min_age RETURN p", + parameters={"min_age": 25}, + cols="p agtype", +) +``` + +### Column Specification + +For custom queries, pass the `cols` option to specify the `AS` clause: + +```python +result = store.execute_query( + "MATCH (a)-[r]->(b) RETURN a, r, b", + cols="a agtype, r agtype, b agtype", +) +``` + +If omitted, the store attempts to infer columns from the `RETURN` clause. + +--- + +## Transactions + +The store uses explicit PostgreSQL transactions: + +- **Success** → `COMMIT` +- **Exception** → `ROLLBACK`, then re-raise as `ProcessingError` +- No silent failures + +--- + +## API Reference + +All methods match the standard Semantica graph store backend interface: + +| Method | Description | +|--------|-------------| +| `connect(**options)` | Connect and initialize AGE | +| `close()` | Close the connection | +| `create_node(labels, properties)` | Create a vertex | +| `create_nodes(nodes)` | Batch create vertices | +| `get_node(node_id)` | Get vertex by AGE ID | +| `get_nodes(labels, properties, limit)` | Query vertices | +| `update_node(node_id, properties, merge)` | Update vertex properties | +| `delete_node(node_id, detach)` | Delete a vertex | +| `create_relationship(start_id, end_id, type, properties)` | Create an edge | +| `get_relationships(node_id, rel_type, direction, limit)` | Query edges | +| `delete_relationship(rel_id)` | Delete an edge | +| `execute_query(query, parameters)` | Run arbitrary Cypher | +| `get_neighbors(node_id, rel_type, direction, depth)` | Graph traversal | +| `shortest_path(start_id, end_id, rel_type, max_depth)` | Path finding | +| `create_index(label, property_name, index_type)` | Create a PostgreSQL index | +| `get_stats()` | Graph statistics | + +--- + +## Docker Setup + +```yaml +services: + age: + image: apache/age:latest + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret + POSTGRES_DB: agedb +``` + +```bash +docker compose up -d +``` + +Then connect: + +```python +store = GraphStore( + backend="age", + connection_string="host=localhost port=5432 dbname=agedb user=postgres password=secret", +) +``` diff --git a/semantica/graph_store/__init__.py b/semantica/graph_store/__init__.py index ea952b2e..51b02251 100644 --- a/semantica/graph_store/__init__.py +++ b/semantica/graph_store/__init__.py @@ -122,6 +122,7 @@ Author: Semantica Contributors License: MIT """ +from .age_store import ApacheAgeStore from .amazon_neptune import ( AmazonNeptuneStore, NeptuneAuthTokenManager, @@ -172,6 +173,8 @@ __all__ = [ "Neo4jStore", "Neo4jDriver", "Neo4jTransaction", + # Apache AGE + "ApacheAgeStore", # Amazon Neptune "AmazonNeptuneStore", "NeptuneAuthTokenManager", diff --git a/semantica/graph_store/age_store.py b/semantica/graph_store/age_store.py new file mode 100644 index 00000000..1f4ea8b6 --- /dev/null +++ b/semantica/graph_store/age_store.py @@ -0,0 +1,1312 @@ +""" +Apache AGE Store Module + +This module provides Apache AGE (PostgreSQL graph extension) integration for +property graph storage and Cypher querying in the Semantica framework, supporting +full CRUD operations, transactions, and graph analytics. + +Apache AGE extends PostgreSQL with graph database functionality, enabling +hybrid relational + graph workloads using openCypher queries executed via SQL. + +Key Features: + - OpenCypher query language support via SQL wrapper + - Node and relationship CRUD operations + - Transaction support with explicit commit/rollback + - Parameterized queries to prevent SQL injection + - AGE internal ID / semantic ID separation + - Multi-label emulation (one AGE label + property array) + - Batch operations with progress tracking + - Optional dependency handling (psycopg2) + +Main Classes: + - ApacheAgeStore: Main AGE store for graph operations + +Example Usage: + >>> from semantica.graph_store.age_store import ApacheAgeStore + >>> store = ApacheAgeStore( + ... connection_string="host=localhost dbname=agedb user=postgres password=secret", + ... graph_name="semantica" + ... ) + >>> store.connect() + >>> node = store.create_node(labels=["Person"], properties={"name": "Alice"}) + >>> results = store.execute_query("MATCH (p:Person) RETURN p") + >>> store.close() + +Note: + - AGE auto-generates internal vertex/edge IDs. + - The ``node_id`` parameter in CRUD methods refers to the AGE internal ID. + - Semantic IDs can be stored in the ``semantica_id`` property. + - AGE supports exactly one label per vertex; additional labels are stored + in a ``labels`` property array. + +Author: Semantica Contributors +License: MIT +""" + +import json +import re +from typing import Any, Dict, List, Optional, Union + +from ..utils.exceptions import ProcessingError, ValidationError +from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker + +# Optional psycopg2 import +try: + import psycopg2 + import psycopg2.extras + + PSYCOPG2_AVAILABLE = True +except (ImportError, OSError): + PSYCOPG2_AVAILABLE = False + psycopg2 = None + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _sanitize_label(label: str) -> str: + """ + Sanitize a Cypher label to prevent injection. + + Only allows alphanumeric characters and underscores. + + Args: + label: Raw label string. + + Returns: + Sanitized label string. + + Raises: + ValidationError: If the label contains invalid characters. + """ + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", label): + raise ValidationError( + f"Invalid label '{label}': must start with a letter or underscore " + "and contain only alphanumeric characters and underscores." + ) + return label + + +def _sanitize_rel_type(rel_type: str) -> str: + """ + Sanitize a relationship type string. + + Args: + rel_type: Raw relationship type. + + Returns: + Sanitized relationship type. + + Raises: + ValidationError: If the type contains invalid characters. + """ + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", rel_type): + raise ValidationError( + f"Invalid relationship type '{rel_type}': must start with a letter or " + "underscore and contain only alphanumeric characters and underscores." + ) + return rel_type + + +def _props_to_cypher_literal(properties: Dict[str, Any]) -> str: + """ + Convert a Python dict to an AGE-compatible Cypher map literal. + + AGE does not support ``$param`` style parameter binding inside + ``cypher()`` calls, so property values must be inlined as literals + with proper escaping. + + Args: + properties: Dictionary of property key-value pairs. + + Returns: + Cypher map literal string, e.g. ``{name: 'Alice', age: 30}``. + """ + if not properties: + return "{}" + parts = [] + for key, value in properties.items(): + # Validate key + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): + raise ValidationError(f"Invalid property key: '{key}'") + parts.append(f"{key}: {_value_to_cypher_literal(value)}") + return "{" + ", ".join(parts) + "}" + + +def _value_to_cypher_literal(value: Any) -> str: + """ + Convert a single Python value to a Cypher literal string. + + Args: + value: Python value. + + Returns: + Cypher literal representation. + """ + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return repr(value) + if isinstance(value, str): + # Escape single quotes for Cypher strings + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + if isinstance(value, (list, tuple)): + inner = ", ".join(_value_to_cypher_literal(v) for v in value) + return f"[{inner}]" + if isinstance(value, dict): + return _props_to_cypher_literal(value) + # Fallback: convert to string + escaped = str(value).replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + +def _parse_agtype(raw: Any) -> Any: + """ + Parse an agtype value returned by AGE into a Python object. + + AGE returns results as ``agtype`` which may be a JSON-like string + with an optional ``::vertex`` / ``::edge`` / ``::path`` suffix. + + Args: + raw: Raw value from the cursor. + + Returns: + Parsed Python object (dict, list, or scalar). + """ + if raw is None: + return None + if not isinstance(raw, str): + return raw + + text = raw.strip() + + # Strip AGE type suffixes + for suffix in ("::vertex", "::edge", "::path", "::numeric", + "::integer", "::float", "::boolean", "::text"): + if text.endswith(suffix): + text = text[: -len(suffix)].strip() + break + + # Try JSON parse + try: + return json.loads(text) + except (json.JSONDecodeError, ValueError): + pass + + # Boolean literals + if text.lower() == "true": + return True + if text.lower() == "false": + return False + + # Numeric + try: + if "." in text: + return float(text) + return int(text) + except ValueError: + pass + + return text + + +def _vertex_to_node_dict(vertex: Any) -> Dict[str, Any]: + """ + Convert a parsed AGE vertex dict to the standard node return format. + + Expected vertex dict shape from AGE:: + + {"id": , "label": "