diff --git a/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb b/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb new file mode 100644 index 00000000..3a2afcfd --- /dev/null +++ b/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb @@ -0,0 +1,558 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.0" + } + }, + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/13_Manual_Ontology_Snowflake_Mapping.ipynb)\n", + "\n", + "# Manual Ontology + Snowflake Mapping\n", + "\n", + "This notebook answers a specific workflow:\n", + "\n", + "> *\"I want to design the ontology myself — not have AI infer it from my tables — and then map Snowflake data to it explicitly.\"*\n", + "\n", + "### What this notebook demonstrates\n", + "\n", + "| Step | What happens | Who controls it |\n", + "|---|---|---|\n", + "| 1 | Design ontology classes and properties | **You** (Python dict) |\n", + "| 2 | Model n-ary facts with reification | **You** (`AssociativeClassBuilder`) |\n", + "| 3 | Pull rows from Snowflake | Semantica `SnowflakeIngestor` |\n", + "| 4 | Map columns → ontology-aligned graph | **You** (explicit transform) |\n", + "| 5 | Validate + export OWL / SHACL | Semantica `OntologyEngine` |\n", + "| 6 | Load to triplet store and query | Semantica `TripletStore` |\n", + "\n", + "### What this notebook does NOT do\n", + "\n", + "- No LLM-driven ontology generation\n", + "- No schema introspection or table-to-class inference\n", + "- No \"suggest ontology from my data\"\n", + "\n", + "### Standards coverage\n", + "\n", + "| Feature | Status |\n", + "|---|---|\n", + "| OWL 2 (Turtle / RDF-XML) | Supported |\n", + "| SHACL 1.1 shapes | Supported |\n", + "| SPARQL 1.1 | Supported |\n", + "| Reification / n-ary facts | Supported via `AssociativeClassBuilder` |\n", + "| SPARQL 1.2 (reifier annotation, `LATERAL`) | Planned |\n", + "| SHACL 1.2 (`sh:severity` extensions, SHACL-AF) | Planned |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-1", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -qU semantica" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-2", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from typing import Any, Dict, List\n", + "\n", + "from semantica.ingest import SnowflakeIngestor\n", + "from semantica.kg.methods import build_kg\n", + "from semantica.ontology import AssociativeClassBuilder, OntologyEngine\n", + "from semantica.triplet_store import TripletStore" + ] + }, + { + "cell_type": "markdown", + "id": "cell-3", + "metadata": {}, + "source": [ + "## Step 1: Hand-Design the Ontology in Python\n", + "\n", + "You define every class and property explicitly. Nothing is read from Snowflake at this stage.\n", + "\n", + "**Design decisions that belong to you:**\n", + "- Which classes exist and what they mean\n", + "- Which properties are datatype vs. object properties\n", + "- Domain, range, and cardinality constraints\n", + "- Which properties are required (later enforced by SHACL)\n", + "\n", + "This dict versions with your code. It does not change when your database schema changes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-4", + "metadata": {}, + "outputs": [], + "source": [ + "BASE_URI = \"https://example.com/hr/\"\n", + "\n", + "# Your ontology — designed by you, not inferred by Semantica.\n", + "ontology: Dict[str, Any] = {\n", + " \"name\": \"EmploymentDomainOntology\",\n", + " \"uri\": f\"{BASE_URI}EmploymentDomainOntology\",\n", + " \"namespace\": {\"base_uri\": BASE_URI},\n", + "\n", + " # You decide the class taxonomy\n", + " \"classes\": [\n", + " {\"name\": \"Person\", \"uri\": f\"{BASE_URI}Person\"},\n", + " {\"name\": \"Organization\", \"uri\": f\"{BASE_URI}Organization\"},\n", + " {\"name\": \"Role\", \"uri\": f\"{BASE_URI}Role\"},\n", + " # EmploymentEvent is a reification node.\n", + " # It connects Person + Organization + Role and carries salary/date context.\n", + " {\"name\": \"EmploymentEvent\", \"uri\": f\"{BASE_URI}EmploymentEvent\"},\n", + " ],\n", + "\n", + " # You decide every property — type, domain, range, cardinality\n", + " \"properties\": [\n", + " # Datatype properties\n", + " {\"name\": \"name\", \"type\": \"datatype\", \"domain\": \"Person\", \"range\": \"string\", \"required\": True},\n", + " {\"name\": \"legalName\", \"type\": \"datatype\", \"domain\": \"Organization\", \"range\": \"string\", \"required\": True},\n", + " {\"name\": \"title\", \"type\": \"datatype\", \"domain\": \"Role\", \"range\": \"string\", \"required\": True},\n", + " {\"name\": \"startDate\", \"type\": \"datatype\", \"domain\": \"EmploymentEvent\", \"range\": \"date\"},\n", + " {\"name\": \"endDate\", \"type\": \"datatype\", \"domain\": \"EmploymentEvent\", \"range\": \"date\"},\n", + " {\"name\": \"salary\", \"type\": \"datatype\", \"domain\": \"EmploymentEvent\", \"range\": \"decimal\"},\n", + "\n", + " # Object properties — reification spokes (required)\n", + " {\"name\": \"employee\", \"type\": \"object\", \"domain\": \"EmploymentEvent\", \"range\": \"Person\", \"required\": True},\n", + " {\"name\": \"employer\", \"type\": \"object\", \"domain\": \"EmploymentEvent\", \"range\": \"Organization\", \"required\": True},\n", + " {\"name\": \"role\", \"type\": \"object\", \"domain\": \"EmploymentEvent\", \"range\": \"Role\", \"required\": True},\n", + "\n", + " # Shortcut edges — direct person→org / person→role without traversing the event node\n", + " {\"name\": \"worksFor\", \"type\": \"object\", \"domain\": \"Person\", \"range\": \"Organization\"},\n", + " {\"name\": \"hasRole\", \"type\": \"object\", \"domain\": \"Person\", \"range\": \"Role\"},\n", + " ],\n", + "}\n", + "\n", + "ontology" + ] + }, + { + "cell_type": "markdown", + "id": "cell-5", + "metadata": {}, + "source": [ + "## Step 2: Reification — Modeling N-Ary Facts\n", + "\n", + "**The problem with binary triples:**\n", + "A simple triple `(Alice, worksFor, Acme)` cannot carry extra context such as salary, start date, or role.\n", + "Standard RDF reification and OWL n-ary patterns solve this by introducing an intermediate node.\n", + "\n", + "Semantica's `AssociativeClassBuilder` is the Pythonic API for this pattern:\n", + "\n", + "```\n", + "EmploymentEvent\n", + " ├── employee → Person (required)\n", + " ├── employer → Organization (required)\n", + " ├── role → Role (required)\n", + " ├── startDate → xsd:date\n", + " ├── endDate → xsd:date\n", + " └── salary → xsd:decimal\n", + "```\n", + "\n", + "**On SPARQL 1.1 vs. SPARQL 1.2:**\n", + "- **SPARQL 1.1 (current):** traverse the event node explicitly — `?event hr:employee ?person ; hr:salary ?salary`\n", + "- **SPARQL 1.2 (planned):** the draft reifier annotation syntax allows attaching context to triples directly, without a separate intermediate node. Semantica will adopt this once the spec is ratified.\n", + "\n", + "**On SHACL 1.1 vs. SHACL 1.2:**\n", + "- **SHACL 1.1 (current):** `sh:NodeShape` + `sh:PropertyShape` constraints are exported for all `required` properties and enforced at load time.\n", + "- **SHACL 1.2 (planned):** `sh:severity` profile extensions and SHACL-AF rules are on the roadmap." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-6", + "metadata": {}, + "outputs": [], + "source": [ + "assoc_builder = AssociativeClassBuilder()\n", + "\n", + "employment_assoc = assoc_builder.create_associative_class(\n", + " name=\"EmploymentEvent\",\n", + " connects=[\"Person\", \"Organization\", \"Role\"],\n", + " temporal=True, # adds startDate / endDate handling\n", + " properties={\n", + " \"startDate\": \"xsd:date\",\n", + " \"endDate\": \"xsd:date\",\n", + " \"salary\": \"xsd:decimal\",\n", + " },\n", + ")\n", + "\n", + "validation_result = assoc_builder.validate_associative_class(employment_assoc)\n", + "\n", + "print(\"AssociativeClass structure:\")\n", + "print(f\" name: {employment_assoc.get('name')}\")\n", + "print(f\" connects: {employment_assoc.get('connects')}\")\n", + "print(f\" temporal: {employment_assoc.get('temporal')}\")\n", + "print(f\" properties: {list(employment_assoc.get('properties', {}).keys())}\")\n", + "print(f\"\\nValidation passed: {validation_result}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-7", + "metadata": {}, + "source": [ + "## Step 3: Ingest Snowflake Rows (Extraction Only)\n", + "\n", + "`SnowflakeIngestor` retrieves rows — nothing more. It does **not**:\n", + "- Inspect your table schema\n", + "- Suggest classes or properties\n", + "- Infer relationships from column names\n", + "\n", + "Set `USE_LIVE_SNOWFLAKE=true` plus the env vars below to connect to a real warehouse.\n", + "Otherwise the stub data is used." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-8", + "metadata": {}, + "outputs": [], + "source": [ + "def fetch_rows_from_snowflake() -> List[Dict[str, Any]]:\n", + " if os.getenv(\"USE_LIVE_SNOWFLAKE\", \"false\").lower() != \"true\":\n", + " return [\n", + " {\n", + " \"EMPLOYEE_ID\": \"E100\",\n", + " \"EMPLOYEE_NAME\": \"Alice Johnson\",\n", + " \"ORG_ID\": \"O10\",\n", + " \"ORG_NAME\": \"Acme Corp\",\n", + " \"ROLE_ID\": \"R7\",\n", + " \"ROLE_TITLE\": \"Senior Engineer\",\n", + " \"START_DATE\": \"2025-01-15\",\n", + " \"END_DATE\": None,\n", + " \"SALARY\": 160000,\n", + " },\n", + " {\n", + " \"EMPLOYEE_ID\": \"E101\",\n", + " \"EMPLOYEE_NAME\": \"Bob Singh\",\n", + " \"ORG_ID\": \"O10\",\n", + " \"ORG_NAME\": \"Acme Corp\",\n", + " \"ROLE_ID\": \"R9\",\n", + " \"ROLE_TITLE\": \"Data Architect\",\n", + " \"START_DATE\": \"2024-09-01\",\n", + " \"END_DATE\": None,\n", + " \"SALARY\": 185000,\n", + " },\n", + " ]\n", + "\n", + " ingestor = SnowflakeIngestor(\n", + " account=os.getenv(\"SNOWFLAKE_ACCOUNT\"),\n", + " user=os.getenv(\"SNOWFLAKE_USER\"),\n", + " password=os.getenv(\"SNOWFLAKE_PASSWORD\"),\n", + " warehouse=os.getenv(\"SNOWFLAKE_WAREHOUSE\"),\n", + " database=os.getenv(\"SNOWFLAKE_DATABASE\"),\n", + " schema=os.getenv(\"SNOWFLAKE_SCHEMA\", \"PUBLIC\"),\n", + " )\n", + " query = (\n", + " \"SELECT EMPLOYEE_ID, EMPLOYEE_NAME, \"\n", + " \"ORG_ID, ORG_NAME, ROLE_ID, ROLE_TITLE, \"\n", + " \"START_DATE, END_DATE, SALARY \"\n", + " \"FROM HR_EMPLOYMENT_FACT\"\n", + " )\n", + " data = ingestor.ingest_query(query)\n", + " ingestor.close()\n", + " return data.data\n", + "\n", + "\n", + "rows = fetch_rows_from_snowflake()\n", + "rows[:2]" + ] + }, + { + "cell_type": "markdown", + "id": "cell-9", + "metadata": {}, + "source": [ + "## Step 4: Map Rows to Ontology Concepts Explicitly\n", + "\n", + "This is the semantic transformation layer — the part that makes your ontology real.\n", + "\n", + "Semantica does not guess which column becomes which entity or property.\n", + "Every assignment is code you write and own:\n", + "\n", + "- **Stable node IDs** — deterministic, collision-safe, derived from business keys\n", + "- **Class assignment** — matches what you declared in Step 1\n", + "- **Property routing** — each column value goes to the correct ontology property\n", + "- **Reification wiring** — `EmploymentEvent` is linked to its three participants\n", + "\n", + "When your Snowflake schema changes, only this function needs updating. The ontology stays stable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "def map_rows_to_kg(rows: List[Dict[str, Any]]) -> Dict[str, Any]:\n", + " entities: Dict[str, Dict[str, Any]] = {}\n", + " relationships: List[Dict[str, Any]] = []\n", + "\n", + " for row in rows:\n", + " # Stable, deterministic node IDs derived from business keys\n", + " person_id = f\"person:{row['EMPLOYEE_ID']}\"\n", + " org_id = f\"org:{row['ORG_ID']}\"\n", + " role_id = f\"role:{row['ROLE_ID']}\"\n", + " # Event ID includes all three participants + start date so that\n", + " # a re-hired employee gets a distinct event node, not an overwrite.\n", + " event_id = f\"employment:{row['EMPLOYEE_ID']}:{row['ORG_ID']}:{row['START_DATE']}\"\n", + "\n", + " # Entities — \"type\" must match a class name from Step 1\n", + " entities[person_id] = {\n", + " \"id\": person_id,\n", + " \"type\": \"Person\",\n", + " \"properties\": {\"name\": row[\"EMPLOYEE_NAME\"]},\n", + " }\n", + " entities[org_id] = {\n", + " \"id\": org_id,\n", + " \"type\": \"Organization\",\n", + " \"properties\": {\"legalName\": row[\"ORG_NAME\"]},\n", + " }\n", + " entities[role_id] = {\n", + " \"id\": role_id,\n", + " \"type\": \"Role\",\n", + " \"properties\": {\"title\": row[\"ROLE_TITLE\"]},\n", + " }\n", + " # Reification node — carries the n-ary context\n", + " entities[event_id] = {\n", + " \"id\": event_id,\n", + " \"type\": \"EmploymentEvent\",\n", + " \"properties\": {\n", + " \"startDate\": row[\"START_DATE\"],\n", + " \"endDate\": row[\"END_DATE\"], # None = still employed\n", + " \"salary\": row[\"SALARY\"],\n", + " },\n", + " }\n", + "\n", + " relationships.extend([\n", + " # Shortcut edges — fast SPARQL when context is not needed\n", + " {\"source\": person_id, \"target\": org_id, \"type\": \"worksFor\"},\n", + " {\"source\": person_id, \"target\": role_id, \"type\": \"hasRole\"},\n", + " # Reification spokes — full context via the event node\n", + " {\"source\": event_id, \"target\": person_id, \"type\": \"employee\"},\n", + " {\"source\": event_id, \"target\": org_id, \"type\": \"employer\"},\n", + " {\"source\": event_id, \"target\": role_id, \"type\": \"role\"},\n", + " ])\n", + "\n", + " return build_kg([{\"entities\": list(entities.values()), \"relationships\": relationships}])\n", + "\n", + "\n", + "kg = map_rows_to_kg(rows)\n", + "print(f\"Entities built: {len(kg.get('entities', []))}\")\n", + "print(f\"Relationships built: {len(kg.get('relationships', []))}\")\n", + "\n", + "sample = next((e for e in kg[\"entities\"] if e[\"type\"] == \"EmploymentEvent\"), None)\n", + "print(f\"\\nSample EmploymentEvent node: {sample}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "## Step 5: Validate Ontology and Export OWL + SHACL\n", + "\n", + "`OntologyEngine` validates your ontology dict and serialises it to standards-compliant files.\n", + "\n", + "**Output files:**\n", + "- `employment_manual_ontology.ttl` — OWL 2 Turtle\n", + "- `employment_manual_shapes.ttl` — SHACL 1.1 node and property shapes\n", + "\n", + "**Standards status:**\n", + "\n", + "| Standard | Semantica support |\n", + "|---|---|\n", + "| SPARQL 1.1 | Full |\n", + "| SHACL 1.1 (`sh:NodeShape`, `sh:PropertyShape`, `sh:minCount`, `sh:datatype`, `sh:class`) | Full |\n", + "| SPARQL 1.2 (reifier annotation syntax, `LATERAL`) | Tracked — not yet implemented |\n", + "| SHACL 1.2 (`sh:severity` profiles, SHACL-AF extensions) | Tracked — not yet implemented |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "engine = OntologyEngine(base_uri=BASE_URI)\n", + "\n", + "validation = engine.validate(ontology)\n", + "owl_ttl = engine.to_owl(ontology, format=\"turtle\")\n", + "shacl_ttl = engine.to_shacl(ontology, format=\"turtle\")\n", + "\n", + "engine.export_owl(ontology, \"employment_manual_ontology.ttl\", format=\"turtle\")\n", + "engine.export_shacl(ontology, \"employment_manual_shapes.ttl\", format=\"turtle\")\n", + "\n", + "print(f\"Ontology valid: {validation.valid}\")\n", + "print(f\"Ontology consistent: {validation.consistent}\")\n", + "print(f\"OWL output: {len(owl_ttl):,} chars → employment_manual_ontology.ttl\")\n", + "print(f\"SHACL output: {len(shacl_ttl):,} chars → employment_manual_shapes.ttl\")\n", + "\n", + "print(\"\\n--- SHACL shapes (first 20 lines) ---\")\n", + "print(\"\\n\".join(shacl_ttl.splitlines()[:20]))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "## Best-Practice Architecture\n", + "\n", + "```\n", + "┌──────────────────────────────────┐\n", + "│ Ontology as code (Python dict) │ ← versioned alongside your application\n", + "│ + AssociativeClass for n-ary │\n", + "└───────────────┬──────────────────┘\n", + " │ validate + export\n", + " ▼\n", + "┌───────────────────────────────────┐\n", + "│ OWL 2 Turtle │ SHACL 1.1 │ ← standards-compliant artifacts\n", + "└───────────────┬───────────────────┘\n", + " │\n", + " ▼\n", + "┌──────────────────────────────────┐\n", + "│ Snowflake — raw data access │ ← no schema introspection\n", + "└───────────────┬──────────────────┘\n", + " │ explicit mapping layer\n", + " ▼\n", + "┌──────────────────────────────────┐\n", + "│ Ontology-aligned KG │ ← types, IDs, edges match Step 1\n", + "└───────────────┬──────────────────┘\n", + " │ optional\n", + " ▼\n", + "┌──────────────────────────────────┐\n", + "│ Triplet store + SPARQL 1.1 │\n", + "└──────────────────────────────────┘\n", + "```\n", + "\n", + "**Why this split matters:**\n", + "If Semantica inferred the ontology from your Snowflake schema, every schema migration would risk silently changing your semantic model.\n", + "With this pattern, schema changes only touch the mapping function in Step 4 — the ontology remains stable and under your control." + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "## SPARQL Query Patterns\n", + "\n", + "Two query styles are available because we wrote both shortcut edges and reification spokes.\n", + "\n", + "### Simple lookup — shortcut edge (no context needed)\n", + "\n", + "```sparql\n", + "PREFIX hr: \n", + "\n", + "SELECT ?personName ?orgName\n", + "WHERE {\n", + " ?person a hr:Person ;\n", + " hr:name ?personName ;\n", + " hr:worksFor ?org .\n", + " ?org hr:legalName ?orgName .\n", + "}\n", + "```\n", + "\n", + "### Contextual lookup — via reification node (salary, dates, role)\n", + "\n", + "```sparql\n", + "PREFIX hr: \n", + "\n", + "SELECT ?personName ?roleTitle ?salary ?startDate\n", + "WHERE {\n", + " ?event a hr:EmploymentEvent ;\n", + " hr:employee ?person ;\n", + " hr:role ?role ;\n", + " hr:salary ?salary ;\n", + " hr:startDate ?startDate .\n", + " ?person hr:name ?personName .\n", + " ?role hr:title ?roleTitle .\n", + "}\n", + "ORDER BY DESC(?salary)\n", + "```\n", + "\n", + "### Future: SPARQL 1.2 reifier syntax\n", + "\n", + "The SPARQL 1.2 draft introduces annotation syntax that lets you attach context directly to triples, without a separate intermediate node.\n", + "Once the spec is ratified Semantica will adopt it, and the contextual query above may be expressible more concisely." + ] + }, + { + "cell_type": "markdown", + "id": "cell-15", + "metadata": {}, + "source": [ + "## Step 6 (Optional): Load to Triplet Store and Run SPARQL\n", + "\n", + "Set `STORE_TO_TRIPLET=true` to load the KG into a live triplet store and run the contextual reification query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-16", + "metadata": {}, + "outputs": [], + "source": [ + "if os.getenv(\"STORE_TO_TRIPLET\", \"false\").lower() == \"true\":\n", + " store = TripletStore(\n", + " backend=os.getenv(\"TRIPLET_BACKEND\", \"blazegraph\"),\n", + " endpoint=os.getenv(\"TRIPLET_ENDPOINT\", \"http://localhost:9999/blazegraph\"),\n", + " namespace=os.getenv(\"TRIPLET_NAMESPACE\", \"kb\"),\n", + " )\n", + " store_result = store.store(knowledge_graph=kg, ontology=ontology)\n", + " print(\"Store result:\", store_result)\n", + "\n", + " # Contextual reification query — person + role + salary via EmploymentEvent\n", + " query = \"\"\"\n", + " PREFIX hr: \n", + "\n", + " SELECT ?personName ?roleTitle ?salary ?startDate\n", + " WHERE {\n", + " ?event a hr:EmploymentEvent ;\n", + " hr:employee ?person ;\n", + " hr:role ?role ;\n", + " hr:salary ?salary ;\n", + " hr:startDate ?startDate .\n", + " ?person hr:name ?personName .\n", + " ?role hr:title ?roleTitle .\n", + " }\n", + " ORDER BY DESC(?salary)\n", + " LIMIT 10\n", + " \"\"\"\n", + " result = store.execute_query(query)\n", + " print(result)\n", + "else:\n", + " print(\"Skipping triplet-store load/query (set STORE_TO_TRIPLET=true to enable)\")" + ] + } + ] +}