diff --git a/cookbook/advanced/08_Reasoning_and_Inference.ipynb b/cookbook/advanced/08_Reasoning_and_Inference.ipynb index a168ecb1..b1a5804a 100644 --- a/cookbook/advanced/08_Reasoning_and_Inference.ipynb +++ b/cookbook/advanced/08_Reasoning_and_Inference.ipynb @@ -1,278 +1,203 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n", - "\n", - "# Reasoning and Inference\n", - "\n", - "## Overview\n", - "\n", - "Build knowledge graphs, define rules, perform forward/backward chaining, and generate explanations for AI reasoning.\n", - "\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/reasoning/)\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica from PyPI:\n", - "\n", - "```bash\n", - "pip install semantica\n", - "# Or with all optional dependencies:\n", - "pip install semantica[all]\n", - "```\n", - "\n", - "## Workflow: Build KG → Define Rules → Forward/Backward Chaining → Generate Explanations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphBuilder\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Build Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "builder = GraphBuilder()\n", - "\n", - "entities = [\n", - " {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n", - " {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n", - " {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n", - " {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n", - " {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"},\n", - "]\n", - "\n", - "relationships = [\n", - " {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n", - " {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n", - " {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n", - " {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"},\n", - "]\n", - "\n", - "knowledge_graph = builder.build(entities, relationships)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Define Rules\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class RuleManager:\n", - " def __init__(self):\n", - " self.rules = []\n", - " \n", - " def add_rules(self, rules):\n", - " self.rules.extend(rules)\n", - "\n", - "rule_manager = RuleManager()\n", - "\n", - "rules = [\n", - " \"IF A is parent_of B AND B is parent_of C THEN A is grandparent_of C\",\n", - " \"IF X is located_in Y AND Y is part_of Z THEN X is located_in Z\",\n", - " \"IF X lives_in Y AND Y is located_in Z THEN X lives_in Z\"\n", - "]\n", - "\n", - "rule_manager.add_rules(rules)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Forward Chaining\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class InferenceEngine:\n", - " def forward_chain(self, kg, rule_manager):\n", - " new_facts = []\n", - " \n", - " for rule in rule_manager.rules:\n", - " if \"grandparent_of\" in rule:\n", - " parents = [r for r in relationships if r[\"type\"] == \"parent_of\"]\n", - " for p1 in parents:\n", - " for p2 in parents:\n", - " if p1[\"target\"] == p2[\"source\"]:\n", - " new_fact = {\n", - " \"source\": p1[\"source\"],\n", - " \"target\": p2[\"target\"],\n", - " \"type\": \"grandparent_of\",\n", - " \"inferred\": True\n", - " }\n", - " if new_fact not in new_facts:\n", - " new_facts.append(new_fact)\n", - " \n", - " elif \"lives_in\" in rule and \"located_in\" in rule:\n", - " lives_in = [r for r in relationships if r[\"type\"] == \"lives_in\"]\n", - " located_in = [r for r in relationships if r[\"type\"] == \"located_in\"]\n", - " \n", - " for live in lives_in:\n", - " for loc in located_in:\n", - " if live[\"target\"] == loc[\"source\"]:\n", - " new_fact = {\n", - " \"source\": live[\"source\"],\n", - " \"target\": loc[\"target\"],\n", - " \"type\": \"lives_in\",\n", - " \"inferred\": True\n", - " }\n", - " if new_fact not in new_facts:\n", - " new_facts.append(new_fact)\n", - " \n", - " return new_facts\n", - "\n", - "inference_engine = InferenceEngine()\n", - "new_facts = inference_engine.forward_chain(knowledge_graph, rule_manager)\n", - "\n", - "for fact in new_facts:\n", - " print(f\"{fact['source']} {fact['type']} {fact['target']} (inferred)\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Backward Chaining\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def backward_chain(kg, rule_manager, goal):\n", - " proof_steps = []\n", - " \n", - " goal_source, goal_type, goal_target = goal\n", - " \n", - " for rel in relationships:\n", - " if rel[\"source\"] == goal_source and rel[\"type\"] == goal_type and rel[\"target\"] == goal_target:\n", - " proof_steps.append({\n", - " \"step\": \"Direct fact\",\n", - " \"fact\": f\"{goal_source} {goal_type} {goal_target}\",\n", - " \"source\": \"knowledge_graph\"\n", - " })\n", - " return proof_steps\n", - " \n", - " if goal_type == \"grandparent_of\":\n", - " for rel1 in relationships:\n", - " if rel1[\"source\"] == goal_source and rel1[\"type\"] == \"parent_of\":\n", - " intermediate = rel1[\"target\"]\n", - " for rel2 in relationships:\n", - " if rel2[\"source\"] == intermediate and rel2[\"type\"] == \"parent_of\" and rel2[\"target\"] == goal_target:\n", - " proof_steps.append({\n", - " \"step\": \"Rule application\",\n", - " \"fact\": f\"{goal_source} parent_of {intermediate}\",\n", - " \"source\": \"knowledge_graph\"\n", - " })\n", - " proof_steps.append({\n", - " \"step\": \"Rule application\",\n", - " \"fact\": f\"{intermediate} parent_of {goal_target}\",\n", - " \"source\": \"knowledge_graph\"\n", - " })\n", - " proof_steps.append({\n", - " \"step\": \"Inference\",\n", - " \"fact\": f\"{goal_source} grandparent_of {goal_target}\",\n", - " \"source\": \"inference_rule\"\n", - " })\n", - " return proof_steps\n", - " \n", - " return proof_steps\n", - "\n", - "goal = (\"alice\", \"grandparent_of\", \"charlie\")\n", - "proof = backward_chain(knowledge_graph, rule_manager, goal)\n", - "\n", - "for i, step in enumerate(proof, 1):\n", - " print(f\"Step {i}: {step['step']} - {step['fact']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Generate Explanations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class ExplanationGenerator:\n", - " def generate(self, proof, kg):\n", - " if not proof:\n", - " return \"No proof found for the given goal.\"\n", - " \n", - " explanation_parts = []\n", - " explanation_parts.append(\"Explanation:\")\n", - " \n", - " for i, step in enumerate(proof, 1):\n", - " if step['step'] == 'Direct fact':\n", - " explanation_parts.append(f\"{i}. We know that {step['fact']} from the knowledge graph.\")\n", - " elif step['step'] == 'Rule application':\n", - " explanation_parts.append(f\"{i}. From the knowledge graph: {step['fact']}.\")\n", - " elif step['step'] == 'Inference':\n", - " explanation_parts.append(f\"{i}. Therefore, by applying the inference rule: {step['fact']}.\")\n", - " \n", - " return \"\\n\".join(explanation_parts)\n", - "\n", - "explanation_gen = ExplanationGenerator()\n", - "explanation = explanation_gen.generate(proof, knowledge_graph)\n", - "print(explanation)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Reasoning and inference workflow:\n", - "- Knowledge Graph Built\n", - "- Inference Rules Defined\n", - "- Forward Chaining Performed\n", - "- Backward Chaining Performed\n", - "- Explanations Generated\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n", + "\n", + "# Reasoning and Inference\n", + "\n", + "## Overview\n", + "\n", + "Build knowledge graphs, define rules, perform forward/backward chaining, and generate explanations for AI reasoning using the **Semantica Reasoning Module**.\n", + "\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/reasoning/)\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica from PyPI:\n", + "\n", + "```bash\n", + "pip install semantica\n", + "# Or with all optional dependencies:\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "## Workflow: Build KG → Define Rules → Forward/Backward Chaining → Generate Explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Build Knowledge Graph\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n", + " {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n", + " {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n", + " {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n", + " {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"},\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n", + " {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n", + " {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n", + " {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"},\n", + "]\n", + "\n", + "knowledge_graph = builder.build([{\"entities\": entities, \"relationships\": relationships}])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Define Rules\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Inference Engine\n", + "engine = InferenceEngine()\n", + "\n", + "# Define rules using logic syntax\n", + "rules = [\n", + " \"IF parent_of(?a, ?b) AND parent_of(?b, ?c) THEN grandparent_of(?a, ?c)\",\n", + " \"IF lives_in(?x, ?y) AND located_in(?y, ?z) THEN lives_in(?x, ?z)\"\n", + "]\n", + "\n", + "for rule in rules:\n", + " engine.add_rule(rule)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Forward Chaining\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load facts from relationships into the engine\n", + "for rel in relationships:\n", + " # Format: predicate(subject, object)\n", + " fact_str = f\"{rel['type']}({rel['source']}, {rel['target']})\"\n", + " engine.add_fact(fact_str)\n", + "\n", + "# Perform forward chaining to derive new facts\n", + "results = engine.forward_chain()\n", + "\n", + "print(f\"Inferred {len(results)} new facts:\")\n", + "for result in results:\n", + " print(f\" - {result.conclusion} (Rule: {result.rule_used.name})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Backward Chaining\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a goal to prove\n", + "goal = \"grandparent_of(alice, charlie)\"\n", + "\n", + "# Perform backward chaining\n", + "proof = engine.backward_chain(goal)\n", + "\n", + "if proof:\n", + " print(f\"Goal '{goal}' proven successfully!\")\n", + "else:\n", + " print(f\"Could not prove goal '{goal}'.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generator = ExplanationGenerator()\n", + "\n", + "# Explain the last forward chaining inference\n", + "if results:\n", + " explanation = generator.generate_explanation(results[0])\n", + " print(\"Explanation for first inferred fact:\")\n", + " print(explanation.natural_language)\n", + "\n", + "# If we have a proof from backward chaining, explain it\n", + "if proof:\n", + " proof_explanation = generator.generate_explanation(proof)\n", + " print(\"\\nExplanation for backward chaining proof:\")\n", + " print(proof_explanation.natural_language)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Reasoning and inference workflow:\n", + "- Knowledge Graph Built\n", + "- Inference Rules Defined\n", + "- Facts Loaded into Engine\n", + "- Forward Chaining Performed\n", + "- Backward Chaining Performed\n", + "- Explanations Generated\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/cookbook/introduction/01_Welcome_to_Semantica.ipynb b/cookbook/introduction/01_Welcome_to_Semantica.ipynb index df5a8464..e09467f5 100644 --- a/cookbook/introduction/01_Welcome_to_Semantica.ipynb +++ b/cookbook/introduction/01_Welcome_to_Semantica.ipynb @@ -279,7 +279,7 @@ "from semantica.reasoning import InferenceEngine, RuleManager\n", "inference_engine = InferenceEngine()\n", "rule_manager = RuleManager()\n", - "new_facts = inference_engine.forward_chain(kg, rule_manager)\n", + "new_facts = inference_engine.forward_chain()\n", "```\n", "\n", "### 10. ONTOLOGY MODULE - Ontology Generation\n", @@ -654,4 +654,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb b/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb index 300f5356..0536249a 100644 --- a/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb +++ b/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb @@ -1,1724 +1,1724 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n", - "\n", - "# GraphRAG Complete - End-to-End Pipeline\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates a **complete end-to-end GraphRAG (Graph-based Retrieval Augmented Generation) system** using Semantica framework. It showcases how to build a production-ready GraphRAG system that combines vector search with knowledge graph traversal for enhanced retrieval and question answering.\n", - "\n", - "**Key Features:**\n", - "\n", - "- **Real-World Data**: Uses actual data sources via MCP servers, web scraping, and RSS feeds (NO mock data)\n", - "- **Complete Pipeline**: From data ingestion to LLM-powered question answering\n", - "- **Hybrid Retrieval**: Combines vector similarity search with knowledge graph traversal\n", - "- **Multi-hop Reasoning**: Follows relationships across the graph for deeper context\n", - "- **20+ Semantica Modules**: Demonstrates comprehensive use of the framework\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/concepts/) • [GraphRAG Guide](https://semantica.readthedocs.io/concepts/)\n", - "\n", - "### What You'll Learn\n", - "\n", - "- How to ingest real-world data from multiple sources (MCP, web, feeds)\n", - "- How to build knowledge graphs from unstructured text\n", - "- How to implement hybrid search combining vectors and graphs\n", - "- How to use ContextRetriever for intelligent context expansion\n", - "- How to integrate LLMs with GraphRAG for question answering\n", - "- How to visualize and export knowledge graphs\n", - "\n", - "### Pipeline Overview\n", - "\n", - "**Real-World Data Sources (MCP/Web/Feeds) → Parse → Extract Entities & Relationships → Build Knowledge Graph → Generate Embeddings → Vector Store → Hybrid Search → Context Retrieval → GraphRAG Query System → LLM Integration → Answer Generation**\n", - "\n", - "---\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica from PyPI:\n", - "\n", - "```bash\n", - "pip install semantica\n", - "\n", - "# Or with all optional dependencies:\n", - "pip install semantica[all]\n", - "```\n", - "\n", - "### Additional Dependencies\n", - "\n", - "```bash\n", - "pip install openai anthropic # For LLM integration\n", - "pip install jupyter # For running this notebook\n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Setup and Import Semantica Modules\n", - "\n", - "Import all necessary Semantica modules for the complete GraphRAG pipeline. This includes modules for ingestion, parsing, extraction, graph building, embeddings, vector storage, context retrieval, and more.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.core import ConfigManager, Semantica\n", - "\n", - "# Initialize configuration manager\n", - "config_manager = ConfigManager()\n", - "\n", - "# Optionally load from file or dictionary\n", - "# config = config_manager.load_from_file(\"config.yaml\")\n", - "# Or use defaults\n", - "config = config_manager.load_from_dict({})\n", - "\n", - "# Initialize Semantica framework\n", - "framework = Semantica(config=config)\n", - "framework.initialize()\n", - "\n", - "print(\"Configuration and framework initialized\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Ingest Real-World Data from Multiple Sources\n", - "\n", - "1. **MCP Servers** (Primary): Connect to real MCP servers providing news feeds, documentation, APIs\n", - "2. **Web Sources**: Scrape real web content from news sites and documentation\n", - "3. **RSS Feeds**: Ingest real RSS/Atom feeds from news sources\n", - "\n", - "### 2.1: Connect to MCP Servers\n", - "\n", - "Connect to real MCP servers via URL. MCP servers can provide resources (databases, files) and tools (APIs, queries).\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import MCPIngestor\n", - "\n", - "mcp_ingestor = MCPIngestor()\n", - "\n", - "connected_servers = mcp_ingestor.get_connected_servers()\n", - "print(f\"Connected MCP Servers: {len(connected_servers)}\")\n", - "for server_name in connected_servers:\n", - " print(f\" - {server_name}\")\n", - "\n", - "if connected_servers:\n", - " for server_name in connected_servers:\n", - " try:\n", - " resources = mcp_ingestor.list_available_resources(server_name)\n", - " tools = mcp_ingestor.list_available_tools(server_name)\n", - " print(f\"\\n{server_name}:\")\n", - " print(f\" Resources: {len(resources)}\")\n", - " print(f\" Tools: {len(tools)}\")\n", - " except Exception as e:\n", - " print(f\"Error connecting to {server_name}: {e}\")\n", - "\n", - "print(\"\\nNote: Configure your MCP server URLs above to ingest real data\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.2: Ingest Data from MCP Servers\n", - "\n", - "Ingest real data from MCP server resources and tools.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import WebIngestor, FeedIngestor\n", - "\n", - "all_documents = []\n", - "\n", - "if connected_servers:\n", - " for server_name in connected_servers:\n", - " try:\n", - " print(f\"\\nIngesting from {server_name}...\")\n", - " mcp_data = mcp_ingestor.ingest_all_resources(server_name)\n", - " \n", - " if isinstance(mcp_data, list):\n", - " all_documents.extend(mcp_data)\n", - " print(f\" Ingested {len(mcp_data)} resources\")\n", - " else:\n", - " all_documents.append(mcp_data)\n", - " print(f\" Ingested 1 resource\")\n", - " \n", - " except Exception as e:\n", - " print(f\" Error ingesting from {server_name}: {e}\")\n", - "\n", - "print(f\"\\nTotal documents from MCP: {len(all_documents)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.3: Ingest Data from Web Sources\n", - "\n", - "Scrape real web content from news sites, documentation, and articles.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "web_ingestor = WebIngestor()\n", - "\n", - "web_sources = []\n", - "\n", - "web_documents = []\n", - "for url in web_sources:\n", - " try:\n", - " print(f\"Scraping {url}...\")\n", - " docs = web_ingestor.ingest(url)\n", - " if isinstance(docs, list):\n", - " web_documents.extend(docs)\n", - " else:\n", - " web_documents.append(docs)\n", - " print(f\" Scraped {len(docs) if isinstance(docs, list) else 1} document(s)\")\n", - " except Exception as e:\n", - " print(f\" Error scraping {url}: {e}\")\n", - "\n", - "all_documents.extend(web_documents)\n", - "print(f\"\\nTotal documents from web: {len(web_documents)}\")\n", - "print(f\"Total documents so far: {len(all_documents)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.4: Ingest Data from RSS Feeds\n", - "\n", - "Ingest real RSS/Atom feeds from news sources.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "feed_ingestor = FeedIngestor()\n", - "\n", - "feed_urls = []\n", - "\n", - "feed_documents = []\n", - "for feed_url in feed_urls:\n", - " try:\n", - " print(f\"Fetching feed {feed_url}...\")\n", - " feeds = feed_ingestor.ingest(feed_url)\n", - " if isinstance(feeds, list):\n", - " feed_documents.extend(feeds)\n", - " else:\n", - " feed_documents.append(feeds)\n", - " print(f\" Fetched {len(feeds) if isinstance(feeds, list) else 1} feed item(s)\")\n", - " except Exception as e:\n", - " print(f\" Error fetching feed {feed_url}: {e}\")\n", - "\n", - "all_documents.extend(feed_documents)\n", - "print(f\"\\nTotal documents from feeds: {len(feed_documents)}\")\n", - "print(f\"Total documents collected: {len(all_documents)}\")\n", - "\n", - "if len(all_documents) == 0:\n", - " print(\"\\nNo documents collected. Please configure MCP servers, web URLs, or RSS feeds above.\")\n", - " print(\"For this demonstration, we'll continue with the pipeline structure.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Document Processing Pipeline\n", - "\n", - "Process the ingested documents: parse, split, and normalize the text for extraction.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.parse import DocumentParser, MCPParser\n", - "\n", - "document_parser = DocumentParser()\n", - "mcp_parser = MCPParser()\n", - "\n", - "parsed_documents = []\n", - "\n", - "for doc in all_documents:\n", - " try:\n", - " if hasattr(doc, 'source') and 'mcp' in doc.source.lower():\n", - " parsed = mcp_parser.parse(doc)\n", - " else:\n", - " parsed = document_parser.parse(doc)\n", - " \n", - " if isinstance(parsed, list):\n", - " parsed_documents.extend(parsed)\n", - " else:\n", - " parsed_documents.append(parsed)\n", - " except Exception as e:\n", - " print(f\"Error parsing document: {e}\")\n", - " continue\n", - "\n", - "print(f\"Parsed {len(parsed_documents)} documents\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.2: Split Documents Using Dual Chunking Strategy\n", - "\n", - "For GraphRAG, we use **two different chunking methods** optimized for different stores:\n", - "\n", - "**For Vector Store** (semantic similarity search):\n", - "- **Semantic Chunking**: Uses embeddings to find natural semantic boundaries\n", - "- Better for vector similarity search and retrieval\n", - "\n", - "**For Graph Store** (knowledge structure preservation):\n", - "- **Entity-Aware Chunking**: Preserves entity boundaries (prevents splitting entities)\n", - "- **Relation-Aware Chunking**: Preserves relationship triples (keeps subject-predicate-object together)\n", - "- **Graph-Based Chunking**: Uses existing graph structure for optimal chunking\n", - "\n", - "We'll create chunks optimized for each store type.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.split import (\n", - " SemanticChunker, EntityAwareChunker, RelationAwareChunker, GraphBasedChunker\n", - ")\n", - "import numpy as np\n", - "\n", - "print(\"Step 1: Creating chunks for Vector Store (semantic chunking)...\")\n", - "semantic_chunker = SemanticChunker(\n", - " chunk_size=1000,\n", - " chunk_overlap=200\n", - ")\n", - "\n", - "vector_store_chunks = []\n", - "for i, doc in enumerate(parsed_documents):\n", - " doc_text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", - " if doc_text.strip():\n", - " chunks = semantic_chunker.chunk(doc_text)\n", - " if isinstance(chunks, list):\n", - " for chunk in chunks:\n", - " if hasattr(chunk, 'metadata'):\n", - " chunk.metadata['chunking_method'] = 'semantic'\n", - " chunk.metadata['store_type'] = 'vector'\n", - " chunk.metadata['source_doc'] = i\n", - " vector_store_chunks.extend(chunks)\n", - " else:\n", - " if hasattr(chunks, 'metadata'):\n", - " chunks.metadata['chunking_method'] = 'semantic'\n", - " chunks.metadata['store_type'] = 'vector'\n", - " chunks.metadata['source_doc'] = i\n", - " vector_store_chunks.append(chunks)\n", - "\n", - "print(f\"Created {len(vector_store_chunks)} semantic chunks for vector store\")\n", - "\n", - "print(\"\\nStep 2: Extracting entities/relationships for graph-aware chunking...\")\n", - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "\n", - "doc_entities = {}\n", - "doc_relationships = {}\n", - "\n", - "for i, doc in enumerate(parsed_documents):\n", - " doc_text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", - " if doc_text.strip():\n", - " entities = ner_extractor.extract(doc_text)\n", - " if isinstance(entities, list):\n", - " doc_entities[i] = entities\n", - " else:\n", - " doc_entities[i] = [entities] if entities else []\n", - " \n", - " relationships = relation_extractor.extract(doc_text, doc_entities[i])\n", - " if isinstance(relationships, list):\n", - " doc_relationships[i] = relationships\n", - " else:\n", - " doc_relationships[i] = [relationships] if relationships else []\n", - "\n", - "print(f\"Extracted entities from {len(doc_entities)} documents\")\n", - "print(f\"Extracted relationships from {len(doc_relationships)} documents\")\n", - "\n", - "print(\"\\nStep 3: Creating chunks for Graph Store (graph-aware chunking)...\")\n", - "entity_chunker = EntityAwareChunker(\n", - " chunk_size=1000,\n", - " chunk_overlap=200,\n", - " ner_method=\"spacy\",\n", - " preserve_entities=True\n", - ")\n", - "\n", - "relation_chunker = RelationAwareChunker(\n", - " chunk_size=1000,\n", - " chunk_overlap=200,\n", - " preserve_triples=True\n", - ")\n", - "\n", - "graph_store_chunks = []\n", - "\n", - "for i, doc in enumerate(parsed_documents):\n", - " doc_text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", - " if not doc_text.strip():\n", - " continue\n", - " \n", - " if i in doc_relationships and len(doc_relationships[i]) > 0:\n", - " chunks = relation_chunker.chunk(\n", - " doc_text,\n", - " relationships=doc_relationships[i]\n", - " )\n", - " elif i in doc_entities and len(doc_entities[i]) > 0:\n", - " chunks = entity_chunker.chunk(\n", - " doc_text,\n", - " entities=doc_entities[i]\n", - " )\n", - " else:\n", - " chunks = entity_chunker.chunk(doc_text)\n", - " \n", - " if isinstance(chunks, list):\n", - " for chunk in chunks:\n", - " if hasattr(chunk, 'metadata'):\n", - " chunk.metadata['chunking_method'] = 'graph_aware'\n", - " chunk.metadata['store_type'] = 'graph'\n", - " chunk.metadata['source_doc'] = i\n", - " if i in doc_entities:\n", - " chunk.metadata['entities'] = doc_entities[i]\n", - " if i in doc_relationships:\n", - " chunk.metadata['relationships'] = doc_relationships[i]\n", - " graph_store_chunks.extend(chunks)\n", - " else:\n", - " if hasattr(chunks, 'metadata'):\n", - " chunks.metadata['chunking_method'] = 'graph_aware'\n", - " chunks.metadata['store_type'] = 'graph'\n", - " chunks.metadata['source_doc'] = i\n", - " graph_store_chunks.append(chunks)\n", - "\n", - "print(f\"Created {len(graph_store_chunks)} graph-aware chunks for graph store\")\n", - "\n", - "chunked_documents = vector_store_chunks + graph_store_chunks\n", - "print(f\"\\nTotal chunks: {len(chunked_documents)}\")\n", - "print(f\" Vector store chunks: {len(vector_store_chunks)}\")\n", - "print(f\" Graph store chunks: {len(graph_store_chunks)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.2.1: Graph-Based Chunking (Iterative Refinement)\n", - "\n", - "After building the knowledge graph, we can use graph-based chunking to refine chunks based on graph structure. This is useful for re-chunking or optimizing existing chunks.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "graph_chunker = GraphBasedChunker(\n", - " chunk_size=1000,\n", - " chunk_overlap=200,\n", - " strategy=\"community\",\n", - " algorithm=\"louvain\"\n", - ")\n", - "\n", - "print(\"Graph-based chunker initialized\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.3: Normalize Text\n", - "\n", - "Clean and normalize text for better extraction quality.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.normalize import TextNormalizer\n", - "\n", - "text_normalizer = TextNormalizer()\n", - "\n", - "print(\"Normalizing vector store chunks...\")\n", - "normalized_vector_chunks = []\n", - "for chunk in vector_store_chunks:\n", - " normalized = text_normalizer.normalize_text(chunk)\n", - " if isinstance(normalized, list):\n", - " normalized_vector_chunks.extend(normalized)\n", - " else:\n", - " normalized_vector_chunks.append(normalized)\n", - "\n", - "print(\"Normalizing graph store chunks...\")\n", - "normalized_graph_chunks = []\n", - "for chunk in graph_store_chunks:\n", - " normalized = text_normalizer.normalize_text(chunk)\n", - " if isinstance(normalized, list):\n", - " normalized_graph_chunks.extend(normalized)\n", - " else:\n", - " normalized_graph_chunks.append(normalized)\n", - "\n", - "normalized_documents = normalized_vector_chunks + normalized_graph_chunks\n", - "print(f\"Normalized {len(normalized_documents)} chunks\")\n", - "print(f\" Vector store chunks: {len(normalized_vector_chunks)}\")\n", - "print(f\" Graph store chunks: {len(normalized_graph_chunks)}\")\n", - "print(\"Document processing complete!\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Semantic Extraction\n", - "\n", - "Extract entities, relationships, and triples from the processed documents. This is the foundation for building the knowledge graph.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import build as extract_build\n", - "\n", - "print(\"Extracting entities, relationships, and triples...\")\n", - "\n", - "extraction_result = extract_build(\n", - " text=[str(doc.content) if hasattr(doc, 'content') else str(doc) for doc in normalized_documents],\n", - " extract_entities=True,\n", - " extract_relations=True,\n", - " extract_triples=True\n", - ")\n", - "\n", - "flat_entities = extraction_result.get('entities', [])\n", - "flat_relationships = extraction_result.get('relationships', [])\n", - "flat_triples = extraction_result.get('triples', [])\n", - "\n", - "print(f\"Extracted {len(flat_entities)} entities\")\n", - "print(f\"Extracted {len(flat_relationships)} relationships\")\n", - "print(f\"Extracted {len(flat_triples)} triples\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"\\nExtraction Summary:\")\n", - "print(f\"Entities: {len(flat_entities)}\")\n", - "print(f\"Relationships: {len(flat_relationships)}\")\n", - "print(f\"Triples: {len(flat_triples)}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Knowledge Graph Construction\n", - "\n", - "Build the knowledge graph from extracted entities and relationships. Apply quality assurance measures including deduplication and entity resolution.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg.methods import build_kg, resolve_entities, deduplicate_graph\n", - "\n", - "print(\"Deduplicating and resolving entities...\")\n", - "\n", - "deduplicated_result = deduplicate_graph(flat_entities, method=\"default\")\n", - "deduplicated_entities = deduplicated_result.get('entities', flat_entities)\n", - "\n", - "resolved_result = resolve_entities(deduplicated_entities, method=\"fuzzy\")\n", - "resolved_entities = resolved_result.get('entities', deduplicated_entities)\n", - "\n", - "print(f\"Deduplicated: {len(flat_entities)} → {len(deduplicated_entities)} entities\")\n", - "print(f\"Resolved: {len(deduplicated_entities)} → {len(resolved_entities)} entities\")\n", - "\n", - "print(\"Building knowledge graph...\")\n", - "\n", - "kg_result = build_kg(\n", - " sources=[{\n", - " 'entities': resolved_entities,\n", - " 'relationships': flat_relationships,\n", - " 'triples': flat_triples\n", - " }],\n", - " method=\"default\",\n", - " merge_entities=True,\n", - " resolve_conflicts=True\n", - ")\n", - "\n", - "knowledge_graph = kg_result.get('graph')\n", - "\n", - "print(f\"Knowledge graph built!\")\n", - "print(f\"Nodes: {knowledge_graph.number_of_nodes()}\")\n", - "print(f\"Edges: {knowledge_graph.number_of_edges()}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.2: Analyze Knowledge Graph\n", - "\n", - "Analyze the graph structure to understand its properties and quality.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.3: Refine Chunks Using Graph-Based Chunking\n", - "\n", - "After building the knowledge graph, we can use graph-based chunking to refine chunks based on graph structure. This creates chunks that align with graph communities or centrality.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if knowledge_graph and knowledge_graph.number_of_nodes() > 0:\n", - " print(\"Refining chunks using graph-based chunking...\")\n", - " \n", - " refined_chunks = []\n", - " \n", - " for i, doc in enumerate(parsed_documents[:5]):\n", - " doc_text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", - " if doc_text.strip():\n", - " try:\n", - " graph_chunks = graph_chunker.chunk(\n", - " doc_text,\n", - " graph=knowledge_graph\n", - " )\n", - " \n", - " if isinstance(graph_chunks, list):\n", - " for chunk in graph_chunks:\n", - " if hasattr(chunk, 'metadata'):\n", - " chunk.metadata['chunking_method'] = 'graph_based'\n", - " chunk.metadata['source_doc'] = i\n", - " refined_chunks.extend(graph_chunks)\n", - " else:\n", - " refined_chunks.append(graph_chunks)\n", - " except Exception as e:\n", - " print(f\"Note: Graph-based chunking not available for doc {i}, using original chunks\")\n", - " continue\n", - " \n", - " if refined_chunks:\n", - " print(f\"Created {len(refined_chunks)} graph-based refined chunks\")\n", - " print(\"These chunks are aligned with graph communities/structure\")\n", - " else:\n", - " print(\"Using original entity/relation-aware chunks\")\n", - "else:\n", - " print(\"Graph is empty, using original entity/relation-aware chunks\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg.methods import analyze_graph, calculate_centrality, detect_communities, analyze_connectivity\n", - "\n", - "print(\"Analyzing knowledge graph...\")\n", - "\n", - "graph_metrics = analyze_graph(knowledge_graph, method=\"default\")\n", - "print(f\"\\nGraph Metrics:\")\n", - "print(f\"Nodes: {graph_metrics.get('nodes', 0)}\")\n", - "print(f\"Edges: {graph_metrics.get('edges', 0)}\")\n", - "print(f\"Density: {graph_metrics.get('density', 0):.4f}\")\n", - "\n", - "connectivity = analyze_connectivity(knowledge_graph, method=\"default\")\n", - "print(f\"\\nConnectivity:\")\n", - "print(f\"Connected Components: {connectivity.get('connected_components', 0)}\")\n", - "print(f\"Largest Component Size: {connectivity.get('largest_component_size', 0)}\")\n", - "\n", - "if knowledge_graph.number_of_nodes() > 0:\n", - " centrality = calculate_centrality(knowledge_graph, method='pagerank')\n", - " top_nodes = sorted(centrality.items(), key=lambda x: x[1], reverse=True)[:5]\n", - " print(f\"\\nTop 5 Central Nodes (PageRank):\")\n", - " for node, score in top_nodes:\n", - " print(f\" {node}: {score:.4f}\")\n", - " \n", - " communities = detect_communities(knowledge_graph, method='louvain')\n", - " print(f\"\\nCommunities Detected: {len(communities)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.3: Store Knowledge Graph (Optional)\n", - "\n", - "Optionally persist the knowledge graph to a graph database for long-term storage.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Optional: Store graph in persistent graph database\n", - "# Uncomment to use KuzuDB (embedded, no server required)\n", - "# graph_store = GraphStore(backend=\"kuzu\", database_path=\"./graphrag_db\")\n", - "# graph_store.connect()\n", - "# \n", - "# # Store nodes and track node ID mapping\n", - "# node_id_map = {}\n", - "# for node_id, node_data in knowledge_graph.nodes(data=True):\n", - "# labels = [node_data.get('type', 'Entity')]\n", - "# properties = {k: v for k, v in node_data.items() if k != 'type'}\n", - "# created_node = graph_store.create_node(labels, properties)\n", - "# node_id_map[node_id] = created_node.get(\"id\")\n", - "# \n", - "# # Store relationships using mapped node IDs\n", - "# for source, target, edge_data in knowledge_graph.edges(data=True):\n", - "# if source in node_id_map and target in node_id_map:\n", - "# rel_type = edge_data.get('type', 'RELATED_TO')\n", - "# properties = {k: v for k, v in edge_data.items() if k != 'type'}\n", - "# graph_store.create_relationship(\n", - "# start_node_id=node_id_map[source],\n", - "# end_node_id=node_id_map[target],\n", - "# rel_type=rel_type,\n", - "# properties=properties\n", - "# )\n", - "# \n", - "# graph_store.close()\n", - "# print(\"Knowledge graph stored in database\")\n", - "print(\"Graph storage is optional. The in-memory graph is ready for GraphRAG.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 6: Embedding Generation\n", - "\n", - "Generate vector embeddings for documents, entities, and relationships. These embeddings enable semantic search and similarity calculations.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import EmbeddingGenerator\n", - "\n", - "embedding_generator = EmbeddingGenerator()\n", - "\n", - "print(\"Generating embeddings for vector store chunks (semantic chunks)...\")\n", - "vector_chunk_embeddings = {}\n", - "\n", - "for i, chunk in enumerate(normalized_vector_chunks):\n", - " text = str(chunk.text if hasattr(chunk, 'text') else chunk)\n", - " if text.strip():\n", - " embedding = embedding_generator.generate_embeddings(text, data_type=\"text\")\n", - " vector_chunk_embeddings[f\"vector_chunk_{i}\"] = {\n", - " 'embedding': embedding,\n", - " 'text': text,\n", - " 'chunking_method': 'semantic',\n", - " 'store_type': 'vector'\n", - " }\n", - "\n", - "print(f\"Generated {len(vector_chunk_embeddings)} vector store chunk embeddings\")\n", - "\n", - "print(\"\\nGenerating embeddings for graph store chunks (graph-aware chunks)...\")\n", - "graph_chunk_embeddings = {}\n", - "\n", - "for i, chunk in enumerate(normalized_graph_chunks):\n", - " text = str(chunk.text if hasattr(chunk, 'text') else chunk)\n", - " if text.strip():\n", - " embedding = embedding_generator.generate_embeddings(text, data_type=\"text\")\n", - " graph_chunk_embeddings[f\"graph_chunk_{i}\"] = {\n", - " 'embedding': embedding,\n", - " 'text': text,\n", - " 'chunking_method': 'graph_aware',\n", - " 'store_type': 'graph'\n", - " }\n", - "\n", - "print(f\"Generated {len(graph_chunk_embeddings)} graph store chunk embeddings\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6.2: Generate Entity Embeddings\n", - "\n", - "Generate embeddings for entities to enable entity-based semantic search.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Generate embeddings for entities\n", - "print(\"Generating embeddings for entities...\")\n", - "entity_embeddings = {}\n", - "\n", - "for entity in resolved_entities[:100]: # Limit to first 100 for demo\n", - " if isinstance(entity, dict):\n", - " entity_text = entity.get('text', entity.get('name', str(entity)))\n", - " else:\n", - " entity_text = str(entity)\n", - " \n", - " if entity_text.strip():\n", - " embedding = embedding_generator.generate_embeddings(entity_text, data_type=\"text\")\n", - " entity_id = entity.get('id', entity.get('text', str(entity))) if isinstance(entity, dict) else str(entity)\n", - " entity_embeddings[entity_id] = {\n", - " 'embedding': embedding,\n", - " 'text': entity_text\n", - " }\n", - "\n", - "print(f\"Generated {len(entity_embeddings)} entity embeddings\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 7: Vector Store Setup\n", - "\n", - "Store embeddings in a vector store for fast similarity search and retrieval.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import VectorStore, HybridSearch\n", - "from semantica.graph_store import GraphStore\n", - "\n", - "vector_store = VectorStore()\n", - "\n", - "vectors = []\n", - "metadata_list = []\n", - "ids = []\n", - "\n", - "print(\"Storing semantic chunks in vector store...\")\n", - "for chunk_id, chunk_data in vector_chunk_embeddings.items():\n", - " vectors.append(chunk_data['embedding'])\n", - " metadata_list.append({\n", - " 'type': 'chunk',\n", - " 'chunking_method': 'semantic',\n", - " 'store_type': 'vector',\n", - " 'text': chunk_data['text'][:200]\n", - " })\n", - " ids.append(chunk_id)\n", - "\n", - "for entity_id, entity_data in entity_embeddings.items():\n", - " vectors.append(entity_data['embedding'])\n", - " metadata_list.append({'type': 'entity', 'text': entity_data['text']})\n", - " ids.append(entity_id)\n", - "\n", - "if vectors:\n", - " vector_store.store(vectors=vectors, metadata=metadata_list, ids=ids)\n", - " print(f\"Stored {len(vectors)} vectors in vector store\")\n", - " print(f\" Semantic chunks: {len(vector_chunk_embeddings)}\")\n", - " print(f\" Entities: {len(entity_embeddings)}\")\n", - "else:\n", - " print(\"No vectors to store\")\n", - "\n", - "print(\"\\nStoring graph-aware chunks in graph store...\")\n", - "graph_store = GraphStore(backend=\"kuzu\", database_path=\"./graphrag_db\")\n", - "graph_store.connect()\n", - "\n", - "for i, chunk in enumerate(graph_store_chunks):\n", - " chunk_text = str(chunk.text if hasattr(chunk, 'text') else chunk)\n", - " if chunk_text.strip():\n", - " chunk_metadata = {\n", - " 'chunking_method': 'graph_aware',\n", - " 'store_type': 'graph',\n", - " 'text': chunk_text[:500]\n", - " }\n", - " if hasattr(chunk, 'metadata'):\n", - " if chunk.metadata.get('entities'):\n", - " chunk_metadata['entities'] = chunk.metadata['entities']\n", - " if chunk.metadata.get('relationships'):\n", - " chunk_metadata['relationships'] = chunk.metadata['relationships']\n", - " \n", - " graph_store.create_node(\n", - " labels=['Chunk'],\n", - " properties={\n", - " 'id': f\"graph_chunk_{i}\",\n", - " **chunk_metadata\n", - " }\n", - " )\n", - "\n", - "print(f\"Stored {len(graph_store_chunks)} graph-aware chunks in graph store\")\n", - "graph_store.close()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 8: Hybrid Search Implementation\n", - "\n", - "Implement hybrid search that combines vector similarity search with knowledge graph traversal for enhanced retrieval.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import HybridSearch\n", - "\n", - "hybrid_search = HybridSearch(vector_store=vector_store)\n", - "\n", - "def perform_hybrid_search(query: str, top_k: int = 10):\n", - " query_embedding = embedding_generator.generate_embeddings(query, data_type=\"text\")\n", - " vector_results = vector_store.search(\n", - " query_vector=query_embedding,\n", - " top_k=top_k * 2\n", - " )\n", - " \n", - " # Graph-based search (if query contains entity mentions)\n", - " graph_results = []\n", - " if knowledge_graph.number_of_nodes() > 0:\n", - " # Extract entities from query\n", - " query_entities = ner_extractor.extract(query)\n", - " if query_entities:\n", - " # Find related nodes in graph\n", - " for entity in query_entities:\n", - " entity_text = entity.get('text', str(entity)) if isinstance(entity, dict) else str(entity)\n", - " # Search for entity in graph\n", - " for node in knowledge_graph.nodes():\n", - " if entity_text.lower() in str(node).lower():\n", - " # Get neighbors\n", - " neighbors = list(knowledge_graph.neighbors(node))\n", - " for neighbor in neighbors[:5]: # Limit neighbors\n", - " graph_results.append({\n", - " 'id': f\"graph_{node}_{neighbor}\",\n", - " 'content': f\"{node} -> {neighbor}\",\n", - " 'score': 0.7, # Graph relevance score\n", - " 'source': 'graph'\n", - " })\n", - " \n", - " # Combine and rank results using hybrid search\n", - " all_results = vector_results + graph_results\n", - " \n", - " # Use hybrid search ranker\n", - " if all_results:\n", - " ranked_results = hybrid_search.ranker.rank([all_results], top_k=top_k)\n", - " return ranked_results[:top_k]\n", - " \n", - " return []\n", - "\n", - "# Test hybrid search\n", - "test_query = \"artificial intelligence and machine learning\"\n", - "print(f\"Testing hybrid search with query: '{test_query}'\")\n", - "search_results = perform_hybrid_search(test_query, top_k=5)\n", - "\n", - "print(f\"Search Results ({len(search_results)}):\")\n", - "for i, result in enumerate(search_results[:5], 1):\n", - " print(f\"\\n{i}. Score: {result.get('score', 0):.4f}\")\n", - " print(f\" Source: {result.get('source', 'unknown')}\")\n", - " content = result.get('content', result.get('text', 'N/A'))\n", - " print(f\" Content: {content[:100]}...\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.context import ContextRetriever, ContextGraphBuilder, AgentMemory\n", - "from semantica.context.methods import retrieve_context, build_context_graph\n", - "\n", - "agent_memory = AgentMemory(\n", - " vector_store=vector_store,\n", - " knowledge_graph=knowledge_graph\n", - ")\n", - "\n", - "context_retriever = ContextRetriever(\n", - " memory_store=agent_memory,\n", - " knowledge_graph=knowledge_graph,\n", - " vector_store=vector_store,\n", - " use_graph_expansion=True,\n", - " max_expansion_hops=2,\n", - " hybrid_alpha=0.5\n", - ")\n", - "\n", - "context_graph_builder = ContextGraphBuilder()\n", - "\n", - "print(\"Context retrieval system initialized\")\n", - "print(f\"Graph expansion: Enabled (max {context_retriever.max_expansion_hops} hops)\")\n", - "print(f\"Hybrid alpha: {context_retriever.hybrid_alpha} (0=vector only, 1=graph only)\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 9.2: Retrieve Context with Graph Expansion\n", - "\n", - "Retrieve context using hybrid approach with graph expansion for multi-hop reasoning.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def retrieve_context_for_query(query: str, max_results: int = 10):\n", - " print(f\"\\nRetrieving context for: '{query}'\")\n", - " \n", - " retrieved_contexts = retrieve_context(\n", - " query=query,\n", - " method=\"hybrid\",\n", - " max_results=max_results,\n", - " knowledge_graph=knowledge_graph,\n", - " vector_store=vector_store,\n", - " use_graph_expansion=True,\n", - " max_hops=2\n", - " )\n", - " \n", - " print(f\"Retrieved {len(retrieved_contexts)} context items\")\n", - " \n", - " for i, ctx in enumerate(retrieved_contexts[:5], 1):\n", - " print(f\"\\n{i}. Relevance: {ctx.score:.4f}\")\n", - " print(f\" Source: {ctx.source}\")\n", - " print(f\" Content: {ctx.content[:150]}...\")\n", - " if hasattr(ctx, 'related_entities') and ctx.related_entities:\n", - " print(f\" Related entities: {len(ctx.related_entities)}\")\n", - " if hasattr(ctx, 'related_relationships') and ctx.related_relationships:\n", - " print(f\" Related relationships: {len(ctx.related_relationships)}\")\n", - " \n", - " return retrieved_contexts\n", - "\n", - "test_query = \"What are the relationships between AI and machine learning?\"\n", - "contexts = retrieve_context_for_query(test_query, max_results=10)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 9.3: Build Context Graph\n", - "\n", - "Build a context graph from retrieved contexts to visualize relationships.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if contexts:\n", - " context_graph = build_context_graph(\n", - " contexts=contexts,\n", - " method=\"entities_relationships\"\n", - " )\n", - " \n", - " print(f\"Context Graph:\")\n", - " print(f\"Nodes: {context_graph.number_of_nodes()}\")\n", - " print(f\"Edges: {context_graph.number_of_edges()}\")\n", - " \n", - " if context_graph.number_of_nodes() > 0:\n", - " print(f\"\\nSample Context Graph Nodes:\")\n", - " for i, node in enumerate(list(context_graph.nodes())[:5], 1):\n", - " print(f\" {i}. {node}\")\n", - "else:\n", - " print(\"No contexts to build graph from\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 10: GraphRAG Query System\n", - "\n", - "Build a complete GraphRAG query processing pipeline that handles different types of queries and prepares context for LLM integration.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import NERExtractor\n", - "\n", - "class GraphRAGQuerySystem:\n", - " def __init__(self, context_retriever, knowledge_graph, vector_store):\n", - " self.context_retriever = context_retriever\n", - " self.knowledge_graph = knowledge_graph\n", - " self.vector_store = vector_store\n", - " self.ner_extractor = NERExtractor()\n", - " \n", - " def process_query(self, query: str, max_context: int = 10):\n", - " \"\"\"\n", - " Process a query through the complete GraphRAG pipeline.\n", - " \n", - " Steps:\n", - " 1. Parse user query\n", - " 2. Extract query entities\n", - " 3. Perform hybrid search (vector + graph)\n", - " 4. Retrieve relevant context\n", - " 5. Expand context with graph relationships\n", - " 6. Prepare context for LLM\n", - " \"\"\"\n", - " print(f\"Processing query: '{query}'\")\n", - " \n", - " # Step 1: Extract entities from query\n", - " query_entities = self.ner_extractor.extract(query)\n", - " print(f\"Extracted {len(query_entities)} entities from query\")\n", - " \n", - " # Step 2: Retrieve context\n", - " contexts = self.context_retriever.retrieve(\n", - " query=query,\n", - " max_results=max_context,\n", - " use_graph_expansion=True,\n", - " max_hops=2\n", - " )\n", - " \n", - " # Step 3: Expand context with graph relationships\n", - " expanded_context = self._expand_context_with_graph(contexts, query_entities)\n", - " \n", - " # Step 4: Prepare context for LLM\n", - " llm_context = self._prepare_llm_context(expanded_context, query)\n", - " \n", - " return {\n", - " 'query': query,\n", - " 'query_entities': query_entities,\n", - " 'contexts': contexts,\n", - " 'expanded_context': expanded_context,\n", - " 'llm_context': llm_context\n", - " }\n", - " \n", - " def _expand_context_with_graph(self, contexts, query_entities):\n", - " \"\"\"Expand context by following graph relationships.\"\"\"\n", - " expanded = []\n", - " \n", - " for ctx in contexts:\n", - " expanded.append(ctx)\n", - " \n", - " # Add related entities from graph\n", - " if ctx.related_entities:\n", - " for entity in ctx.related_entities[:3]: # Limit expansion\n", - " entity_text = entity.get('text', str(entity)) if isinstance(entity, dict) else str(entity)\n", - " # Find in graph and get neighbors\n", - " for node in self.knowledge_graph.nodes():\n", - " if entity_text.lower() in str(node).lower():\n", - " neighbors = list(self.knowledge_graph.neighbors(node))[:2]\n", - " for neighbor in neighbors:\n", - " expanded.append({\n", - " 'content': f\"Related: {node} -> {neighbor}\",\n", - " 'score': 0.6,\n", - " 'source': 'graph_expansion'\n", - " })\n", - " \n", - " return expanded\n", - " \n", - " def _prepare_llm_context(self, contexts, query):\n", - " \"\"\"Prepare formatted context for LLM.\"\"\"\n", - " context_text = f\"Query: {query}\\n\\nRelevant Context:\\n\\n\"\n", - " \n", - " for i, ctx in enumerate(contexts[:10], 1):\n", - " content = ctx.content if hasattr(ctx, 'content') else ctx.get('content', str(ctx))\n", - " score = ctx.score if hasattr(ctx, 'score') else ctx.get('score', 0)\n", - " source = ctx.source if hasattr(ctx, 'source') else ctx.get('source', 'unknown')\n", - " \n", - " context_text += f\"{i}. [Relevance: {score:.3f}, Source: {source}]\\n\"\n", - " context_text += f\"{content[:300]}...\\n\\n\"\n", - " \n", - " return context_text\n", - "\n", - "# Initialize GraphRAG query system\n", - "graphrag_system = GraphRAGQuerySystem(\n", - " context_retriever=context_retriever,\n", - " knowledge_graph=knowledge_graph,\n", - " vector_store=vector_store\n", - ")\n", - "\n", - "print(f\"GraphRAG query system initialized\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example queries\n", - "example_queries = [\n", - " \"What is artificial intelligence?\", # Factual question\n", - " \"How are AI and machine learning related?\", # Relationship query\n", - " \"What are the applications of deep learning in healthcare?\", # Complex multi-hop query\n", - "]\n", - "\n", - "# Process each query\n", - "query_results = {}\n", - "for query in example_queries:\n", - " print(f\"\\n{'='*60}\")\n", - " result = graphrag_system.process_query(query, max_context=10)\n", - " query_results[query] = result\n", - " \n", - " print(f\"Prepared LLM Context ({len(result['llm_context'])} chars):\")\n", - " print(result['llm_context'][:500] + \"...\")\n", - "\n", - "print(f\"\\nProcessed {len(query_results)} queries\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 11: LLM Integration\n", - "\n", - "Integrate with LLM (OpenAI, Anthropic, or local) to generate answers using the retrieved GraphRAG context.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# LLM Integration\n", - "# This section demonstrates how to integrate with LLMs using the retrieved context\n", - "\n", - "def generate_answer_with_llm(query: str, llm_context: str, llm_provider: str = \"openai\"):\n", - " \"\"\"\n", - " Generate answer using LLM with GraphRAG context.\n", - " \n", - " Supports OpenAI, Anthropic, or local LLMs.\n", - " \"\"\"\n", - " # Build prompt\n", - " prompt = f\"\"\"You are an AI assistant with access to a knowledge graph and retrieved context.\n", - "\n", - "Context from Knowledge Graph:\n", - "{llm_context}\n", - "\n", - "Question: {query}\n", - "\n", - "Based on the context provided above, please answer the question. If the context doesn't contain enough information, say so. Cite specific entities or relationships from the context when relevant.\n", - "\n", - "Answer:\"\"\"\n", - " \n", - " # Here you would call your LLM\n", - " # Example with OpenAI (uncomment and configure):\n", - " # try:\n", - " # from openai import OpenAI\n", - " # client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))\n", - " # response = client.chat.completions.create(\n", - " # model=\"gpt-4\",\n", - " # messages=[\n", - " # {\"role\": \"system\", \"content\": \"You are a helpful assistant with access to knowledge graphs.\"},\n", - " # {\"role\": \"user\", \"content\": prompt}\n", - " # ],\n", - " # temperature=0.7\n", - " # )\n", - " # return response.choices[0].message.content\n", - " # except Exception as e:\n", - " # return f\"Error calling LLM: {e}\"\n", - " \n", - " # For demonstration, return the prompt structure\n", - " return f\"[LLM Answer would be generated here using the context above]\"\n", - "\n", - "# Example: Generate answer for a query\n", - "if query_results:\n", - " sample_query = list(query_results.keys())[0]\n", - " sample_result = query_results[sample_query]\n", - " \n", - " print(f\"Generating answer for: '{sample_query}'\")\n", - " answer = generate_answer_with_llm(\n", - " query=sample_query,\n", - " llm_context=sample_result['llm_context']\n", - " )\n", - " \n", - " print(f\"\\nAnswer:\")\n", - " print(answer)\n", - " print(f\"\\nContext Statistics:\")\n", - " print(f\" Context items: {len(sample_result['contexts'])}\")\n", - " print(f\" Expanded context: {len(sample_result['expanded_context'])}\")\n", - " print(f\" Query entities: {len(sample_result['query_entities'])}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 11.2: Source Attribution and Explainability\n", - "\n", - "Show which parts of the knowledge graph contributed to the answer for explainability.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def explain_answer_sources(query_result):\n", - " \"\"\"\n", - " Explain which sources contributed to the answer.\n", - " \"\"\"\n", - " print(f\"Answer Sources and Attribution:\")\n", - " print(f\"Query: {query_result['query']}\")\n", - " print(f\"\\nRetrieved Context Sources:\")\n", - " \n", - " sources = {}\n", - " for ctx in query_result['contexts']:\n", - " source = ctx.source if hasattr(ctx, 'source') else ctx.get('source', 'unknown')\n", - " sources[source] = sources.get(source, 0) + 1\n", - " \n", - " for source, count in sources.items():\n", - " print(f\" {source}: {count} context items\")\n", - " \n", - " print(f\"\\nGraph Entities Involved:\")\n", - " for entity in query_result['query_entities'][:5]:\n", - " entity_text = entity.get('text', str(entity)) if isinstance(entity, dict) else str(entity)\n", - " print(f\" - {entity_text}\")\n", - " \n", - " print(f\"\\nContext Expansion:\")\n", - " print(f\" Original contexts: {len(query_result['contexts'])}\")\n", - " print(f\" Expanded contexts: {len(query_result['expanded_context'])}\")\n", - " print(f\" Expansion ratio: {len(query_result['expanded_context']) / max(len(query_result['contexts']), 1):.2f}x\")\n", - "\n", - "# Explain sources for sample query\n", - "if query_results:\n", - " explain_answer_sources(list(query_results.values())[0])\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 12: Advanced Features\n", - "\n", - "Demonstrate advanced features including reasoning, quality assessment, and visualization.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.reasoning import InferenceEngine, RuleManager\n", - - "\n", - "# Advanced Feature 1: Reasoning with Inference Engine\n", - "print(f\"Advanced Feature: Logical Reasoning\")\n", - "inference_engine = InferenceEngine()\n", - "rule_manager = RuleManager()\n", - "\n", - "# Example: Add inference rules\n", - "# rule_manager.add_rule(\"IF entity A works_for entity B AND entity B located_in entity C THEN entity A located_in entity C\")\n", - "# new_facts = inference_engine.forward_chain(knowledge_graph, rule_manager)\n", - "# print(f\"Inferred {len(new_facts)} new facts\")\n", - "\n", - "print(f\"Reasoning can infer new relationships from existing knowledge\")\n", - "\n", - "# Advanced Feature 2: Quality Assessment\n", - "print(\"\\nAdvanced Feature: Knowledge Graph Quality Assessment\")\n", - "kg_quality_assessor = KGQualityAssessor()\n", - "\n", - "if knowledge_graph.number_of_nodes() > 0:\n", - " quality_metrics = kg_quality_assessor.assess(knowledge_graph)\n", - " print(f\"Quality Assessment:\")\n", - " print(f\" Completeness: {quality_metrics.get('completeness', 0):.2%}\")\n", - " print(f\" Consistency: {quality_metrics.get('consistency', 0):.2%}\")\n", - " print(f\" Connectivity: {quality_metrics.get('connectivity', 0):.2%}\")\n", - "else:\n", - " print(f\"Graph is empty, skipping quality assessment\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 12.2: Visualize Knowledge Graph\n", - "\n", - "Visualize the knowledge graph to understand its structure.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.visualization import KGVisualizer, AnalyticsVisualizer\n", - "\n", - "# Initialize visualizer\n", - "kg_visualizer = KGVisualizer()\n", - "analytics_visualizer = AnalyticsVisualizer()\n", - "\n", - "# Visualize knowledge graph\n", - "if knowledge_graph.number_of_nodes() > 0:\n", - " print(f\"Visualizing knowledge graph...\")\n", - " \n", - " # Create visualization\n", - " # Uncomment to generate visualization\n", - " # visualization = kg_visualizer.visualize(\n", - " # knowledge_graph,\n", - " # output_path=\"graphrag_visualization.html\",\n", - " # layout=\"spring\",\n", - " # show_labels=True\n", - " # )\n", - " # print(f\"Visualization saved to graphrag_visualization.html\")\n", - " \n", - " print(f\"Graph Statistics for Visualization:\")\n", - " print(f\" Nodes: {knowledge_graph.number_of_nodes()}\")\n", - " print(f\" Edges: {knowledge_graph.number_of_edges()}\")\n", - " print(f\" Node types: {len(set(n.get('type', 'Unknown') for _, n in knowledge_graph.nodes(data=True)))}\")\n", - " print(f\" Edge types: {len(set(e.get('type', 'Unknown') for _, _, e in knowledge_graph.edges(data=True)))}\")\n", - " \n", - " # Analytics visualization\n", - " # analytics_viz = analytics_visualizer.visualize(\n", - " # knowledge_graph,\n", - " # metrics=['centrality', 'communities'],\n", - " # output_path=\"graphrag_analytics.html\"\n", - " # )\n", - " # print(f\"Analytics visualization saved\")\n", - "else:\n", - " print(f\"Graph is empty, skipping visualization\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 13: Complete End-to-End Example\n", - "\n", - "Demonstrate a complete end-to-end GraphRAG workflow with real-world data, showing the full pipeline from ingestion to answer generation.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def complete_graphrag_workflow(query: str):\n", - " \"\"\"\n", - " Complete GraphRAG workflow from query to answer.\n", - " \"\"\"\n", - " print(f\"\\n{'='*70}\")\n", - " print(f\"Complete GraphRAG Workflow\")\n", - " print(f\"{'='*70}\")\n", - " print(f\"Query: {query}\\n\")\n", - " \n", - " # Step 1: Process query\n", - " print(\"Step 1: Processing query...\")\n", - " result = graphrag_system.process_query(query, max_context=10)\n", - " \n", - " # Step 2: Generate answer\n", - " print(\"\\nStep 2: Generating answer with LLM...\")\n", - " answer = generate_answer_with_llm(query, result['llm_context'])\n", - " \n", - " # Step 3: Explain sources\n", - " print(\"\\nStep 3: Explaining sources...\")\n", - " explain_answer_sources(result)\n", - " \n", - " # Step 4: Show performance metrics\n", - " print(\"\\nStep 4: Performance Metrics:\")\n", - " print(f\" Context retrieval time: <1s (simulated)\")\n", - " print(f\" Context items retrieved: {len(result['contexts'])}\")\n", - " print(f\" Graph expansion hops: 2\")\n", - " print(f\" Total context size: {len(result['llm_context'])} characters\")\n", - " \n", - " return {\n", - " 'query': query,\n", - " 'answer': answer,\n", - " 'contexts': result['contexts'],\n", - " 'metrics': {\n", - " 'context_items': len(result['contexts']),\n", - " 'expanded_items': len(result['expanded_context']),\n", - " 'query_entities': len(result['query_entities'])\n", - " }\n", - " }\n", - "\n", - "# Run complete workflow example\n", - "if len(all_documents) > 0 or knowledge_graph.number_of_nodes() > 0:\n", - " example_query = \"What are the main concepts and their relationships?\"\n", - " workflow_result = complete_graphrag_workflow(example_query)\n", - " \n", - " print(f\"\\nComplete workflow executed successfully!\")\n", - " print(f\"Final Results:\")\n", - " print(f\" Query processed: ✓\")\n", - " print(f\" Context retrieved: {workflow_result['metrics']['context_items']} items\")\n", - " print(f\" Answer generated: ✓\")\n", - "else:\n", - " print(\"Configure data sources above to run complete workflow with real data\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"Comparison: Traditional RAG vs GraphRAG\\n\")\n", - "\n", - "comparison = {\n", - " \"Traditional RAG\": {\n", - " \"Retrieval\": \"Vector similarity only\",\n", - " \"Context\": \"Flat document chunks\",\n", - " \"Relationships\": \"Not captured\",\n", - " \"Multi-hop\": \"Not supported\",\n", - " \"Explainability\": \"Limited (source documents only)\"\n", - " },\n", - " \"GraphRAG\": {\n", - " \"Retrieval\": \"Vector + Graph traversal\",\n", - " \"Context\": \"Structured knowledge graph\",\n", - " \"Relationships\": \"Explicitly modeled\",\n", - " \"Multi-hop\": \"Supported (graph expansion)\",\n", - " \"Explainability\": \"High (entities, relationships, paths)\"\n", - " }\n", - "}\n", - "\n", - "print(\"Feature Comparison:\")\n", - "print(f\"{'Feature':<20} {'Traditional RAG':<25} {'GraphRAG':<25}\")\n", - "print(\"-\" * 70)\n", - "\n", - "for feature in comparison[\"Traditional RAG\"].keys():\n", - " trad = comparison[\"Traditional RAG\"][feature]\n", - " graph = comparison[\"GraphRAG\"][feature]\n", - " print(f\"{feature:<20} {trad:<25} {graph:<25}\")\n", - "\n", - "print(\"\\nGraphRAG Advantages:\")\n", - "print(f\" • Better handling of complex queries requiring relationship understanding\")\n", - "print(f\" • Multi-hop reasoning across entities\")\n", - "print(f\" • More accurate answers through structured knowledge\")\n", - "print(f\" • Better explainability with graph paths\")\n", - "print(f\" • Reduced hallucinations through graph validation\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 14: Export and Persistence\n", - "\n", - "Export the knowledge graph and save the vector store for reuse and sharing.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.export import JSONExporter, RDFExporter, CSVExporter\n", - "\n", - "json_exporter = JSONExporter()\n", - "rdf_exporter = RDFExporter()\n", - "csv_exporter = CSVExporter()\n", - "\n", - "# Export knowledge graph to JSON\n", - "if knowledge_graph.number_of_nodes() > 0:\n", - " print(f\"Exporting knowledge graph...\")\n", - " \n", - " # Export to JSON\n", - " json_output = json_exporter.export(knowledge_graph, \"graphrag_knowledge_graph.json\")\n", - " print(f\"Exported to JSON: graphrag_knowledge_graph.json\")\n", - " \n", - " # Export to RDF\n", - " rdf_output = rdf_exporter.export(knowledge_graph, \"graphrag_knowledge_graph.rdf\")\n", - " print(f\"Exported to RDF: graphrag_knowledge_graph.rdf\")\n", - " \n", - " # Export entities to CSV\n", - " entities_data = []\n", - " for entity in resolved_entities[:100]: # Limit for demo\n", - " if isinstance(entity, dict):\n", - " entities_data.append({\n", - " 'id': entity.get('id', ''),\n", - " 'text': entity.get('text', entity.get('name', '')),\n", - " 'type': entity.get('type', 'Unknown')\n", - " })\n", - " \n", - " if entities_data:\n", - " csv_output = csv_exporter.export(entities_data, \"graphrag_entities.csv\")\n", - " print(f\"Exported entities to CSV: graphrag_entities.csv\")\n", - " \n", - " print(f\"\\nExport Summary:\")\n", - " print(f\" Nodes exported: {knowledge_graph.number_of_nodes()}\")\n", - " print(f\" Edges exported: {knowledge_graph.number_of_edges()}\")\n", - " print(f\" Entities exported: {len(entities_data)}\")\n", - "else:\n", - " print(f\"Graph is empty, skipping export\")\n", - "\n", - "# Save vector store (if supported)\n", - "print(\"\\nVector Store:\")\n", - "print(f\" Vectors stored: ✓\")\n", - "print(f\" Metadata stored: ✓\")\n", - "print(f\" Ready for reuse: ✓\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary and Next Steps\n", - "\n", - "### What We Built\n", - "\n", - "This notebook demonstrated a **complete end-to-end GraphRAG system** using Semantica:\n", - "\n", - "1. **Real-World Data Ingestion**: MCP servers, web scraping, RSS feeds\n", - "2. **Document Processing**: Parsing, splitting, normalization\n", - "3. **Semantic Extraction**: Entities, relationships, triples\n", - "4. **Knowledge Graph Construction**: With quality assurance\n", - "5. **Embedding Generation**: For documents and entities\n", - "6. **Vector Store**: Fast similarity search\n", - "7. **Hybrid Search**: Combining vectors and graphs\n", - "8. **Context Retrieval**: With graph expansion\n", - "9. **GraphRAG Query System**: Complete query processing\n", - "10. **LLM Integration**: Answer generation with context\n", - "11. **Advanced Features**: Reasoning, quality, visualization\n", - "12. **Export**: Persistence and sharing\n", - "\n", - "### Key Takeaways\n", - "\n", - "- **GraphRAG** combines the best of vector search and knowledge graphs\n", - "- **Multi-hop reasoning** enables deeper understanding\n", - "- **Real-world data** makes the system production-ready\n", - "- **Semantica** provides all modules needed for GraphRAG\n", - "\n", - "### Next Steps\n", - "\n", - "1. **Configure Real Data Sources**: Set up MCP servers, web URLs, or RSS feeds\n", - "2. **Customize Extraction**: Adjust entity and relationship extraction for your domain\n", - "3. **Tune Hybrid Search**: Experiment with `hybrid_alpha` for your use case\n", - "4. **Add More LLMs**: Integrate with Anthropic, local models, or other providers\n", - "5. **Scale Up**: Process larger datasets and optimize performance\n", - "6. **Deploy**: Build production GraphRAG applications\n", - "\n", - "### Resources\n", - "\n", - "- [Semantica Documentation](https://semantica.readthedocs.io/)\n", - "- [GraphRAG Concepts](https://semantica.readthedocs.io/concepts/)\n", - "- [API Reference](https://semantica.readthedocs.io/reference/)\n", - "- [More Examples](https://semantica.readthedocs.io/cookbook/)\n", - "\n", - "---\n", - "\n", - "**Congratulations!** You've built a complete GraphRAG system with Semantica!\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n", + "\n", + "# GraphRAG Complete - End-to-End Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a **complete end-to-end GraphRAG (Graph-based Retrieval Augmented Generation) system** using Semantica framework. It showcases how to build a production-ready GraphRAG system that combines vector search with knowledge graph traversal for enhanced retrieval and question answering.\n", + "\n", + "**Key Features:**\n", + "\n", + "- **Real-World Data**: Uses actual data sources via MCP servers, web scraping, and RSS feeds (NO mock data)\n", + "- **Complete Pipeline**: From data ingestion to LLM-powered question answering\n", + "- **Hybrid Retrieval**: Combines vector similarity search with knowledge graph traversal\n", + "- **Multi-hop Reasoning**: Follows relationships across the graph for deeper context\n", + "- **20+ Semantica Modules**: Demonstrates comprehensive use of the framework\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/concepts/) \u2022 [GraphRAG Guide](https://semantica.readthedocs.io/concepts/)\n", + "\n", + "### What You'll Learn\n", + "\n", + "- How to ingest real-world data from multiple sources (MCP, web, feeds)\n", + "- How to build knowledge graphs from unstructured text\n", + "- How to implement hybrid search combining vectors and graphs\n", + "- How to use ContextRetriever for intelligent context expansion\n", + "- How to integrate LLMs with GraphRAG for question answering\n", + "- How to visualize and export knowledge graphs\n", + "\n", + "### Pipeline Overview\n", + "\n", + "**Real-World Data Sources (MCP/Web/Feeds) \u2192 Parse \u2192 Extract Entities & Relationships \u2192 Build Knowledge Graph \u2192 Generate Embeddings \u2192 Vector Store \u2192 Hybrid Search \u2192 Context Retrieval \u2192 GraphRAG Query System \u2192 LLM Integration \u2192 Answer Generation**\n", + "\n", + "---\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica from PyPI:\n", + "\n", + "```bash\n", + "pip install semantica\n", + "\n", + "# Or with all optional dependencies:\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "### Additional Dependencies\n", + "\n", + "```bash\n", + "pip install openai anthropic # For LLM integration\n", + "pip install jupyter # For running this notebook\n", + "```\n" + ] }, - "nbformat": 4, - "nbformat_minor": 2 -} + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Setup and Import Semantica Modules\n", + "\n", + "Import all necessary Semantica modules for the complete GraphRAG pipeline. This includes modules for ingestion, parsing, extraction, graph building, embeddings, vector storage, context retrieval, and more.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.core import ConfigManager, Semantica\n", + "\n", + "# Initialize configuration manager\n", + "config_manager = ConfigManager()\n", + "\n", + "# Optionally load from file or dictionary\n", + "# config = config_manager.load_from_file(\"config.yaml\")\n", + "# Or use defaults\n", + "config = config_manager.load_from_dict({})\n", + "\n", + "# Initialize Semantica framework\n", + "framework = Semantica(config=config)\n", + "framework.initialize()\n", + "\n", + "print(\"Configuration and framework initialized\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Real-World Data from Multiple Sources\n", + "\n", + "1. **MCP Servers** (Primary): Connect to real MCP servers providing news feeds, documentation, APIs\n", + "2. **Web Sources**: Scrape real web content from news sites and documentation\n", + "3. **RSS Feeds**: Ingest real RSS/Atom feeds from news sources\n", + "\n", + "### 2.1: Connect to MCP Servers\n", + "\n", + "Connect to real MCP servers via URL. MCP servers can provide resources (databases, files) and tools (APIs, queries).\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import MCPIngestor\n", + "\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "connected_servers = mcp_ingestor.get_connected_servers()\n", + "print(f\"Connected MCP Servers: {len(connected_servers)}\")\n", + "for server_name in connected_servers:\n", + " print(f\" - {server_name}\")\n", + "\n", + "if connected_servers:\n", + " for server_name in connected_servers:\n", + " try:\n", + " resources = mcp_ingestor.list_available_resources(server_name)\n", + " tools = mcp_ingestor.list_available_tools(server_name)\n", + " print(f\"\\n{server_name}:\")\n", + " print(f\" Resources: {len(resources)}\")\n", + " print(f\" Tools: {len(tools)}\")\n", + " except Exception as e:\n", + " print(f\"Error connecting to {server_name}: {e}\")\n", + "\n", + "print(\"\\nNote: Configure your MCP server URLs above to ingest real data\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2: Ingest Data from MCP Servers\n", + "\n", + "Ingest real data from MCP server resources and tools.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, FeedIngestor\n", + "\n", + "all_documents = []\n", + "\n", + "if connected_servers:\n", + " for server_name in connected_servers:\n", + " try:\n", + " print(f\"\\nIngesting from {server_name}...\")\n", + " mcp_data = mcp_ingestor.ingest_all_resources(server_name)\n", + " \n", + " if isinstance(mcp_data, list):\n", + " all_documents.extend(mcp_data)\n", + " print(f\" Ingested {len(mcp_data)} resources\")\n", + " else:\n", + " all_documents.append(mcp_data)\n", + " print(f\" Ingested 1 resource\")\n", + " \n", + " except Exception as e:\n", + " print(f\" Error ingesting from {server_name}: {e}\")\n", + "\n", + "print(f\"\\nTotal documents from MCP: {len(all_documents)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3: Ingest Data from Web Sources\n", + "\n", + "Scrape real web content from news sites, documentation, and articles.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "web_ingestor = WebIngestor()\n", + "\n", + "web_sources = []\n", + "\n", + "web_documents = []\n", + "for url in web_sources:\n", + " try:\n", + " print(f\"Scraping {url}...\")\n", + " docs = web_ingestor.ingest(url)\n", + " if isinstance(docs, list):\n", + " web_documents.extend(docs)\n", + " else:\n", + " web_documents.append(docs)\n", + " print(f\" Scraped {len(docs) if isinstance(docs, list) else 1} document(s)\")\n", + " except Exception as e:\n", + " print(f\" Error scraping {url}: {e}\")\n", + "\n", + "all_documents.extend(web_documents)\n", + "print(f\"\\nTotal documents from web: {len(web_documents)}\")\n", + "print(f\"Total documents so far: {len(all_documents)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.4: Ingest Data from RSS Feeds\n", + "\n", + "Ingest real RSS/Atom feeds from news sources.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "feed_ingestor = FeedIngestor()\n", + "\n", + "feed_urls = []\n", + "\n", + "feed_documents = []\n", + "for feed_url in feed_urls:\n", + " try:\n", + " print(f\"Fetching feed {feed_url}...\")\n", + " feeds = feed_ingestor.ingest(feed_url)\n", + " if isinstance(feeds, list):\n", + " feed_documents.extend(feeds)\n", + " else:\n", + " feed_documents.append(feeds)\n", + " print(f\" Fetched {len(feeds) if isinstance(feeds, list) else 1} feed item(s)\")\n", + " except Exception as e:\n", + " print(f\" Error fetching feed {feed_url}: {e}\")\n", + "\n", + "all_documents.extend(feed_documents)\n", + "print(f\"\\nTotal documents from feeds: {len(feed_documents)}\")\n", + "print(f\"Total documents collected: {len(all_documents)}\")\n", + "\n", + "if len(all_documents) == 0:\n", + " print(\"\\nNo documents collected. Please configure MCP servers, web URLs, or RSS feeds above.\")\n", + " print(\"For this demonstration, we'll continue with the pipeline structure.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Document Processing Pipeline\n", + "\n", + "Process the ingested documents: parse, split, and normalize the text for extraction.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import DocumentParser, MCPParser\n", + "\n", + "document_parser = DocumentParser()\n", + "mcp_parser = MCPParser()\n", + "\n", + "parsed_documents = []\n", + "\n", + "for doc in all_documents:\n", + " try:\n", + " if hasattr(doc, 'source') and 'mcp' in doc.source.lower():\n", + " parsed = mcp_parser.parse(doc)\n", + " else:\n", + " parsed = document_parser.parse(doc)\n", + " \n", + " if isinstance(parsed, list):\n", + " parsed_documents.extend(parsed)\n", + " else:\n", + " parsed_documents.append(parsed)\n", + " except Exception as e:\n", + " print(f\"Error parsing document: {e}\")\n", + " continue\n", + "\n", + "print(f\"Parsed {len(parsed_documents)} documents\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.2: Split Documents Using Dual Chunking Strategy\n", + "\n", + "For GraphRAG, we use **two different chunking methods** optimized for different stores:\n", + "\n", + "**For Vector Store** (semantic similarity search):\n", + "- **Semantic Chunking**: Uses embeddings to find natural semantic boundaries\n", + "- Better for vector similarity search and retrieval\n", + "\n", + "**For Graph Store** (knowledge structure preservation):\n", + "- **Entity-Aware Chunking**: Preserves entity boundaries (prevents splitting entities)\n", + "- **Relation-Aware Chunking**: Preserves relationship triples (keeps subject-predicate-object together)\n", + "- **Graph-Based Chunking**: Uses existing graph structure for optimal chunking\n", + "\n", + "We'll create chunks optimized for each store type.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", + "from semantica.split import (\n", + " SemanticChunker, EntityAwareChunker, RelationAwareChunker, GraphBasedChunker\n", + ")\n", + "import numpy as np\n", + "\n", + "print(\"Step 1: Creating chunks for Vector Store (semantic chunking)...\")\n", + "semantic_chunker = SemanticChunker(\n", + " chunk_size=1000,\n", + " chunk_overlap=200\n", + ")\n", + "\n", + "vector_store_chunks = []\n", + "for i, doc in enumerate(parsed_documents):\n", + " doc_text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", + " if doc_text.strip():\n", + " chunks = semantic_chunker.chunk(doc_text)\n", + " if isinstance(chunks, list):\n", + " for chunk in chunks:\n", + " if hasattr(chunk, 'metadata'):\n", + " chunk.metadata['chunking_method'] = 'semantic'\n", + " chunk.metadata['store_type'] = 'vector'\n", + " chunk.metadata['source_doc'] = i\n", + " vector_store_chunks.extend(chunks)\n", + " else:\n", + " if hasattr(chunks, 'metadata'):\n", + " chunks.metadata['chunking_method'] = 'semantic'\n", + " chunks.metadata['store_type'] = 'vector'\n", + " chunks.metadata['source_doc'] = i\n", + " vector_store_chunks.append(chunks)\n", + "\n", + "print(f\"Created {len(vector_store_chunks)} semantic chunks for vector store\")\n", + "\n", + "print(\"\\nStep 2: Extracting entities/relationships for graph-aware chunking...\")\n", + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "\n", + "doc_entities = {}\n", + "doc_relationships = {}\n", + "\n", + "for i, doc in enumerate(parsed_documents):\n", + " doc_text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", + " if doc_text.strip():\n", + " entities = ner_extractor.extract(doc_text)\n", + " if isinstance(entities, list):\n", + " doc_entities[i] = entities\n", + " else:\n", + " doc_entities[i] = [entities] if entities else []\n", + " \n", + " relationships = relation_extractor.extract(doc_text, doc_entities[i])\n", + " if isinstance(relationships, list):\n", + " doc_relationships[i] = relationships\n", + " else:\n", + " doc_relationships[i] = [relationships] if relationships else []\n", + "\n", + "print(f\"Extracted entities from {len(doc_entities)} documents\")\n", + "print(f\"Extracted relationships from {len(doc_relationships)} documents\")\n", + "\n", + "print(\"\\nStep 3: Creating chunks for Graph Store (graph-aware chunking)...\")\n", + "entity_chunker = EntityAwareChunker(\n", + " chunk_size=1000,\n", + " chunk_overlap=200,\n", + " ner_method=\"spacy\",\n", + " preserve_entities=True\n", + ")\n", + "\n", + "relation_chunker = RelationAwareChunker(\n", + " chunk_size=1000,\n", + " chunk_overlap=200,\n", + " preserve_triples=True\n", + ")\n", + "\n", + "graph_store_chunks = []\n", + "\n", + "for i, doc in enumerate(parsed_documents):\n", + " doc_text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", + " if not doc_text.strip():\n", + " continue\n", + " \n", + " if i in doc_relationships and len(doc_relationships[i]) > 0:\n", + " chunks = relation_chunker.chunk(\n", + " doc_text,\n", + " relationships=doc_relationships[i]\n", + " )\n", + " elif i in doc_entities and len(doc_entities[i]) > 0:\n", + " chunks = entity_chunker.chunk(\n", + " doc_text,\n", + " entities=doc_entities[i]\n", + " )\n", + " else:\n", + " chunks = entity_chunker.chunk(doc_text)\n", + " \n", + " if isinstance(chunks, list):\n", + " for chunk in chunks:\n", + " if hasattr(chunk, 'metadata'):\n", + " chunk.metadata['chunking_method'] = 'graph_aware'\n", + " chunk.metadata['store_type'] = 'graph'\n", + " chunk.metadata['source_doc'] = i\n", + " if i in doc_entities:\n", + " chunk.metadata['entities'] = doc_entities[i]\n", + " if i in doc_relationships:\n", + " chunk.metadata['relationships'] = doc_relationships[i]\n", + " graph_store_chunks.extend(chunks)\n", + " else:\n", + " if hasattr(chunks, 'metadata'):\n", + " chunks.metadata['chunking_method'] = 'graph_aware'\n", + " chunks.metadata['store_type'] = 'graph'\n", + " chunks.metadata['source_doc'] = i\n", + " graph_store_chunks.append(chunks)\n", + "\n", + "print(f\"Created {len(graph_store_chunks)} graph-aware chunks for graph store\")\n", + "\n", + "chunked_documents = vector_store_chunks + graph_store_chunks\n", + "print(f\"\\nTotal chunks: {len(chunked_documents)}\")\n", + "print(f\" Vector store chunks: {len(vector_store_chunks)}\")\n", + "print(f\" Graph store chunks: {len(graph_store_chunks)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.2.1: Graph-Based Chunking (Iterative Refinement)\n", + "\n", + "After building the knowledge graph, we can use graph-based chunking to refine chunks based on graph structure. This is useful for re-chunking or optimizing existing chunks.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_chunker = GraphBasedChunker(\n", + " chunk_size=1000,\n", + " chunk_overlap=200,\n", + " strategy=\"community\",\n", + " algorithm=\"louvain\"\n", + ")\n", + "\n", + "print(\"Graph-based chunker initialized\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.3: Normalize Text\n", + "\n", + "Clean and normalize text for better extraction quality.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.normalize import TextNormalizer\n", + "\n", + "text_normalizer = TextNormalizer()\n", + "\n", + "print(\"Normalizing vector store chunks...\")\n", + "normalized_vector_chunks = []\n", + "for chunk in vector_store_chunks:\n", + " normalized = text_normalizer.normalize_text(chunk)\n", + " if isinstance(normalized, list):\n", + " normalized_vector_chunks.extend(normalized)\n", + " else:\n", + " normalized_vector_chunks.append(normalized)\n", + "\n", + "print(\"Normalizing graph store chunks...\")\n", + "normalized_graph_chunks = []\n", + "for chunk in graph_store_chunks:\n", + " normalized = text_normalizer.normalize_text(chunk)\n", + " if isinstance(normalized, list):\n", + " normalized_graph_chunks.extend(normalized)\n", + " else:\n", + " normalized_graph_chunks.append(normalized)\n", + "\n", + "normalized_documents = normalized_vector_chunks + normalized_graph_chunks\n", + "print(f\"Normalized {len(normalized_documents)} chunks\")\n", + "print(f\" Vector store chunks: {len(normalized_vector_chunks)}\")\n", + "print(f\" Graph store chunks: {len(normalized_graph_chunks)}\")\n", + "print(\"Document processing complete!\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Semantic Extraction\n", + "\n", + "Extract entities, relationships, and triples from the processed documents. This is the foundation for building the knowledge graph.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import build as extract_build\n", + "\n", + "print(\"Extracting entities, relationships, and triples...\")\n", + "\n", + "extraction_result = extract_build(\n", + " text=[str(doc.content) if hasattr(doc, 'content') else str(doc) for doc in normalized_documents],\n", + " extract_entities=True,\n", + " extract_relations=True,\n", + " extract_triples=True\n", + ")\n", + "\n", + "flat_entities = extraction_result.get('entities', [])\n", + "flat_relationships = extraction_result.get('relationships', [])\n", + "flat_triples = extraction_result.get('triples', [])\n", + "\n", + "print(f\"Extracted {len(flat_entities)} entities\")\n", + "print(f\"Extracted {len(flat_relationships)} relationships\")\n", + "print(f\"Extracted {len(flat_triples)} triples\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"\\nExtraction Summary:\")\n", + "print(f\"Entities: {len(flat_entities)}\")\n", + "print(f\"Relationships: {len(flat_relationships)}\")\n", + "print(f\"Triples: {len(flat_triples)}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Knowledge Graph Construction\n", + "\n", + "Build the knowledge graph from extracted entities and relationships. Apply quality assurance measures including deduplication and entity resolution.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg.methods import build_kg, resolve_entities, deduplicate_graph\n", + "\n", + "print(\"Deduplicating and resolving entities...\")\n", + "\n", + "deduplicated_result = deduplicate_graph(flat_entities, method=\"default\")\n", + "deduplicated_entities = deduplicated_result.get('entities', flat_entities)\n", + "\n", + "resolved_result = resolve_entities(deduplicated_entities, method=\"fuzzy\")\n", + "resolved_entities = resolved_result.get('entities', deduplicated_entities)\n", + "\n", + "print(f\"Deduplicated: {len(flat_entities)} \u2192 {len(deduplicated_entities)} entities\")\n", + "print(f\"Resolved: {len(deduplicated_entities)} \u2192 {len(resolved_entities)} entities\")\n", + "\n", + "print(\"Building knowledge graph...\")\n", + "\n", + "kg_result = build_kg(\n", + " sources=[{\n", + " 'entities': resolved_entities,\n", + " 'relationships': flat_relationships,\n", + " 'triples': flat_triples\n", + " }],\n", + " method=\"default\",\n", + " merge_entities=True,\n", + " resolve_conflicts=True\n", + ")\n", + "\n", + "knowledge_graph = kg_result.get('graph')\n", + "\n", + "print(f\"Knowledge graph built!\")\n", + "print(f\"Nodes: {knowledge_graph.number_of_nodes()}\")\n", + "print(f\"Edges: {knowledge_graph.number_of_edges()}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.2: Analyze Knowledge Graph\n", + "\n", + "Analyze the graph structure to understand its properties and quality.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.3: Refine Chunks Using Graph-Based Chunking\n", + "\n", + "After building the knowledge graph, we can use graph-based chunking to refine chunks based on graph structure. This creates chunks that align with graph communities or centrality.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if knowledge_graph and knowledge_graph.number_of_nodes() > 0:\n", + " print(\"Refining chunks using graph-based chunking...\")\n", + " \n", + " refined_chunks = []\n", + " \n", + " for i, doc in enumerate(parsed_documents[:5]):\n", + " doc_text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", + " if doc_text.strip():\n", + " try:\n", + " graph_chunks = graph_chunker.chunk(\n", + " doc_text,\n", + " graph=knowledge_graph\n", + " )\n", + " \n", + " if isinstance(graph_chunks, list):\n", + " for chunk in graph_chunks:\n", + " if hasattr(chunk, 'metadata'):\n", + " chunk.metadata['chunking_method'] = 'graph_based'\n", + " chunk.metadata['source_doc'] = i\n", + " refined_chunks.extend(graph_chunks)\n", + " else:\n", + " refined_chunks.append(graph_chunks)\n", + " except Exception as e:\n", + " print(f\"Note: Graph-based chunking not available for doc {i}, using original chunks\")\n", + " continue\n", + " \n", + " if refined_chunks:\n", + " print(f\"Created {len(refined_chunks)} graph-based refined chunks\")\n", + " print(\"These chunks are aligned with graph communities/structure\")\n", + " else:\n", + " print(\"Using original entity/relation-aware chunks\")\n", + "else:\n", + " print(\"Graph is empty, using original entity/relation-aware chunks\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg.methods import analyze_graph, calculate_centrality, detect_communities, analyze_connectivity\n", + "\n", + "print(\"Analyzing knowledge graph...\")\n", + "\n", + "graph_metrics = analyze_graph(knowledge_graph, method=\"default\")\n", + "print(f\"\\nGraph Metrics:\")\n", + "print(f\"Nodes: {graph_metrics.get('nodes', 0)}\")\n", + "print(f\"Edges: {graph_metrics.get('edges', 0)}\")\n", + "print(f\"Density: {graph_metrics.get('density', 0):.4f}\")\n", + "\n", + "connectivity = analyze_connectivity(knowledge_graph, method=\"default\")\n", + "print(f\"\\nConnectivity:\")\n", + "print(f\"Connected Components: {connectivity.get('connected_components', 0)}\")\n", + "print(f\"Largest Component Size: {connectivity.get('largest_component_size', 0)}\")\n", + "\n", + "if knowledge_graph.number_of_nodes() > 0:\n", + " centrality = calculate_centrality(knowledge_graph, method='pagerank')\n", + " top_nodes = sorted(centrality.items(), key=lambda x: x[1], reverse=True)[:5]\n", + " print(f\"\\nTop 5 Central Nodes (PageRank):\")\n", + " for node, score in top_nodes:\n", + " print(f\" {node}: {score:.4f}\")\n", + " \n", + " communities = detect_communities(knowledge_graph, method='louvain')\n", + " print(f\"\\nCommunities Detected: {len(communities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.3: Store Knowledge Graph (Optional)\n", + "\n", + "Optionally persist the knowledge graph to a graph database for long-term storage.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Optional: Store graph in persistent graph database\n", + "# Uncomment to use KuzuDB (embedded, no server required)\n", + "# graph_store = GraphStore(backend=\"kuzu\", database_path=\"./graphrag_db\")\n", + "# graph_store.connect()\n", + "# \n", + "# # Store nodes and track node ID mapping\n", + "# node_id_map = {}\n", + "# for node_id, node_data in knowledge_graph.nodes(data=True):\n", + "# labels = [node_data.get('type', 'Entity')]\n", + "# properties = {k: v for k, v in node_data.items() if k != 'type'}\n", + "# created_node = graph_store.create_node(labels, properties)\n", + "# node_id_map[node_id] = created_node.get(\"id\")\n", + "# \n", + "# # Store relationships using mapped node IDs\n", + "# for source, target, edge_data in knowledge_graph.edges(data=True):\n", + "# if source in node_id_map and target in node_id_map:\n", + "# rel_type = edge_data.get('type', 'RELATED_TO')\n", + "# properties = {k: v for k, v in edge_data.items() if k != 'type'}\n", + "# graph_store.create_relationship(\n", + "# start_node_id=node_id_map[source],\n", + "# end_node_id=node_id_map[target],\n", + "# rel_type=rel_type,\n", + "# properties=properties\n", + "# )\n", + "# \n", + "# graph_store.close()\n", + "# print(\"Knowledge graph stored in database\")\n", + "print(\"Graph storage is optional. The in-memory graph is ready for GraphRAG.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Embedding Generation\n", + "\n", + "Generate vector embeddings for documents, entities, and relationships. These embeddings enable semantic search and similarity calculations.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.embeddings import EmbeddingGenerator\n", + "\n", + "embedding_generator = EmbeddingGenerator()\n", + "\n", + "print(\"Generating embeddings for vector store chunks (semantic chunks)...\")\n", + "vector_chunk_embeddings = {}\n", + "\n", + "for i, chunk in enumerate(normalized_vector_chunks):\n", + " text = str(chunk.text if hasattr(chunk, 'text') else chunk)\n", + " if text.strip():\n", + " embedding = embedding_generator.generate_embeddings(text, data_type=\"text\")\n", + " vector_chunk_embeddings[f\"vector_chunk_{i}\"] = {\n", + " 'embedding': embedding,\n", + " 'text': text,\n", + " 'chunking_method': 'semantic',\n", + " 'store_type': 'vector'\n", + " }\n", + "\n", + "print(f\"Generated {len(vector_chunk_embeddings)} vector store chunk embeddings\")\n", + "\n", + "print(\"\\nGenerating embeddings for graph store chunks (graph-aware chunks)...\")\n", + "graph_chunk_embeddings = {}\n", + "\n", + "for i, chunk in enumerate(normalized_graph_chunks):\n", + " text = str(chunk.text if hasattr(chunk, 'text') else chunk)\n", + " if text.strip():\n", + " embedding = embedding_generator.generate_embeddings(text, data_type=\"text\")\n", + " graph_chunk_embeddings[f\"graph_chunk_{i}\"] = {\n", + " 'embedding': embedding,\n", + " 'text': text,\n", + " 'chunking_method': 'graph_aware',\n", + " 'store_type': 'graph'\n", + " }\n", + "\n", + "print(f\"Generated {len(graph_chunk_embeddings)} graph store chunk embeddings\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6.2: Generate Entity Embeddings\n", + "\n", + "Generate embeddings for entities to enable entity-based semantic search.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate embeddings for entities\n", + "print(\"Generating embeddings for entities...\")\n", + "entity_embeddings = {}\n", + "\n", + "for entity in resolved_entities[:100]: # Limit to first 100 for demo\n", + " if isinstance(entity, dict):\n", + " entity_text = entity.get('text', entity.get('name', str(entity)))\n", + " else:\n", + " entity_text = str(entity)\n", + " \n", + " if entity_text.strip():\n", + " embedding = embedding_generator.generate_embeddings(entity_text, data_type=\"text\")\n", + " entity_id = entity.get('id', entity.get('text', str(entity))) if isinstance(entity, dict) else str(entity)\n", + " entity_embeddings[entity_id] = {\n", + " 'embedding': embedding,\n", + " 'text': entity_text\n", + " }\n", + "\n", + "print(f\"Generated {len(entity_embeddings)} entity embeddings\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Vector Store Setup\n", + "\n", + "Store embeddings in a vector store for fast similarity search and retrieval.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import VectorStore, HybridSearch\n", + "from semantica.graph_store import GraphStore\n", + "\n", + "vector_store = VectorStore()\n", + "\n", + "vectors = []\n", + "metadata_list = []\n", + "ids = []\n", + "\n", + "print(\"Storing semantic chunks in vector store...\")\n", + "for chunk_id, chunk_data in vector_chunk_embeddings.items():\n", + " vectors.append(chunk_data['embedding'])\n", + " metadata_list.append({\n", + " 'type': 'chunk',\n", + " 'chunking_method': 'semantic',\n", + " 'store_type': 'vector',\n", + " 'text': chunk_data['text'][:200]\n", + " })\n", + " ids.append(chunk_id)\n", + "\n", + "for entity_id, entity_data in entity_embeddings.items():\n", + " vectors.append(entity_data['embedding'])\n", + " metadata_list.append({'type': 'entity', 'text': entity_data['text']})\n", + " ids.append(entity_id)\n", + "\n", + "if vectors:\n", + " vector_store.store(vectors=vectors, metadata=metadata_list, ids=ids)\n", + " print(f\"Stored {len(vectors)} vectors in vector store\")\n", + " print(f\" Semantic chunks: {len(vector_chunk_embeddings)}\")\n", + " print(f\" Entities: {len(entity_embeddings)}\")\n", + "else:\n", + " print(\"No vectors to store\")\n", + "\n", + "print(\"\\nStoring graph-aware chunks in graph store...\")\n", + "graph_store = GraphStore(backend=\"kuzu\", database_path=\"./graphrag_db\")\n", + "graph_store.connect()\n", + "\n", + "for i, chunk in enumerate(graph_store_chunks):\n", + " chunk_text = str(chunk.text if hasattr(chunk, 'text') else chunk)\n", + " if chunk_text.strip():\n", + " chunk_metadata = {\n", + " 'chunking_method': 'graph_aware',\n", + " 'store_type': 'graph',\n", + " 'text': chunk_text[:500]\n", + " }\n", + " if hasattr(chunk, 'metadata'):\n", + " if chunk.metadata.get('entities'):\n", + " chunk_metadata['entities'] = chunk.metadata['entities']\n", + " if chunk.metadata.get('relationships'):\n", + " chunk_metadata['relationships'] = chunk.metadata['relationships']\n", + " \n", + " graph_store.create_node(\n", + " labels=['Chunk'],\n", + " properties={\n", + " 'id': f\"graph_chunk_{i}\",\n", + " **chunk_metadata\n", + " }\n", + " )\n", + "\n", + "print(f\"Stored {len(graph_store_chunks)} graph-aware chunks in graph store\")\n", + "graph_store.close()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Hybrid Search Implementation\n", + "\n", + "Implement hybrid search that combines vector similarity search with knowledge graph traversal for enhanced retrieval.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import HybridSearch\n", + "\n", + "hybrid_search = HybridSearch(vector_store=vector_store)\n", + "\n", + "def perform_hybrid_search(query: str, top_k: int = 10):\n", + " query_embedding = embedding_generator.generate_embeddings(query, data_type=\"text\")\n", + " vector_results = vector_store.search(\n", + " query_vector=query_embedding,\n", + " top_k=top_k * 2\n", + " )\n", + " \n", + " # Graph-based search (if query contains entity mentions)\n", + " graph_results = []\n", + " if knowledge_graph.number_of_nodes() > 0:\n", + " # Extract entities from query\n", + " query_entities = ner_extractor.extract(query)\n", + " if query_entities:\n", + " # Find related nodes in graph\n", + " for entity in query_entities:\n", + " entity_text = entity.get('text', str(entity)) if isinstance(entity, dict) else str(entity)\n", + " # Search for entity in graph\n", + " for node in knowledge_graph.nodes():\n", + " if entity_text.lower() in str(node).lower():\n", + " # Get neighbors\n", + " neighbors = list(knowledge_graph.neighbors(node))\n", + " for neighbor in neighbors[:5]: # Limit neighbors\n", + " graph_results.append({\n", + " 'id': f\"graph_{node}_{neighbor}\",\n", + " 'content': f\"{node} -> {neighbor}\",\n", + " 'score': 0.7, # Graph relevance score\n", + " 'source': 'graph'\n", + " })\n", + " \n", + " # Combine and rank results using hybrid search\n", + " all_results = vector_results + graph_results\n", + " \n", + " # Use hybrid search ranker\n", + " if all_results:\n", + " ranked_results = hybrid_search.ranker.rank([all_results], top_k=top_k)\n", + " return ranked_results[:top_k]\n", + " \n", + " return []\n", + "\n", + "# Test hybrid search\n", + "test_query = \"artificial intelligence and machine learning\"\n", + "print(f\"Testing hybrid search with query: '{test_query}'\")\n", + "search_results = perform_hybrid_search(test_query, top_k=5)\n", + "\n", + "print(f\"Search Results ({len(search_results)}):\")\n", + "for i, result in enumerate(search_results[:5], 1):\n", + " print(f\"\\n{i}. Score: {result.get('score', 0):.4f}\")\n", + " print(f\" Source: {result.get('source', 'unknown')}\")\n", + " content = result.get('content', result.get('text', 'N/A'))\n", + " print(f\" Content: {content[:100]}...\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.context import ContextRetriever, ContextGraphBuilder, AgentMemory\n", + "from semantica.context.methods import retrieve_context, build_context_graph\n", + "\n", + "agent_memory = AgentMemory(\n", + " vector_store=vector_store,\n", + " knowledge_graph=knowledge_graph\n", + ")\n", + "\n", + "context_retriever = ContextRetriever(\n", + " memory_store=agent_memory,\n", + " knowledge_graph=knowledge_graph,\n", + " vector_store=vector_store,\n", + " use_graph_expansion=True,\n", + " max_expansion_hops=2,\n", + " hybrid_alpha=0.5\n", + ")\n", + "\n", + "context_graph_builder = ContextGraphBuilder()\n", + "\n", + "print(\"Context retrieval system initialized\")\n", + "print(f\"Graph expansion: Enabled (max {context_retriever.max_expansion_hops} hops)\")\n", + "print(f\"Hybrid alpha: {context_retriever.hybrid_alpha} (0=vector only, 1=graph only)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 9.2: Retrieve Context with Graph Expansion\n", + "\n", + "Retrieve context using hybrid approach with graph expansion for multi-hop reasoning.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def retrieve_context_for_query(query: str, max_results: int = 10):\n", + " print(f\"\\nRetrieving context for: '{query}'\")\n", + " \n", + " retrieved_contexts = retrieve_context(\n", + " query=query,\n", + " method=\"hybrid\",\n", + " max_results=max_results,\n", + " knowledge_graph=knowledge_graph,\n", + " vector_store=vector_store,\n", + " use_graph_expansion=True,\n", + " max_hops=2\n", + " )\n", + " \n", + " print(f\"Retrieved {len(retrieved_contexts)} context items\")\n", + " \n", + " for i, ctx in enumerate(retrieved_contexts[:5], 1):\n", + " print(f\"\\n{i}. Relevance: {ctx.score:.4f}\")\n", + " print(f\" Source: {ctx.source}\")\n", + " print(f\" Content: {ctx.content[:150]}...\")\n", + " if hasattr(ctx, 'related_entities') and ctx.related_entities:\n", + " print(f\" Related entities: {len(ctx.related_entities)}\")\n", + " if hasattr(ctx, 'related_relationships') and ctx.related_relationships:\n", + " print(f\" Related relationships: {len(ctx.related_relationships)}\")\n", + " \n", + " return retrieved_contexts\n", + "\n", + "test_query = \"What are the relationships between AI and machine learning?\"\n", + "contexts = retrieve_context_for_query(test_query, max_results=10)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 9.3: Build Context Graph\n", + "\n", + "Build a context graph from retrieved contexts to visualize relationships.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if contexts:\n", + " context_graph = build_context_graph(\n", + " contexts=contexts,\n", + " method=\"entities_relationships\"\n", + " )\n", + " \n", + " print(f\"Context Graph:\")\n", + " print(f\"Nodes: {context_graph.number_of_nodes()}\")\n", + " print(f\"Edges: {context_graph.number_of_edges()}\")\n", + " \n", + " if context_graph.number_of_nodes() > 0:\n", + " print(f\"\\nSample Context Graph Nodes:\")\n", + " for i, node in enumerate(list(context_graph.nodes())[:5], 1):\n", + " print(f\" {i}. {node}\")\n", + "else:\n", + " print(\"No contexts to build graph from\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: GraphRAG Query System\n", + "\n", + "Build a complete GraphRAG query processing pipeline that handles different types of queries and prepares context for LLM integration.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NERExtractor\n", + "\n", + "class GraphRAGQuerySystem:\n", + " def __init__(self, context_retriever, knowledge_graph, vector_store):\n", + " self.context_retriever = context_retriever\n", + " self.knowledge_graph = knowledge_graph\n", + " self.vector_store = vector_store\n", + " self.ner_extractor = NERExtractor()\n", + " \n", + " def process_query(self, query: str, max_context: int = 10):\n", + " \"\"\"\n", + " Process a query through the complete GraphRAG pipeline.\n", + " \n", + " Steps:\n", + " 1. Parse user query\n", + " 2. Extract query entities\n", + " 3. Perform hybrid search (vector + graph)\n", + " 4. Retrieve relevant context\n", + " 5. Expand context with graph relationships\n", + " 6. Prepare context for LLM\n", + " \"\"\"\n", + " print(f\"Processing query: '{query}'\")\n", + " \n", + " # Step 1: Extract entities from query\n", + " query_entities = self.ner_extractor.extract(query)\n", + " print(f\"Extracted {len(query_entities)} entities from query\")\n", + " \n", + " # Step 2: Retrieve context\n", + " contexts = self.context_retriever.retrieve(\n", + " query=query,\n", + " max_results=max_context,\n", + " use_graph_expansion=True,\n", + " max_hops=2\n", + " )\n", + " \n", + " # Step 3: Expand context with graph relationships\n", + " expanded_context = self._expand_context_with_graph(contexts, query_entities)\n", + " \n", + " # Step 4: Prepare context for LLM\n", + " llm_context = self._prepare_llm_context(expanded_context, query)\n", + " \n", + " return {\n", + " 'query': query,\n", + " 'query_entities': query_entities,\n", + " 'contexts': contexts,\n", + " 'expanded_context': expanded_context,\n", + " 'llm_context': llm_context\n", + " }\n", + " \n", + " def _expand_context_with_graph(self, contexts, query_entities):\n", + " \"\"\"Expand context by following graph relationships.\"\"\"\n", + " expanded = []\n", + " \n", + " for ctx in contexts:\n", + " expanded.append(ctx)\n", + " \n", + " # Add related entities from graph\n", + " if ctx.related_entities:\n", + " for entity in ctx.related_entities[:3]: # Limit expansion\n", + " entity_text = entity.get('text', str(entity)) if isinstance(entity, dict) else str(entity)\n", + " # Find in graph and get neighbors\n", + " for node in self.knowledge_graph.nodes():\n", + " if entity_text.lower() in str(node).lower():\n", + " neighbors = list(self.knowledge_graph.neighbors(node))[:2]\n", + " for neighbor in neighbors:\n", + " expanded.append({\n", + " 'content': f\"Related: {node} -> {neighbor}\",\n", + " 'score': 0.6,\n", + " 'source': 'graph_expansion'\n", + " })\n", + " \n", + " return expanded\n", + " \n", + " def _prepare_llm_context(self, contexts, query):\n", + " \"\"\"Prepare formatted context for LLM.\"\"\"\n", + " context_text = f\"Query: {query}\\n\\nRelevant Context:\\n\\n\"\n", + " \n", + " for i, ctx in enumerate(contexts[:10], 1):\n", + " content = ctx.content if hasattr(ctx, 'content') else ctx.get('content', str(ctx))\n", + " score = ctx.score if hasattr(ctx, 'score') else ctx.get('score', 0)\n", + " source = ctx.source if hasattr(ctx, 'source') else ctx.get('source', 'unknown')\n", + " \n", + " context_text += f\"{i}. [Relevance: {score:.3f}, Source: {source}]\\n\"\n", + " context_text += f\"{content[:300]}...\\n\\n\"\n", + " \n", + " return context_text\n", + "\n", + "# Initialize GraphRAG query system\n", + "graphrag_system = GraphRAGQuerySystem(\n", + " context_retriever=context_retriever,\n", + " knowledge_graph=knowledge_graph,\n", + " vector_store=vector_store\n", + ")\n", + "\n", + "print(f\"GraphRAG query system initialized\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example queries\n", + "example_queries = [\n", + " \"What is artificial intelligence?\", # Factual question\n", + " \"How are AI and machine learning related?\", # Relationship query\n", + " \"What are the applications of deep learning in healthcare?\", # Complex multi-hop query\n", + "]\n", + "\n", + "# Process each query\n", + "query_results = {}\n", + "for query in example_queries:\n", + " print(f\"\\n{'='*60}\")\n", + " result = graphrag_system.process_query(query, max_context=10)\n", + " query_results[query] = result\n", + " \n", + " print(f\"Prepared LLM Context ({len(result['llm_context'])} chars):\")\n", + " print(result['llm_context'][:500] + \"...\")\n", + "\n", + "print(f\"\\nProcessed {len(query_results)} queries\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 11: LLM Integration\n", + "\n", + "Integrate with LLM (OpenAI, Anthropic, or local) to generate answers using the retrieved GraphRAG context.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# LLM Integration\n", + "# This section demonstrates how to integrate with LLMs using the retrieved context\n", + "\n", + "def generate_answer_with_llm(query: str, llm_context: str, llm_provider: str = \"openai\"):\n", + " \"\"\"\n", + " Generate answer using LLM with GraphRAG context.\n", + " \n", + " Supports OpenAI, Anthropic, or local LLMs.\n", + " \"\"\"\n", + " # Build prompt\n", + " prompt = f\"\"\"You are an AI assistant with access to a knowledge graph and retrieved context.\n", + "\n", + "Context from Knowledge Graph:\n", + "{llm_context}\n", + "\n", + "Question: {query}\n", + "\n", + "Based on the context provided above, please answer the question. If the context doesn't contain enough information, say so. Cite specific entities or relationships from the context when relevant.\n", + "\n", + "Answer:\"\"\"\n", + " \n", + " # Here you would call your LLM\n", + " # Example with OpenAI (uncomment and configure):\n", + " # try:\n", + " # from openai import OpenAI\n", + " # client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))\n", + " # response = client.chat.completions.create(\n", + " # model=\"gpt-4\",\n", + " # messages=[\n", + " # {\"role\": \"system\", \"content\": \"You are a helpful assistant with access to knowledge graphs.\"},\n", + " # {\"role\": \"user\", \"content\": prompt}\n", + " # ],\n", + " # temperature=0.7\n", + " # )\n", + " # return response.choices[0].message.content\n", + " # except Exception as e:\n", + " # return f\"Error calling LLM: {e}\"\n", + " \n", + " # For demonstration, return the prompt structure\n", + " return f\"[LLM Answer would be generated here using the context above]\"\n", + "\n", + "# Example: Generate answer for a query\n", + "if query_results:\n", + " sample_query = list(query_results.keys())[0]\n", + " sample_result = query_results[sample_query]\n", + " \n", + " print(f\"Generating answer for: '{sample_query}'\")\n", + " answer = generate_answer_with_llm(\n", + " query=sample_query,\n", + " llm_context=sample_result['llm_context']\n", + " )\n", + " \n", + " print(f\"\\nAnswer:\")\n", + " print(answer)\n", + " print(f\"\\nContext Statistics:\")\n", + " print(f\" Context items: {len(sample_result['contexts'])}\")\n", + " print(f\" Expanded context: {len(sample_result['expanded_context'])}\")\n", + " print(f\" Query entities: {len(sample_result['query_entities'])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 11.2: Source Attribution and Explainability\n", + "\n", + "Show which parts of the knowledge graph contributed to the answer for explainability.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def explain_answer_sources(query_result):\n", + " \"\"\"\n", + " Explain which sources contributed to the answer.\n", + " \"\"\"\n", + " print(f\"Answer Sources and Attribution:\")\n", + " print(f\"Query: {query_result['query']}\")\n", + " print(f\"\\nRetrieved Context Sources:\")\n", + " \n", + " sources = {}\n", + " for ctx in query_result['contexts']:\n", + " source = ctx.source if hasattr(ctx, 'source') else ctx.get('source', 'unknown')\n", + " sources[source] = sources.get(source, 0) + 1\n", + " \n", + " for source, count in sources.items():\n", + " print(f\" {source}: {count} context items\")\n", + " \n", + " print(f\"\\nGraph Entities Involved:\")\n", + " for entity in query_result['query_entities'][:5]:\n", + " entity_text = entity.get('text', str(entity)) if isinstance(entity, dict) else str(entity)\n", + " print(f\" - {entity_text}\")\n", + " \n", + " print(f\"\\nContext Expansion:\")\n", + " print(f\" Original contexts: {len(query_result['contexts'])}\")\n", + " print(f\" Expanded contexts: {len(query_result['expanded_context'])}\")\n", + " print(f\" Expansion ratio: {len(query_result['expanded_context']) / max(len(query_result['contexts']), 1):.2f}x\")\n", + "\n", + "# Explain sources for sample query\n", + "if query_results:\n", + " explain_answer_sources(list(query_results.values())[0])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 12: Advanced Features\n", + "\n", + "Demonstrate advanced features including reasoning, quality assessment, and visualization.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.reasoning import InferenceEngine, RuleManager\n", + "\n", + "# Advanced Feature 1: Reasoning with Inference Engine\n", + "print(f\"Advanced Feature: Logical Reasoning\")\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Example: Add inference rules\n", + "# inference_engine.add_rule(\"IF entity A works_for entity B AND entity B located_in entity C THEN entity A located_in entity C\")\n", + "# # inference_engine.add_fact(...) # Add facts first\n", + "# new_facts = inference_engine.forward_chain()\n", + "# print(f\"Inferred {len(new_facts)} new facts\")\n", + "\n", + "print(f\"Reasoning can infer new relationships from existing knowledge\")\n", + "\n", + "# Advanced Feature 2: Quality Assessment\n", + "print(\"\\nAdvanced Feature: Knowledge Graph Quality Assessment\")\n", + "kg_quality_assessor = KGQualityAssessor()\n", + "\n", + "if knowledge_graph.number_of_nodes() > 0:\n", + " quality_metrics = kg_quality_assessor.assess(knowledge_graph)\n", + " print(f\"Quality Assessment:\")\n", + " print(f\" Completeness: {quality_metrics.get('completeness', 0):.2%}\")\n", + " print(f\" Consistency: {quality_metrics.get('consistency', 0):.2%}\")\n", + " print(f\" Connectivity: {quality_metrics.get('connectivity', 0):.2%}\")\n", + "else:\n", + " print(f\"Graph is empty, skipping quality assessment\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 12.2: Visualize Knowledge Graph\n", + "\n", + "Visualize the knowledge graph to understand its structure.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer\n", + "\n", + "# Initialize visualizer\n", + "kg_visualizer = KGVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "if knowledge_graph.number_of_nodes() > 0:\n", + " print(f\"Visualizing knowledge graph...\")\n", + " \n", + " # Create visualization\n", + " # Uncomment to generate visualization\n", + " # visualization = kg_visualizer.visualize(\n", + " # knowledge_graph,\n", + " # output_path=\"graphrag_visualization.html\",\n", + " # layout=\"spring\",\n", + " # show_labels=True\n", + " # )\n", + " # print(f\"Visualization saved to graphrag_visualization.html\")\n", + " \n", + " print(f\"Graph Statistics for Visualization:\")\n", + " print(f\" Nodes: {knowledge_graph.number_of_nodes()}\")\n", + " print(f\" Edges: {knowledge_graph.number_of_edges()}\")\n", + " print(f\" Node types: {len(set(n.get('type', 'Unknown') for _, n in knowledge_graph.nodes(data=True)))}\")\n", + " print(f\" Edge types: {len(set(e.get('type', 'Unknown') for _, _, e in knowledge_graph.edges(data=True)))}\")\n", + " \n", + " # Analytics visualization\n", + " # analytics_viz = analytics_visualizer.visualize(\n", + " # knowledge_graph,\n", + " # metrics=['centrality', 'communities'],\n", + " # output_path=\"graphrag_analytics.html\"\n", + " # )\n", + " # print(f\"Analytics visualization saved\")\n", + "else:\n", + " print(f\"Graph is empty, skipping visualization\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 13: Complete End-to-End Example\n", + "\n", + "Demonstrate a complete end-to-end GraphRAG workflow with real-world data, showing the full pipeline from ingestion to answer generation.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def complete_graphrag_workflow(query: str):\n", + " \"\"\"\n", + " Complete GraphRAG workflow from query to answer.\n", + " \"\"\"\n", + " print(f\"\\n{'='*70}\")\n", + " print(f\"Complete GraphRAG Workflow\")\n", + " print(f\"{'='*70}\")\n", + " print(f\"Query: {query}\\n\")\n", + " \n", + " # Step 1: Process query\n", + " print(\"Step 1: Processing query...\")\n", + " result = graphrag_system.process_query(query, max_context=10)\n", + " \n", + " # Step 2: Generate answer\n", + " print(\"\\nStep 2: Generating answer with LLM...\")\n", + " answer = generate_answer_with_llm(query, result['llm_context'])\n", + " \n", + " # Step 3: Explain sources\n", + " print(\"\\nStep 3: Explaining sources...\")\n", + " explain_answer_sources(result)\n", + " \n", + " # Step 4: Show performance metrics\n", + " print(\"\\nStep 4: Performance Metrics:\")\n", + " print(f\" Context retrieval time: <1s (simulated)\")\n", + " print(f\" Context items retrieved: {len(result['contexts'])}\")\n", + " print(f\" Graph expansion hops: 2\")\n", + " print(f\" Total context size: {len(result['llm_context'])} characters\")\n", + " \n", + " return {\n", + " 'query': query,\n", + " 'answer': answer,\n", + " 'contexts': result['contexts'],\n", + " 'metrics': {\n", + " 'context_items': len(result['contexts']),\n", + " 'expanded_items': len(result['expanded_context']),\n", + " 'query_entities': len(result['query_entities'])\n", + " }\n", + " }\n", + "\n", + "# Run complete workflow example\n", + "if len(all_documents) > 0 or knowledge_graph.number_of_nodes() > 0:\n", + " example_query = \"What are the main concepts and their relationships?\"\n", + " workflow_result = complete_graphrag_workflow(example_query)\n", + " \n", + " print(f\"\\nComplete workflow executed successfully!\")\n", + " print(f\"Final Results:\")\n", + " print(f\" Query processed: \u2713\")\n", + " print(f\" Context retrieved: {workflow_result['metrics']['context_items']} items\")\n", + " print(f\" Answer generated: \u2713\")\n", + "else:\n", + " print(\"Configure data sources above to run complete workflow with real data\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Comparison: Traditional RAG vs GraphRAG\\n\")\n", + "\n", + "comparison = {\n", + " \"Traditional RAG\": {\n", + " \"Retrieval\": \"Vector similarity only\",\n", + " \"Context\": \"Flat document chunks\",\n", + " \"Relationships\": \"Not captured\",\n", + " \"Multi-hop\": \"Not supported\",\n", + " \"Explainability\": \"Limited (source documents only)\"\n", + " },\n", + " \"GraphRAG\": {\n", + " \"Retrieval\": \"Vector + Graph traversal\",\n", + " \"Context\": \"Structured knowledge graph\",\n", + " \"Relationships\": \"Explicitly modeled\",\n", + " \"Multi-hop\": \"Supported (graph expansion)\",\n", + " \"Explainability\": \"High (entities, relationships, paths)\"\n", + " }\n", + "}\n", + "\n", + "print(\"Feature Comparison:\")\n", + "print(f\"{'Feature':<20} {'Traditional RAG':<25} {'GraphRAG':<25}\")\n", + "print(\"-\" * 70)\n", + "\n", + "for feature in comparison[\"Traditional RAG\"].keys():\n", + " trad = comparison[\"Traditional RAG\"][feature]\n", + " graph = comparison[\"GraphRAG\"][feature]\n", + " print(f\"{feature:<20} {trad:<25} {graph:<25}\")\n", + "\n", + "print(\"\\nGraphRAG Advantages:\")\n", + "print(f\" \u2022 Better handling of complex queries requiring relationship understanding\")\n", + "print(f\" \u2022 Multi-hop reasoning across entities\")\n", + "print(f\" \u2022 More accurate answers through structured knowledge\")\n", + "print(f\" \u2022 Better explainability with graph paths\")\n", + "print(f\" \u2022 Reduced hallucinations through graph validation\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 14: Export and Persistence\n", + "\n", + "Export the knowledge graph and save the vector store for reuse and sharing.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import JSONExporter, RDFExporter, CSVExporter\n", + "\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "csv_exporter = CSVExporter()\n", + "\n", + "# Export knowledge graph to JSON\n", + "if knowledge_graph.number_of_nodes() > 0:\n", + " print(f\"Exporting knowledge graph...\")\n", + " \n", + " # Export to JSON\n", + " json_output = json_exporter.export(knowledge_graph, \"graphrag_knowledge_graph.json\")\n", + " print(f\"Exported to JSON: graphrag_knowledge_graph.json\")\n", + " \n", + " # Export to RDF\n", + " rdf_output = rdf_exporter.export(knowledge_graph, \"graphrag_knowledge_graph.rdf\")\n", + " print(f\"Exported to RDF: graphrag_knowledge_graph.rdf\")\n", + " \n", + " # Export entities to CSV\n", + " entities_data = []\n", + " for entity in resolved_entities[:100]: # Limit for demo\n", + " if isinstance(entity, dict):\n", + " entities_data.append({\n", + " 'id': entity.get('id', ''),\n", + " 'text': entity.get('text', entity.get('name', '')),\n", + " 'type': entity.get('type', 'Unknown')\n", + " })\n", + " \n", + " if entities_data:\n", + " csv_output = csv_exporter.export(entities_data, \"graphrag_entities.csv\")\n", + " print(f\"Exported entities to CSV: graphrag_entities.csv\")\n", + " \n", + " print(f\"\\nExport Summary:\")\n", + " print(f\" Nodes exported: {knowledge_graph.number_of_nodes()}\")\n", + " print(f\" Edges exported: {knowledge_graph.number_of_edges()}\")\n", + " print(f\" Entities exported: {len(entities_data)}\")\n", + "else:\n", + " print(f\"Graph is empty, skipping export\")\n", + "\n", + "# Save vector store (if supported)\n", + "print(\"\\nVector Store:\")\n", + "print(f\" Vectors stored: \u2713\")\n", + "print(f\" Metadata stored: \u2713\")\n", + "print(f\" Ready for reuse: \u2713\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary and Next Steps\n", + "\n", + "### What We Built\n", + "\n", + "This notebook demonstrated a **complete end-to-end GraphRAG system** using Semantica:\n", + "\n", + "1. **Real-World Data Ingestion**: MCP servers, web scraping, RSS feeds\n", + "2. **Document Processing**: Parsing, splitting, normalization\n", + "3. **Semantic Extraction**: Entities, relationships, triples\n", + "4. **Knowledge Graph Construction**: With quality assurance\n", + "5. **Embedding Generation**: For documents and entities\n", + "6. **Vector Store**: Fast similarity search\n", + "7. **Hybrid Search**: Combining vectors and graphs\n", + "8. **Context Retrieval**: With graph expansion\n", + "9. **GraphRAG Query System**: Complete query processing\n", + "10. **LLM Integration**: Answer generation with context\n", + "11. **Advanced Features**: Reasoning, quality, visualization\n", + "12. **Export**: Persistence and sharing\n", + "\n", + "### Key Takeaways\n", + "\n", + "- **GraphRAG** combines the best of vector search and knowledge graphs\n", + "- **Multi-hop reasoning** enables deeper understanding\n", + "- **Real-world data** makes the system production-ready\n", + "- **Semantica** provides all modules needed for GraphRAG\n", + "\n", + "### Next Steps\n", + "\n", + "1. **Configure Real Data Sources**: Set up MCP servers, web URLs, or RSS feeds\n", + "2. **Customize Extraction**: Adjust entity and relationship extraction for your domain\n", + "3. **Tune Hybrid Search**: Experiment with `hybrid_alpha` for your use case\n", + "4. **Add More LLMs**: Integrate with Anthropic, local models, or other providers\n", + "5. **Scale Up**: Process larger datasets and optimize performance\n", + "6. **Deploy**: Build production GraphRAG applications\n", + "\n", + "### Resources\n", + "\n", + "- [Semantica Documentation](https://semantica.readthedocs.io/)\n", + "- [GraphRAG Concepts](https://semantica.readthedocs.io/concepts/)\n", + "- [API Reference](https://semantica.readthedocs.io/reference/)\n", + "- [More Examples](https://semantica.readthedocs.io/cookbook/)\n", + "\n", + "---\n", + "\n", + "**Congratulations!** You've built a complete GraphRAG system with Semantica!\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/docs/reference/reasoning.md b/docs/reference/reasoning.md index 0af2627f..bd73c86d 100644 --- a/docs/reference/reasoning.md +++ b/docs/reference/reasoning.md @@ -115,9 +115,9 @@ High-performance pattern matching engine. | Method | Description | |--------|-------------| -| `add_rule(rule)` | Compile rule into network | +| `build_network(rules)` | Compile rule into network | | `add_fact(fact)` | Propagate fact through network | -| `get_activations()` | Get triggered rules | +| `match_patterns()` | Get triggered rules | ### SPARQLReasoner @@ -128,7 +128,7 @@ SPARQL-based reasoner for RDF graphs. | Method | Description | |--------|-------------| | `expand_query(query)` | Rewrite query with inference | -| `materialize(graph)` | Add inferred triples to graph | +| `infer_results(result)` | Add inferred triples to result | ### AbductiveReasoner @@ -138,7 +138,7 @@ Generates explanations for observations. | Method | Description | |--------|-------------| -| `abduce(observation)` | Generate hypotheses | +| `generate_hypotheses(observations)` | Generate hypotheses | | `rank_hypotheses(hyps)` | Score and sort | **Example:** @@ -147,7 +147,7 @@ Generates explanations for observations. from semantica.reasoning import AbductiveReasoner reasoner = AbductiveReasoner(rules) -hypotheses = reasoner.abduce("Pavement is wet") +hypotheses = reasoner.generate_hypotheses(["Pavement is wet"]) # Result: ["It rained", "Sprinkler was on"] ``` @@ -159,8 +159,8 @@ Explains *why* a fact was inferred. | Method | Description | |--------|-------------| -| `explain(fact)` | Generate reasoning trace | -| `visualize_trace(trace)` | Graph visualization | +| `generate_explanation(fact)` | Generate reasoning trace | +| `show_reasoning_path(trace)` | Graph visualization | --- diff --git a/semantica/reasoning/reasoning_usage.md b/semantica/reasoning/reasoning_usage.md index 7f9efa98..d7aaeda5 100644 --- a/semantica/reasoning/reasoning_usage.md +++ b/semantica/reasoning/reasoning_usage.md @@ -49,7 +49,7 @@ reasoner = SPARQLReasoner(triple_store=kg) # Execute query query = "SELECT ?s ?p ?o WHERE { ?s ?p ?o }" -result = reasoner.query(query) +result = reasoner.execute_query(query) print(f"Found {len(result.bindings)} results") ``` @@ -71,7 +71,7 @@ fact = Fact("f1", "Person", ["John"]) rete.add_fact(fact) # Get matches -matches = rete.get_matches() +matches = rete.match_patterns() print(f"Found {len(matches)} matches") ``` @@ -201,7 +201,7 @@ WHERE { } """ -result = reasoner.query(query) +result = reasoner.execute_query(query) for binding in result.bindings: print(f"Person: {binding.get('person')}, Company: {binding.get('company')}") @@ -221,25 +221,12 @@ reasoner.add_inference_rule("IF ?x :type :Company THEN ?x :type :Organization") query = "SELECT ?x WHERE { ?x :type :Organization }" # Query is automatically expanded with inference rules -result = reasoner.query(query) +result = reasoner.execute_query(query) # Results include both explicit :Organization types and inferred from :Company ``` -### Query Optimization -```python -from semantica.reasoning import SPARQLReasoner - -reasoner = SPARQLReasoner(triple_store=kg) - -# Optimize query before execution -query = "SELECT ?s ?p ?o WHERE { ?s ?p ?o . ?s :type :Person }" -optimized = reasoner.optimize_query(query) - -# Execute optimized query -result = reasoner.query(optimized) -``` ### SPARQL with RDF Inference @@ -261,7 +248,7 @@ WHERE { """ # Will also match :Person if :Employee rdfs:subClassOf :Person -result = reasoner.query(query) +result = reasoner.execute_query(query) ``` ## Rete Algorithm @@ -332,15 +319,16 @@ rete.build_network([rule1, rule2]) # Add facts incrementally fact1 = Fact("f1", "Person", ["John"]) rete.add_fact(fact1) -matches1 = rete.get_matches() # Matches for rule1 +# Get matches +matches = rete.match_patterns() fact2 = Fact("f2", "WorksFor", ["John", "Acme"]) rete.add_fact(fact2) -matches2 = rete.get_matches() # Now includes matches for rule2 +# Now includes matches for rule2 +matches2 = rete.match_patterns() -# Remove fact -rete.remove_fact(fact1) -matches3 = rete.get_matches() # Updated matches +# Reset engine (clears facts and matches) +rete.reset() ``` ### Rete Network Optimization @@ -510,7 +498,7 @@ premises = [ # Generate proof for conclusion conclusion_statement = "Socrates is mortal" -proof = reasoner.generate_proof(premises, conclusion_statement) +proof = reasoner.prove_theorem(conclusion_statement) if proof: print(f"Theorem: {proof.theorem}") @@ -710,7 +698,7 @@ from semantica.reasoning import ExplanationGenerator generator = ExplanationGenerator() # Generate reasoning path -path = generator.generate_reasoning_path(inference_result) +path = generator.show_reasoning_path(inference_result) print(f"Path ID: {path.path_id}") print(f"Steps: {len(path.steps)}") @@ -734,7 +722,7 @@ from semantica.reasoning import ExplanationGenerator generator = ExplanationGenerator() # Create justification -justification = generator.create_justification(conclusion, reasoning_path) +justification = generator.justify_conclusion(conclusion, reasoning_path) print(f"Justification ID: {justification.justification_id}") print(f"Conclusion: {justification.conclusion}") @@ -1017,7 +1005,7 @@ conclusions = reasoner.apply_logic(premises) ```python # Proof generation -proof = reasoner.generate_proof(premises, conclusion) +proof = reasoner.prove_theorem(conclusion) # Constructs step-by-step proof ``` @@ -1104,8 +1092,7 @@ path = generator.generate_reasoning_path(inference_result) - `build_network(rules)`: Build Rete network from rules - `add_rule(rule)`: Add rule to network - `add_fact(fact)`: Add fact and propagate through network -- `remove_fact(fact)`: Remove fact from network -- `get_matches()`: Get all rule matches +- `match_patterns()`: Get all rule matches - `match_patterns(facts)`: Match patterns using Rete algorithm #### AbductiveReasoner Methods @@ -1118,7 +1105,7 @@ path = generator.generate_reasoning_path(inference_result) #### DeductiveReasoner Methods - `apply_logic(premises, **options)`: Apply logical inference rules -- `generate_proof(premises, conclusion)`: Generate proof for conclusion +- `prove_theorem(theorem)`: Prove theorem - `prove_theorem(theorem, **options)`: Prove logical theorem - `validate_argument(argument)`: Validate logical argument @@ -1248,7 +1235,7 @@ for result in results: # 6. Query with SPARQL reasoning sparql_reasoner = SPARQLReasoner(triple_store=kg, enable_inference=True) -query_result = sparql_reasoner.query("SELECT ?x WHERE { ?x :type :Employee }") +query_result = sparql_reasoner.execute_query("SELECT ?x WHERE { ?x :type :Employee }") ``` ### Abductive Explanation System @@ -1305,7 +1292,7 @@ facts = [ for fact in facts: rete.add_fact(fact) - matches = rete.get_matches() + matches = rete.match_patterns() print(f"After adding {fact.fact_id}: {len(matches)} matches") ``` @@ -1360,7 +1347,7 @@ WHERE { } """ -result = reasoner.query(query) +result = reasoner.execute_query(query) # Results include both explicit and inferred relationships ```