diff --git a/cookbook/advanced/01_Advanced_Extraction.ipynb b/cookbook/advanced/01_Advanced_Extraction.ipynb index 2d6219ae..38736d04 100644 --- a/cookbook/advanced/01_Advanced_Extraction.ipynb +++ b/cookbook/advanced/01_Advanced_Extraction.ipynb @@ -37,7 +37,7 @@ "\n", "---\n", "\n", - "## Workflow: Event Detection → Coreference Resolution → Triple Extraction → Semantic Analysis → Network Extraction → LLM Enhancement → Validation\n" + "## Workflow: Event Detection \u2192 Coreference Resolution \u2192 Triple Extraction \u2192 Semantic Analysis \u2192 Network Extraction \u2192 LLM Enhancement \u2192 Validation\n" ] }, { @@ -46,9 +46,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -57,7 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "from semantica.semantic_extract import (\n", + "from semantica.semantic_extract import (\n", " EventDetector, CoreferenceResolver, TripleExtractor,\n", " SemanticAnalyzer, SemanticNetworkExtractor, LLMEnhancer, ExtractionValidator\n", ")\n", @@ -234,4 +232,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/advanced/03_Complete_Visualization_Suite.ipynb b/cookbook/advanced/03_Complete_Visualization_Suite.ipynb index 719ae3da..46a51f79 100644 --- a/cookbook/advanced/03_Complete_Visualization_Suite.ipynb +++ b/cookbook/advanced/03_Complete_Visualization_Suite.ipynb @@ -1,5 +1,5 @@ { - "cells": [ + "cells": [ { "cell_type": "markdown", "metadata": {}, @@ -31,9 +31,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -51,7 +49,6 @@ ")\n", "from semantica.kg import GraphBuilder, GraphAnalyzer\n", "from semantica.embeddings import EmbeddingGenerator\n", - "import numpy as np\n" ] }, @@ -287,4 +284,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/advanced/04_Conflict_Resolution_Strategies.ipynb b/cookbook/advanced/04_Conflict_Resolution_Strategies.ipynb index 119b5330..86a2ca5f 100644 --- a/cookbook/advanced/04_Conflict_Resolution_Strategies.ipynb +++ b/cookbook/advanced/04_Conflict_Resolution_Strategies.ipynb @@ -25,7 +25,7 @@ "pip install semantica[all]\n", "```\n", "\n", - "## Workflow: Detect Conflicts → Multiple Resolution Strategies → Track Sources → Audit\n" + "## Workflow: Detect Conflicts \u2192 Multiple Resolution Strategies \u2192 Track Sources \u2192 Audit\n" ] }, { @@ -34,9 +34,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -255,4 +253,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/advanced/05_Multi_Format_Export.ipynb b/cookbook/advanced/05_Multi_Format_Export.ipynb index 061f51eb..4e1cf204 100644 --- a/cookbook/advanced/05_Multi_Format_Export.ipynb +++ b/cookbook/advanced/05_Multi_Format_Export.ipynb @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -604,14 +602,14 @@ " \"exports/report.txt\"\n", "]\n", "\n", - "print(\"📊 Export Summary:\")\n", + "print(\"\ud83d\udcca Export Summary:\")\n", "print(\"=\" * 60)\n", "for file in export_files:\n", " if os.path.exists(file):\n", " size = os.path.getsize(file)\n", - " print(f\"✅ {file:50} ({size:>10,} bytes)\")\n", + " print(f\"\u2705 {file:50} ({size:>10,} bytes)\")\n", " else:\n", - " print(f\"❌ {file:50} (not found)\")\n", + " print(f\"\u274c {file:50} (not found)\")\n", "\n", "print(\"=\" * 60)\n", "print(f\"Total files checked: {len(export_files)}\")\n", @@ -631,4 +629,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb b/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb index 6d250020..db0567f4 100644 --- a/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb +++ b/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb @@ -35,7 +35,7 @@ "\n", "---\n", "\n", - "## Workflow: Multi-Source Ingestion → Entity Resolution → Conflict Detection → Provenance Tracking → Unified KG\n" + "## Workflow: Multi-Source Ingestion \u2192 Entity Resolution \u2192 Conflict Detection \u2192 Provenance Tracking \u2192 Unified KG\n" ] }, { @@ -44,9 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -218,4 +216,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/advanced/07_Pipeline_Orchestration.ipynb b/cookbook/advanced/07_Pipeline_Orchestration.ipynb index af5c0d77..225f4d04 100644 --- a/cookbook/advanced/07_Pipeline_Orchestration.ipynb +++ b/cookbook/advanced/07_Pipeline_Orchestration.ipynb @@ -25,7 +25,7 @@ "pip install semantica[all]\n", "```\n", "\n", - "## Workflow: Build Pipelines → Execute → Handle Failures → Parallel Processing → Monitor\n" + "## Workflow: Build Pipelines \u2192 Execute \u2192 Handle Failures \u2192 Parallel Processing \u2192 Monitor\n" ] }, { @@ -34,9 +34,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -241,4 +239,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/advanced/08_Reasoning_and_Inference.ipynb b/cookbook/advanced/08_Reasoning_and_Inference.ipynb index 12b06e72..add6d98b 100644 --- a/cookbook/advanced/08_Reasoning_and_Inference.ipynb +++ b/cookbook/advanced/08_Reasoning_and_Inference.ipynb @@ -1,347 +1,345 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n", - "\n", - "# Reasoning and Inference\n", - "\n", - "## Overview\n", - "\n", - "Build knowledge graphs, define rules, perform forward/backward chaining, and generate explanations for AI reasoning using the **Semantica Reasoning Module**.\n", - "\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/reasoning/)\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica from PyPI:\n", - "\n", - "```bash\n", - "pip install semantica\n", - "# Or with all optional dependencies:\n", - "pip install semantica[all]\n", - "```\n", - "\n", - "## Workflow: Build KG → Define Rules → Forward/Backward Chaining → Generate Explanations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphBuilder\n", - "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Build Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "builder = GraphBuilder()\n", - "\n", - "entities = [\n", - " {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n", - " {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n", - " {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n", - " {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n", - " {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"},\n", - "]\n", - "\n", - "relationships = [\n", - " {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n", - " {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n", - " {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n", - " {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"},\n", - "]\n", - "\n", - "knowledge_graph = builder.build([{\"entities\": entities, \"relationships\": relationships}])\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Define Rules\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize Inference Engine\n", - "engine = InferenceEngine()\n", - "\n", - "# Define rules using logic syntax\n", - "rules = [\n", - " \"IF parent_of(?a, ?b) AND parent_of(?b, ?c) THEN grandparent_of(?a, ?c)\",\n", - " \"IF lives_in(?x, ?y) AND located_in(?y, ?z) THEN lives_in(?x, ?z)\"\n", - "]\n", - "\n", - "for rule in rules:\n", - " engine.add_rule(rule)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Forward Chaining\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load facts from relationships into the engine\n", - "for rel in relationships:\n", - " # Format: predicate(subject, object)\n", - " fact_str = f\"{rel['type']}({rel['source']}, {rel['target']})\"\n", - " engine.add_fact(fact_str)\n", - "\n", - "# Perform forward chaining to derive new facts\n", - "results = engine.forward_chain()\n", - "\n", - "print(f\"Inferred {len(results)} new facts:\")\n", - "for result in results:\n", - " print(f\" - {result.conclusion} (Rule: {result.rule_used.name})\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Backward Chaining\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Define a goal to prove\n", - "goal = \"grandparent_of(alice, charlie)\"\n", - "\n", - "# Perform backward chaining\n", - "proof = engine.backward_chain(goal)\n", - "\n", - "if proof:\n", - " print(f\"Goal '{goal}' proven successfully!\")\n", - "else:\n", - " print(f\"Could not prove goal '{goal}'.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Generate Explanations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "generator = ExplanationGenerator()\n", - "\n", - "# Explain the last forward chaining inference\n", - "if results:\n", - " explanation = generator.generate_explanation(results[0])\n", - " print(\"Explanation for first inferred fact:\")\n", - " print(explanation.natural_language)\n", - "\n", - "# If we have a proof from backward chaining, explain it\n", - "if proof:\n", - " proof_explanation = generator.generate_explanation(proof)\n", - " print(\"\\nExplanation for backward chaining proof:\")\n", - " print(proof_explanation.natural_language)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Reasoning and inference workflow:\n", - "- Knowledge Graph Built\n", - "- Inference Rules Defined\n", - "- Facts Loaded into Engine\n", - "- Forward Chaining Performed\n", - "- Backward Chaining Performed\n", - "- Explanations Generated\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "### Deep Dive: Reasoning Module\n", - "\n", - "This section provides an in-depth guide to Semantica's reasoning capabilities. Learn rule syntax, fact formats, chaining strategies, and explanation generation with robust, reproducible examples.\n", - "\n", - "**What you'll practice**\n", - "- Defining rules with variables and predicates\n", - "- Loading facts in predicate form\n", - "- Running forward and backward chaining\n", - "- Generating human-readable explanations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphBuilder\n", - "from semantica.reasoning import InferenceEngine, ExplanationGenerator\n", - "\n", - "builder = GraphBuilder()\n", - "engine = InferenceEngine()\n", - "explainer = ExplanationGenerator()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Rule Syntax\n", - "\n", - "Rules use predicate logic with variables prefixed by `?`.\n", - "\n", - "- Example: `IF parent_of(?a, ?b) AND parent_of(?b, ?c) THEN grandparent_of(?a, ?c)`\n", - "- Variables unify across predicates in the same rule\n", - "- Conclusions are added as new facts when conditions match\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "entities = [\n", - " {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n", - " {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n", - " {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n", - " {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n", - " {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"}\n", - "]\n", - "\n", - "relationships = [\n", - " {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n", - " {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n", - " {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n", - " {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"}\n", - "]\n", - "\n", - "knowledge_graph = builder.build([{\"entities\": entities, \"relationships\": relationships}])\n", - "print(len(knowledge_graph.get(\"entities\", [])))\n", - "print(len(knowledge_graph.get(\"relationships\", [])))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "rules = [\n", - " \"IF parent_of(?a, ?b) AND parent_of(?b, ?c) THEN grandparent_of(?a, ?c)\",\n", - " \"IF lives_in(?x, ?y) AND located_in(?y, ?z) THEN lives_in(?x, ?z)\"\n", - "]\n", - "for r in rules:\n", - " engine.add_rule(r)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for rel in relationships:\n", - " fact = f\"{rel['type']}({rel['source']}, {rel['target']})\"\n", - " engine.add_fact(fact)\n", - "\n", - "derived = engine.forward_chain()\n", - "print(len(derived))\n", - "for d in derived:\n", - " print(d.conclusion)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "goals = [\n", - " \"grandparent_of(alice, charlie)\",\n", - " \"lives_in(alice, california)\"\n", - "]\n", - "for g in goals:\n", - " proof = engine.backward_chain(g)\n", - " print(g)\n", - " print(bool(proof))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if derived:\n", - " exp = explainer.generate_explanation(derived[0])\n", - " print(exp.natural_language)\n", - "\n", - "goal = \"grandparent_of(alice, charlie)\"\n", - "proof = engine.backward_chain(goal)\n", - "if proof:\n", - " pexp = explainer.generate_explanation(proof)\n", - " print(pexp.natural_language)\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)\n", + "\n", + "# Reasoning and Inference\n", + "\n", + "## Overview\n", + "\n", + "Build knowledge graphs, define rules, perform forward/backward chaining, and generate explanations for AI reasoning using the **Semantica Reasoning Module**.\n", + "\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/reasoning/)\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica from PyPI:\n", + "\n", + "```bash\n", + "pip install semantica\n", + "# Or with all optional dependencies:\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "## Workflow: Build KG \u2192 Define Rules \u2192 Forward/Backward Chaining \u2192 Generate Explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Build Knowledge Graph\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n", + " {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n", + " {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n", + " {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n", + " {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"},\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n", + " {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n", + " {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n", + " {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"},\n", + "]\n", + "\n", + "knowledge_graph = builder.build([{\"entities\": entities, \"relationships\": relationships}])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Define Rules\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Inference Engine\n", + "engine = InferenceEngine()\n", + "\n", + "# Define rules using logic syntax\n", + "rules = [\n", + " \"IF parent_of(?a, ?b) AND parent_of(?b, ?c) THEN grandparent_of(?a, ?c)\",\n", + " \"IF lives_in(?x, ?y) AND located_in(?y, ?z) THEN lives_in(?x, ?z)\"\n", + "]\n", + "\n", + "for rule in rules:\n", + " engine.add_rule(rule)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Forward Chaining\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load facts from relationships into the engine\n", + "for rel in relationships:\n", + " # Format: predicate(subject, object)\n", + " fact_str = f\"{rel['type']}({rel['source']}, {rel['target']})\"\n", + " engine.add_fact(fact_str)\n", + "\n", + "# Perform forward chaining to derive new facts\n", + "results = engine.forward_chain()\n", + "\n", + "print(f\"Inferred {len(results)} new facts:\")\n", + "for result in results:\n", + " print(f\" - {result.conclusion} (Rule: {result.rule_used.name})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Backward Chaining\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a goal to prove\n", + "goal = \"grandparent_of(alice, charlie)\"\n", + "\n", + "# Perform backward chaining\n", + "proof = engine.backward_chain(goal)\n", + "\n", + "if proof:\n", + " print(f\"Goal '{goal}' proven successfully!\")\n", + "else:\n", + " print(f\"Could not prove goal '{goal}'.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generator = ExplanationGenerator()\n", + "\n", + "# Explain the last forward chaining inference\n", + "if results:\n", + " explanation = generator.generate_explanation(results[0])\n", + " print(\"Explanation for first inferred fact:\")\n", + " print(explanation.natural_language)\n", + "\n", + "# If we have a proof from backward chaining, explain it\n", + "if proof:\n", + " proof_explanation = generator.generate_explanation(proof)\n", + " print(\"\\nExplanation for backward chaining proof:\")\n", + " print(proof_explanation.natural_language)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Reasoning and inference workflow:\n", + "- Knowledge Graph Built\n", + "- Inference Rules Defined\n", + "- Facts Loaded into Engine\n", + "- Forward Chaining Performed\n", + "- Backward Chaining Performed\n", + "- Explanations Generated\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "### Deep Dive: Reasoning Module\n", + "\n", + "This section provides an in-depth guide to Semantica's reasoning capabilities. Learn rule syntax, fact formats, chaining strategies, and explanation generation with robust, reproducible examples.\n", + "\n", + "**What you'll practice**\n", + "- Defining rules with variables and predicates\n", + "- Loading facts in predicate form\n", + "- Running forward and backward chaining\n", + "- Generating human-readable explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "from semantica.reasoning import InferenceEngine, ExplanationGenerator\n", + "\n", + "builder = GraphBuilder()\n", + "engine = InferenceEngine()\n", + "explainer = ExplanationGenerator()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Rule Syntax\n", + "\n", + "Rules use predicate logic with variables prefixed by `?`.\n", + "\n", + "- Example: `IF parent_of(?a, ?b) AND parent_of(?b, ?c) THEN grandparent_of(?a, ?c)`\n", + "- Variables unify across predicates in the same rule\n", + "- Conclusions are added as new facts when conditions match\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "entities = [\n", + " {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n", + " {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n", + " {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n", + " {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n", + " {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"}\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n", + " {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n", + " {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n", + " {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"}\n", + "]\n", + "\n", + "knowledge_graph = builder.build([{\"entities\": entities, \"relationships\": relationships}])\n", + "print(len(knowledge_graph.get(\"entities\", [])))\n", + "print(len(knowledge_graph.get(\"relationships\", [])))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rules = [\n", + " \"IF parent_of(?a, ?b) AND parent_of(?b, ?c) THEN grandparent_of(?a, ?c)\",\n", + " \"IF lives_in(?x, ?y) AND located_in(?y, ?z) THEN lives_in(?x, ?z)\"\n", + "]\n", + "for r in rules:\n", + " engine.add_rule(r)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for rel in relationships:\n", + " fact = f\"{rel['type']}({rel['source']}, {rel['target']})\"\n", + " engine.add_fact(fact)\n", + "\n", + "derived = engine.forward_chain()\n", + "print(len(derived))\n", + "for d in derived:\n", + " print(d.conclusion)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "goals = [\n", + " \"grandparent_of(alice, charlie)\",\n", + " \"lives_in(alice, california)\"\n", + "]\n", + "for g in goals:\n", + " proof = engine.backward_chain(g)\n", + " print(g)\n", + " print(bool(proof))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if derived:\n", + " exp = explainer.generate_explanation(derived[0])\n", + " print(exp.natural_language)\n", + "\n", + "goal = \"grandparent_of(alice, charlie)\"\n", + "proof = engine.backward_chain(goal)\n", + "if proof:\n", + " pexp = explainer.generate_explanation(proof)\n", + " print(pexp.natural_language)\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/cookbook/advanced/09_Semantic_Layer_Construction.ipynb b/cookbook/advanced/09_Semantic_Layer_Construction.ipynb index c23daae8..a392c350 100644 --- a/cookbook/advanced/09_Semantic_Layer_Construction.ipynb +++ b/cookbook/advanced/09_Semantic_Layer_Construction.ipynb @@ -25,7 +25,7 @@ "pip install semantica[all]\n", "```\n", "\n", - "## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triplet Store\n" + "## Workflow: Build KG \u2192 Generate Ontology \u2192 Create Semantic Layer \u2192 Export RDF \u2192 Triplet Store\n" ] }, { @@ -34,9 +34,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -208,4 +206,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb b/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb index 38c2d393..7289b337 100644 --- a/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb +++ b/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb @@ -34,7 +34,7 @@ "\n", "---\n", "\n", - "## Workflow: Build Temporal KG → Time-Aware Queries → Pattern Detection → Version Management → Visualization\n" + "## Workflow: Build Temporal KG \u2192 Time-Aware Queries \u2192 Pattern Detection \u2192 Version Management \u2192 Visualization\n" ] }, { @@ -43,9 +43,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -238,4 +236,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/advanced/11_Advanced_Context_Engineering.ipynb b/cookbook/advanced/11_Advanced_Context_Engineering.ipynb index c8abb8c7..fbe16366 100644 --- a/cookbook/advanced/11_Advanced_Context_Engineering.ipynb +++ b/cookbook/advanced/11_Advanced_Context_Engineering.ipynb @@ -26,6 +26,15 @@ "We'll start by setting up a mock vector store and importing necessary components." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, { "cell_type": "code", "execution_count": null, @@ -244,4 +253,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/cookbook/advanced/12_Unstructured_to_Ontology.ipynb b/cookbook/advanced/12_Unstructured_to_Ontology.ipynb index d8da84d2..352c8252 100644 --- a/cookbook/advanced/12_Unstructured_to_Ontology.ipynb +++ b/cookbook/advanced/12_Unstructured_to_Ontology.ipynb @@ -1,297 +1,286 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)\n", - "\n", - "# Advanced: Unstructured Text to Ontology\n", - "\n", - "Welcome to the advanced guide on extracting structured ontologies from unstructured text. This notebook explores two powerful paradigms available in Semantica:\n", - "\n", - "1. **Classical NLP Pipeline**: Using Named Entity Recognition (NER) and Relation Extraction.\n", - "2. **Generative AI Pipeline**: Using Large Language Models (LLMs) for direct conceptual modeling.\n", - "\n", - "We will compare both approaches, visualize the results, and validate the generated ontologies.\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/ontology/)\n", - "\n", - "## Setup and Installation\n", - "\n", - "Ensure you have Semantica installed with all dependencies." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install semantica[all]\n", - "\n", - "from semantica.utils.logging import get_logger\n", - "\n", - "logger = get_logger(\"unstructured_guide\")\n", - "print(\"Environment setup complete.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## The Input Text\n", - "\n", - "We will use a rich paragraph of text describing a technology company to test both extraction methods." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "text_corpus = \"\"\"\n", - "QuantumDynamics is a leading AI research lab founded by Dr. Elena Rostova in 2018. \n", - "The lab is headquartered in Zurich, Switzerland, and focuses on quantum computing algorithms. \n", - "Dr. Rostova serves as the Chief Scientist. \n", - "The lab has released products like the Q-1 Processor and the NeuralBridge SDK. \n", - "QuantumDynamics collaborates with major universities such as MIT and ETH Zurich.\n", - "\"\"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Approach 1: The Classical NLP Pipeline\n", - "\n", - "This approach builds the ontology from the bottom up:\n", - "1. **Extract Entities**: Identify nouns/proper nouns (e.g., \"QuantumDynamics\", \"Zurich\").\n", - "2. **Extract Relations**: Identify verbs connecting them (e.g., \"headquartered in\").\n", - "3. **Generate Ontology**: Map these triplets to Classes and Properties.\n", - "\n", - "**Pros**: Deterministic, traceable, works offline.\n", - "**Cons**: Dependent on the underlying NLP model's vocabulary and flexibility." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.ontology import OntologyGenerator, OntologyOptimizer\n", - "\n", - "# 1. Initialize Extractors\n", - "ner = NERExtractor()\n", - "re = RelationExtractor()\n", - "\n", - "# 2. Extract Entities\n", - "print(\"Extracting entities...\")\n", - "entities = ner.extract(text_corpus)\n", - "print(f\"Found {len(entities)} entities: {[e['text'] for e in entities]}\")\n", - "\n", - "# 3. Extract Relationships\n", - "print(\"Extracting relationships...\")\n", - "relationships = re.extract(text_corpus, entities)\n", - "for r in relationships:\n", - " print(f\" - {r['source']} -> {r['type']} -> {r['target']}\")\n", - "\n", - "# 4. Generate Structure\n", - "generator = OntologyGenerator()\n", - "nlp_ontology = generator.generate(entities, relationships, name=\"QuantumOntologyNLP\")\n", - "\n", - "# 5. Optimize (Clean up)\n", - "optimizer = OntologyOptimizer()\n", - "nlp_ontology = optimizer.optimize_ontology(nlp_ontology, remove_redundancy=True)\n", - "\n", - "print(f\"\\nGenerated NLP Ontology with {len(nlp_ontology['classes'])} classes and {len(nlp_ontology['properties'])} properties.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Approach 2: The Generative AI Pipeline (LLM)\n", - "\n", - "This approach uses a Large Language Model to \"read\" the text and directly propose a schema.\n", - "\n", - "**Pros**: Context-aware, can handle ambiguity, generates human-like class names.\n", - "**Cons**: Non-deterministic, requires API access.\n", - "\n", - "*Note: This step requires a configured LLM provider (e.g., OpenAI).* " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import LLMOntologyGenerator\n", - "\n", - "try:\n", - " # Initialize LLM Generator (ensure OPENAI_API_KEY is set in env)\n", - " llm_gen = LLMOntologyGenerator(provider=\"openai\", model=\"gpt-4\")\n", - " \n", - " print(\"Generating ontology with LLM...\")\n", - " llm_ontology = llm_gen.generate_ontology_from_text(\n", - " text=text_corpus,\n", - " name=\"QuantumOntologyLLM\"\n", - " )\n", - " \n", - " print(f\"Generated LLM Ontology with {len(llm_ontology['classes'])} classes and {len(llm_ontology['properties'])} properties.\")\n", - " print(\"Classes detected:\", [c['name'] for c in llm_ontology['classes']])\n", - " \n", - "except Exception as e:\n", - " print(f\"Skipping LLM generation: {e}\")\n", - " llm_ontology = None" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Comparing Results with Visualization\n", - "\n", - "Let's visualize both ontologies side-by-side (if available) to see the difference in structure. The NLP model tends to be more literal, while the LLM model tends to be more conceptual." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.visualization import OntologyVisualizer\n", - "\n", - "visualizer = OntologyVisualizer()\n", - "\n", - "print(\"--- NLP Approach Visualization ---\")\n", - "fig_nlp = visualizer.visualize_structure(nlp_ontology, output=\"interactive\")\n", - "if fig_nlp: fig_nlp.show()\n", - "\n", - "if llm_ontology:\n", - " print(\"--- LLM Approach Visualization ---\")\n", - " fig_llm = visualizer.visualize_structure(llm_ontology, output=\"interactive\")\n", - " if fig_llm: fig_llm.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Validation\n", - "\n", - "No matter the method, validation is crucial. We check for structural integrity and logical consistency." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import OntologyValidator\n", - "\n", - "validator = OntologyValidator()\n", - "\n", - "def print_report(name, ont):\n", - " if not ont: return\n", - " res = validator.validate_ontology(ont)\n", - " print(f\"[{name}] Valid: {res.valid}, Errors: {len(res.errors)}\")\n", - " if res.metrics:\n", - " print(f\" Depth: {res.metrics.get('hierarchy_depth')}, Concepts: {res.metrics.get('class_count')}\")\n", - "\n", - "print_report(\"Classical NLP\", nlp_ontology)\n", - "print_report(\"Generative AI\", llm_ontology)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Export to OWL\n", - "\n", - "Finally, we choose the best model (or merge them using `ReuseManager`, covered in other guides) and export it." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.export import OWLExporter\n", - "\n", - "exporter = OWLExporter()\n", - "\n", - "# Export the NLP ontology by default, or the LLM one if preferred\n", - "target_ontology = llm_ontology if llm_ontology else nlp_ontology\n", - "\n", - "output_file = \"quantum_ontology.ttl\"\n", - "exporter.export(target_ontology, output_file, format=\"turtle\")\n", - "print(f\"Successfully exported ontology to {output_file}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "You have learned to:\n", - "1. **Extract Ontologies Programmatically**: Using `NERExtractor` for reliable, data-driven modeling.\n", - "2. **Generate Ontologies with AI**: Using `LLMOntologyGenerator` for conceptual, high-level modeling.\n", - "3. **Visualize and Compare**: Using `OntologyVisualizer` to inspect the structural differences.\n", - "4. **Validate and Export**: Ensuring quality before saving to OWL standards." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)\n", + "\n", + "# Advanced: Unstructured Text to Ontology\n", + "\n", + "Welcome to the advanced guide on extracting structured ontologies from unstructured text. This notebook explores two powerful paradigms available in Semantica:\n", + "\n", + "1. **Classical NLP Pipeline**: Using Named Entity Recognition (NER) and Relation Extraction.\n", + "2. **Generative AI Pipeline**: Using Large Language Models (LLMs) for direct conceptual modeling.\n", + "\n", + "We will compare both approaches, visualize the results, and validate the generated ontologies.\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/ontology/)\n", + "\n", + "## Setup and Installation\n", + "\n", + "Ensure you have Semantica installed with all dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install semantica[all]\n", + "\n", + "from semantica.utils.logging import get_logger\n", + "\n", + "logger = get_logger(\"unstructured_guide\")\n", + "print(\"Environment setup complete.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## The Input Text\n", + "\n", + "We will use a rich paragraph of text describing a technology company to test both extraction methods." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "text_corpus = \"\"\"\n", + "QuantumDynamics is a leading AI research lab founded by Dr. Elena Rostova in 2018. \n", + "The lab is headquartered in Zurich, Switzerland, and focuses on quantum computing algorithms. \n", + "Dr. Rostova serves as the Chief Scientist. \n", + "The lab has released products like the Q-1 Processor and the NeuralBridge SDK. \n", + "QuantumDynamics collaborates with major universities such as MIT and ETH Zurich.\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Approach 1: The Classical NLP Pipeline\n", + "\n", + "This approach builds the ontology from the bottom up:\n", + "1. **Extract Entities**: Identify nouns/proper nouns (e.g., \"QuantumDynamics\", \"Zurich\").\n", + "2. **Extract Relations**: Identify verbs connecting them (e.g., \"headquartered in\").\n", + "3. **Generate Ontology**: Map these triplets to Classes and Properties.\n", + "\n", + "**Pros**: Deterministic, traceable, works offline.\n", + "**Cons**: Dependent on the underlying NLP model's vocabulary and flexibility." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", + "from semantica.ontology import OntologyGenerator, OntologyOptimizer\n", + "\n", + "# 1. Initialize Extractors\n", + "ner = NERExtractor()\n", + "re = RelationExtractor()\n", + "\n", + "# 2. Extract Entities\n", + "print(\"Extracting entities...\")\n", + "entities = ner.extract(text_corpus)\n", + "print(f\"Found {len(entities)} entities: {[e['text'] for e in entities]}\")\n", + "\n", + "# 3. Extract Relationships\n", + "print(\"Extracting relationships...\")\n", + "relationships = re.extract(text_corpus, entities)\n", + "for r in relationships:\n", + " print(f\" - {r['source']} -> {r['type']} -> {r['target']}\")\n", + "\n", + "# 4. Generate Structure\n", + "generator = OntologyGenerator()\n", + "nlp_ontology = generator.generate(entities, relationships, name=\"QuantumOntologyNLP\")\n", + "\n", + "# 5. Optimize (Clean up)\n", + "optimizer = OntologyOptimizer()\n", + "nlp_ontology = optimizer.optimize_ontology(nlp_ontology, remove_redundancy=True)\n", + "\n", + "print(f\"\\nGenerated NLP Ontology with {len(nlp_ontology['classes'])} classes and {len(nlp_ontology['properties'])} properties.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Approach 2: The Generative AI Pipeline (LLM)\n", + "\n", + "This approach uses a Large Language Model to \"read\" the text and directly propose a schema.\n", + "\n", + "**Pros**: Context-aware, can handle ambiguity, generates human-like class names.\n", + "**Cons**: Non-deterministic, requires API access.\n", + "\n", + "*Note: This step requires a configured LLM provider (e.g., OpenAI).* " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import LLMOntologyGenerator\n", + "\n", + "try:\n", + " # Initialize LLM Generator (ensure OPENAI_API_KEY is set in env)\n", + " llm_gen = LLMOntologyGenerator(provider=\"openai\", model=\"gpt-4\")\n", + " \n", + " print(\"Generating ontology with LLM...\")\n", + " llm_ontology = llm_gen.generate_ontology_from_text(\n", + " text=text_corpus,\n", + " name=\"QuantumOntologyLLM\"\n", + " )\n", + " \n", + " print(f\"Generated LLM Ontology with {len(llm_ontology['classes'])} classes and {len(llm_ontology['properties'])} properties.\")\n", + " print(\"Classes detected:\", [c['name'] for c in llm_ontology['classes']])\n", + " \n", + "except Exception as e:\n", + " print(f\"Skipping LLM generation: {e}\")\n", + " llm_ontology = None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Comparing Results with Visualization\n", + "\n", + "Let's visualize both ontologies side-by-side (if available) to see the difference in structure. The NLP model tends to be more literal, while the LLM model tends to be more conceptual." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import OntologyVisualizer\n", + "\n", + "visualizer = OntologyVisualizer()\n", + "\n", + "print(\"--- NLP Approach Visualization ---\")\n", + "fig_nlp = visualizer.visualize_structure(nlp_ontology, output=\"interactive\")\n", + "if fig_nlp: fig_nlp.show()\n", + "\n", + "if llm_ontology:\n", + " print(\"--- LLM Approach Visualization ---\")\n", + " fig_llm = visualizer.visualize_structure(llm_ontology, output=\"interactive\")\n", + " if fig_llm: fig_llm.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Validation\n", + "\n", + "No matter the method, validation is crucial. We check for structural integrity and logical consistency." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import OntologyValidator\n", + "\n", + "validator = OntologyValidator()\n", + "\n", + "def print_report(name, ont):\n", + " if not ont: return\n", + " res = validator.validate_ontology(ont)\n", + " print(f\"[{name}] Valid: {res.valid}, Errors: {len(res.errors)}\")\n", + " if res.metrics:\n", + " print(f\" Depth: {res.metrics.get('hierarchy_depth')}, Concepts: {res.metrics.get('class_count')}\")\n", + "\n", + "print_report(\"Classical NLP\", nlp_ontology)\n", + "print_report(\"Generative AI\", llm_ontology)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Export to OWL\n", + "\n", + "Finally, we choose the best model (or merge them using `ReuseManager`, covered in other guides) and export it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import OWLExporter\n", + "\n", + "exporter = OWLExporter()\n", + "\n", + "# Export the NLP ontology by default, or the LLM one if preferred\n", + "target_ontology = llm_ontology if llm_ontology else nlp_ontology\n", + "\n", + "output_file = \"quantum_ontology.ttl\"\n", + "exporter.export(target_ontology, output_file, format=\"turtle\")\n", + "print(f\"Successfully exported ontology to {output_file}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You have learned to:\n", + "1. **Extract Ontologies Programmatically**: Using `NERExtractor` for reliable, data-driven modeling.\n", + "2. **Generate Ontologies with AI**: Using `LLMOntologyGenerator` for conceptual, high-level modeling.\n", + "3. **Visualize and Compare**: Using `OntologyVisualizer` to inspect the structural differences.\n", + "4. **Validate and Export**: Ensuring quality before saving to OWL standards." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb b/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb index b7bbf2c6..10822733 100644 --- a/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb +++ b/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb @@ -1,371 +1,380 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)\n", - "\n", - "# Advanced Vector Store - Made Easy\n", - "\n", - "## What You'll Learn\n", - "\n", - "This notebook shows you **practical ways** to use vector stores in real applications. Each example is simple and ready to use.\n", - "\n", - "### Topics\n", - "\n", - "1. **Choosing the Right Index** - Which one to use and when\n", - "2. **Smart Filtering** - Find exactly what you need\n", - "3. **Combining Results** - Merge searches from different sources\n", - "4. **Organizing Data** - Keep different users' data separate\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Part 0: Setup Embeddings\n", - "\n", - "First, let's select our embedding provider and model. Semantica supports multiple providers like Sentence Transformers and FastEmbed.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import TextEmbedder\n", - "\n", - "# Choose provider and model\n", - "embedder = TextEmbedder(method=\"fastembed\", model_name=\"BAAI/bge-small-en-v1.5\")\n", - "dimension = embedder.get_embedding_dimension()\n", - "\n", - "print(f\"Selected model: {embedder.get_model_info()['model_name']}\")\n", - "print(f\"Embedding dimension: {dimension}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Part 1: Choosing the Right Index\n", - "\n", - "Think of an index like choosing a filing system:\n", - "- **Flat**: Like a small notebook - slow but perfect\n", - "- **HNSW**: Like a well-organized library - fast and accurate\n", - "- **IVF**: Like a warehouse with sections - very fast for huge collections\n", - "\n", - "### Simple Rule\n", - "- Less than 10,000 items? Use **Flat**\n", - "- Between 10,000 and 1 million? Use **HNSW** ✅ (recommended)\n", - "- More than 1 million? Use **IVF**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import FAISSAdapter\n", - "import numpy as np\n", - "\n", - "# Create some example vectors (like document embeddings)\n", - "vectors = np.random.rand(5000, 768).astype('float32')\n", - "query = np.random.rand(768).astype('float32')\n", - "\n", - "adapter = FAISSAdapter(dimension=768)\n", - "\n", - "# HNSW Index - Best for most cases\n", - "index = adapter.create_index(index_type=\"hnsw\", metric=\"L2\", m=16)\n", - "adapter.add_vectors(index, vectors, ids=[f\"doc_{i}\" for i in range(len(vectors))])\n", - "\n", - "# Search for similar vectors\n", - "distances, indices = adapter.search(index, query, k=5)\n", - "\n", - "print(\"Found 5 most similar documents:\")\n", - "for i, (dist, idx) in enumerate(zip(distances, indices), 1):\n", - " print(f\" {i}. Document {idx} (distance: {dist:.3f})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Part 2: Smart Filtering with Metadata\n", - "\n", - "Imagine searching for \"similar articles\" but only from 2024 and only in the \"Technology\" category. That's what metadata filtering does!\n", - "\n", - "### Real-World Example\n", - "You're building a document search where users want:\n", - "- Similar documents (vector search)\n", - "- From specific categories (metadata filter)\n", - "- From recent years (metadata filter)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import HybridSearch, MetadataFilter\n", - "import numpy as np\n", - "\n", - "# Create sample documents with metadata\n", - "documents = [\n", - " {\"id\": 0, \"text\": \"AI in Healthcare\", \"category\": \"Technology\", \"year\": 2024},\n", - " {\"id\": 1, \"text\": \"Machine Learning Basics\", \"category\": \"Technology\", \"year\": 2023},\n", - " {\"id\": 2, \"text\": \"Business Strategy\", \"category\": \"Business\", \"year\": 2024},\n", - " {\"id\": 3, \"text\": \"Data Science Guide\", \"category\": \"Technology\", \"year\": 2024},\n", - " {\"id\": 4, \"text\": \"Marketing Tips\", \"category\": \"Business\", \"year\": 2023},\n", - "]\n", - "\n", - "# Create vectors for each document\n", - "vectors = [np.random.rand(768) for _ in documents]\n", - "metadata = [{\"category\": d[\"category\"], \"year\": d[\"year\"]} for d in documents]\n", - "vector_ids = [f\"doc_{d['id']}\" for d in documents]\n", - "\n", - "# Create search\n", - "search = HybridSearch()\n", - "query = np.random.rand(768)\n", - "\n", - "# Example 1: Find Technology articles from 2024\n", - "filter1 = MetadataFilter().eq(\"category\", \"Technology\").eq(\"year\", 2024)\n", - "results = search.search(query, vectors, metadata, vector_ids, filter=filter1, k=10)\n", - "\n", - "print(\"Technology articles from 2024:\")\n", - "for r in results:\n", - " doc_id = int(r['id'].split('_')[1])\n", - " print(f\" - {documents[doc_id]['text']}\")\n", - "\n", - "# Example 2: Find any article from 2024\n", - "filter2 = MetadataFilter().eq(\"year\", 2024)\n", - "results2 = search.search(query, vectors, metadata, vector_ids, filter=filter2, k=10)\n", - "\n", - "print(\"\\nAll articles from 2024:\")\n", - "for r in results2:\n", - " doc_id = int(r['id'].split('_')[1])\n", - " print(f\" - {documents[doc_id]['text']} ({documents[doc_id]['category']})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Part 3: Combining Search Results\n", - "\n", - "Sometimes you want to search in multiple places and combine the results. Like searching both your email and documents, then showing the best matches from both.\n", - "\n", - "### When to Use This\n", - "- Searching multiple databases\n", - "- Combining different search strategies\n", - "- Giving more weight to certain sources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import SearchRanker\n", - "\n", - "# Simulate two different searches\n", - "# Search 1: Recent documents\n", - "recent_results = [\n", - " {\"id\": \"doc_3\", \"score\": 0.95, \"source\": \"recent\"},\n", - " {\"id\": \"doc_0\", \"score\": 0.90, \"source\": \"recent\"},\n", - " {\"id\": \"doc_2\", \"score\": 0.85, \"source\": \"recent\"},\n", - "]\n", - "\n", - "# Search 2: Popular documents\n", - "popular_results = [\n", - " {\"id\": \"doc_1\", \"score\": 0.92, \"source\": \"popular\"},\n", - " {\"id\": \"doc_3\", \"score\": 0.88, \"source\": \"popular\"},\n", - " {\"id\": \"doc_4\", \"score\": 0.80, \"source\": \"popular\"},\n", - "]\n", - "\n", - "# Method 1: Fair combination (RRF)\n", - "ranker = SearchRanker(strategy=\"reciprocal_rank_fusion\")\n", - "combined = ranker.rank([recent_results, popular_results])\n", - "\n", - "print(\"Combined results (fair ranking):\")\n", - "for i, result in enumerate(combined[:3], 1):\n", - " doc_id = int(result['id'].split('_')[1])\n", - " print(f\" {i}. {documents[doc_id]['text']} (score: {result['score']:.3f})\")\n", - "\n", - "# Method 2: Prefer recent documents (70% recent, 30% popular)\n", - "weighted_ranker = SearchRanker(strategy=\"weighted_average\")\n", - "weighted_combined = weighted_ranker.rank(\n", - " [recent_results, popular_results],\n", - " weights=[0.7, 0.3]\n", - ")\n", - "\n", - "print(\"\\nCombined results (prefer recent):\")\n", - "for i, result in enumerate(weighted_combined[:3], 1):\n", - " doc_id = int(result['id'].split('_')[1])\n", - " print(f\" {i}. {documents[doc_id]['text']} (score: {result['score']:.3f})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Part 4: Keeping User Data Separate\n", - "\n", - "If you're building an app with multiple users or companies, you need to keep their data separate. Namespaces do this automatically.\n", - "\n", - "### Real Example\n", - "You're building a SaaS app where:\n", - "- Company A has their documents\n", - "- Company B has their documents\n", - "- They should never see each other's data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import NamespaceManager\n", - "\n", - "# Create manager\n", - "manager = NamespaceManager()\n", - "\n", - "# Create separate spaces for each company\n", - "company_a = manager.create_namespace(\"company_a\", \"Company A's documents\")\n", - "company_b = manager.create_namespace(\"company_b\", \"Company B's documents\")\n", - "\n", - "# Add documents to Company A\n", - "for i in range(10):\n", - " manager.add_vector_to_namespace(f\"company_a_doc_{i}\", \"company_a\")\n", - "\n", - "# Add documents to Company B\n", - "for i in range(15):\n", - " manager.add_vector_to_namespace(f\"company_b_doc_{i}\", \"company_b\")\n", - "\n", - "# Get each company's documents\n", - "a_docs = manager.get_namespace_vectors(\"company_a\")\n", - "b_docs = manager.get_namespace_vectors(\"company_b\")\n", - "\n", - "print(f\"Company A has {len(a_docs)} documents\")\n", - "print(f\"Company B has {len(b_docs)} documents\")\n", - "\n", - "# Set permissions (who can access what)\n", - "company_a.set_access_control(\"admin@companya.com\", [\"read\", \"write\", \"delete\"])\n", - "company_a.set_access_control(\"user@companya.com\", [\"read\"]) # Read-only\n", - "\n", - "# Check permissions\n", - "print(f\"\\nAdmin can delete: {company_a.has_permission('admin@companya.com', 'delete')}\")\n", - "print(f\"User can delete: {company_a.has_permission('user@companya.com', 'delete')}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Quick Reference Guide\n", - "\n", - "### Which Index Should I Use?\n", - "\n", - "```python\n", - "# Small dataset (< 10,000 items)\n", - "index = adapter.create_index(index_type=\"flat\", metric=\"L2\")\n", - "\n", - "# Medium dataset (10,000 - 1,000,000 items) ✅ RECOMMENDED\n", - "index = adapter.create_index(index_type=\"hnsw\", metric=\"L2\", m=16)\n", - "\n", - "# Large dataset (> 1,000,000 items)\n", - "index = adapter.create_index(index_type=\"ivf\", metric=\"L2\", nlist=100)\n", - "```\n", - "\n", - "### How Do I Filter Results?\n", - "\n", - "```python\n", - "# Single condition\n", - "filter = MetadataFilter().eq(\"category\", \"Technology\")\n", - "\n", - "# Multiple conditions (AND)\n", - "filter = MetadataFilter() \\\n", - " .eq(\"category\", \"Technology\") \\\n", - " .eq(\"year\", 2024)\n", - "\n", - "# Greater than / Less than\n", - "filter = MetadataFilter().gt(\"year\", 2020)\n", - "```\n", - "\n", - "### How Do I Combine Results?\n", - "\n", - "```python\n", - "# Fair combination\n", - "ranker = SearchRanker(strategy=\"reciprocal_rank_fusion\")\n", - "combined = ranker.rank([results1, results2])\n", - "\n", - "# Weighted combination (prefer first source)\n", - "ranker = SearchRanker(strategy=\"weighted_average\")\n", - "combined = ranker.rank([results1, results2], weights=[0.7, 0.3])\n", - "```\n", - "\n", - "### How Do I Separate User Data?\n", - "\n", - "```python\n", - "# Create namespace for each user/company\n", - "manager = NamespaceManager()\n", - "user_space = manager.create_namespace(\"user_123\", \"User 123's data\")\n", - "\n", - "# Add data to namespace\n", - "manager.add_vector_to_namespace(\"doc_1\", \"user_123\")\n", - "\n", - "# Get user's data\n", - "user_docs = manager.get_namespace_vectors(\"user_123\")\n", - "```\n", - "\n", - "---\n", - "\n", - "## Summary\n", - "\n", - "You've learned:\n", - "\n", - "1. ✅ **Index Selection**: Use HNSW for most cases\n", - "2. ✅ **Smart Filtering**: Combine vector search with metadata\n", - "3. ✅ **Result Fusion**: Merge searches from different sources\n", - "4. ✅ **Data Isolation**: Keep users' data separate\n", - "\n", - "### Next Steps\n", - "\n", - "- Try these examples with your own data\n", - "- Experiment with different filters\n", - "- Build a multi-user application\n", - "- Explore the [introduction notebook](../introduction/13_Vector_Store.ipynb) for more basics\n", - "\n", - "**Need Help?** Check our [documentation](https://semantica.readthedocs.io) or ask on [GitHub](https://github.com/Hawksight-AI/semantica)." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)\n", + "\n", + "# Advanced Vector Store - Made Easy\n", + "\n", + "## What You'll Learn\n", + "\n", + "This notebook shows you **practical ways** to use vector stores in real applications. Each example is simple and ready to use.\n", + "\n", + "### Topics\n", + "\n", + "1. **Choosing the Right Index** - Which one to use and when\n", + "2. **Smart Filtering** - Find exactly what you need\n", + "3. **Combining Results** - Merge searches from different sources\n", + "4. **Organizing Data** - Keep different users' data separate\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 0: Setup Embeddings\n", + "\n", + "First, let's select our embedding provider and model. Semantica supports multiple providers like Sentence Transformers and FastEmbed.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.embeddings import TextEmbedder\n", + "\n", + "# Choose provider and model\n", + "embedder = TextEmbedder(method=\"fastembed\", model_name=\"BAAI/bge-small-en-v1.5\")\n", + "dimension = embedder.get_embedding_dimension()\n", + "\n", + "print(f\"Selected model: {embedder.get_model_info()['model_name']}\")\n", + "print(f\"Embedding dimension: {dimension}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 1: Choosing the Right Index\n", + "\n", + "Think of an index like choosing a filing system:\n", + "- **Flat**: Like a small notebook - slow but perfect\n", + "- **HNSW**: Like a well-organized library - fast and accurate\n", + "- **IVF**: Like a warehouse with sections - very fast for huge collections\n", + "\n", + "### Simple Rule\n", + "- Less than 10,000 items? Use **Flat**\n", + "- Between 10,000 and 1 million? Use **HNSW** \u2705 (recommended)\n", + "- More than 1 million? Use **IVF**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import FAISSAdapter\n", + "import numpy as np\n", + "\n", + "# Create some example vectors (like document embeddings)\n", + "vectors = np.random.rand(5000, 768).astype('float32')\n", + "query = np.random.rand(768).astype('float32')\n", + "\n", + "adapter = FAISSAdapter(dimension=768)\n", + "\n", + "# HNSW Index - Best for most cases\n", + "index = adapter.create_index(index_type=\"hnsw\", metric=\"L2\", m=16)\n", + "adapter.add_vectors(index, vectors, ids=[f\"doc_{i}\" for i in range(len(vectors))])\n", + "\n", + "# Search for similar vectors\n", + "distances, indices = adapter.search(index, query, k=5)\n", + "\n", + "print(\"Found 5 most similar documents:\")\n", + "for i, (dist, idx) in enumerate(zip(distances, indices), 1):\n", + " print(f\" {i}. Document {idx} (distance: {dist:.3f})\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 2: Smart Filtering with Metadata\n", + "\n", + "Imagine searching for \"similar articles\" but only from 2024 and only in the \"Technology\" category. That's what metadata filtering does!\n", + "\n", + "### Real-World Example\n", + "You're building a document search where users want:\n", + "- Similar documents (vector search)\n", + "- From specific categories (metadata filter)\n", + "- From recent years (metadata filter)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import HybridSearch, MetadataFilter\n", + "import numpy as np\n", + "\n", + "# Create sample documents with metadata\n", + "documents = [\n", + " {\"id\": 0, \"text\": \"AI in Healthcare\", \"category\": \"Technology\", \"year\": 2024},\n", + " {\"id\": 1, \"text\": \"Machine Learning Basics\", \"category\": \"Technology\", \"year\": 2023},\n", + " {\"id\": 2, \"text\": \"Business Strategy\", \"category\": \"Business\", \"year\": 2024},\n", + " {\"id\": 3, \"text\": \"Data Science Guide\", \"category\": \"Technology\", \"year\": 2024},\n", + " {\"id\": 4, \"text\": \"Marketing Tips\", \"category\": \"Business\", \"year\": 2023},\n", + "]\n", + "\n", + "# Create vectors for each document\n", + "vectors = [np.random.rand(768) for _ in documents]\n", + "metadata = [{\"category\": d[\"category\"], \"year\": d[\"year\"]} for d in documents]\n", + "vector_ids = [f\"doc_{d['id']}\" for d in documents]\n", + "\n", + "# Create search\n", + "search = HybridSearch()\n", + "query = np.random.rand(768)\n", + "\n", + "# Example 1: Find Technology articles from 2024\n", + "filter1 = MetadataFilter().eq(\"category\", \"Technology\").eq(\"year\", 2024)\n", + "results = search.search(query, vectors, metadata, vector_ids, filter=filter1, k=10)\n", + "\n", + "print(\"Technology articles from 2024:\")\n", + "for r in results:\n", + " doc_id = int(r['id'].split('_')[1])\n", + " print(f\" - {documents[doc_id]['text']}\")\n", + "\n", + "# Example 2: Find any article from 2024\n", + "filter2 = MetadataFilter().eq(\"year\", 2024)\n", + "results2 = search.search(query, vectors, metadata, vector_ids, filter=filter2, k=10)\n", + "\n", + "print(\"\\nAll articles from 2024:\")\n", + "for r in results2:\n", + " doc_id = int(r['id'].split('_')[1])\n", + " print(f\" - {documents[doc_id]['text']} ({documents[doc_id]['category']})\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 3: Combining Search Results\n", + "\n", + "Sometimes you want to search in multiple places and combine the results. Like searching both your email and documents, then showing the best matches from both.\n", + "\n", + "### When to Use This\n", + "- Searching multiple databases\n", + "- Combining different search strategies\n", + "- Giving more weight to certain sources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import SearchRanker\n", + "\n", + "# Simulate two different searches\n", + "# Search 1: Recent documents\n", + "recent_results = [\n", + " {\"id\": \"doc_3\", \"score\": 0.95, \"source\": \"recent\"},\n", + " {\"id\": \"doc_0\", \"score\": 0.90, \"source\": \"recent\"},\n", + " {\"id\": \"doc_2\", \"score\": 0.85, \"source\": \"recent\"},\n", + "]\n", + "\n", + "# Search 2: Popular documents\n", + "popular_results = [\n", + " {\"id\": \"doc_1\", \"score\": 0.92, \"source\": \"popular\"},\n", + " {\"id\": \"doc_3\", \"score\": 0.88, \"source\": \"popular\"},\n", + " {\"id\": \"doc_4\", \"score\": 0.80, \"source\": \"popular\"},\n", + "]\n", + "\n", + "# Method 1: Fair combination (RRF)\n", + "ranker = SearchRanker(strategy=\"reciprocal_rank_fusion\")\n", + "combined = ranker.rank([recent_results, popular_results])\n", + "\n", + "print(\"Combined results (fair ranking):\")\n", + "for i, result in enumerate(combined[:3], 1):\n", + " doc_id = int(result['id'].split('_')[1])\n", + " print(f\" {i}. {documents[doc_id]['text']} (score: {result['score']:.3f})\")\n", + "\n", + "# Method 2: Prefer recent documents (70% recent, 30% popular)\n", + "weighted_ranker = SearchRanker(strategy=\"weighted_average\")\n", + "weighted_combined = weighted_ranker.rank(\n", + " [recent_results, popular_results],\n", + " weights=[0.7, 0.3]\n", + ")\n", + "\n", + "print(\"\\nCombined results (prefer recent):\")\n", + "for i, result in enumerate(weighted_combined[:3], 1):\n", + " doc_id = int(result['id'].split('_')[1])\n", + " print(f\" {i}. {documents[doc_id]['text']} (score: {result['score']:.3f})\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 4: Keeping User Data Separate\n", + "\n", + "If you're building an app with multiple users or companies, you need to keep their data separate. Namespaces do this automatically.\n", + "\n", + "### Real Example\n", + "You're building a SaaS app where:\n", + "- Company A has their documents\n", + "- Company B has their documents\n", + "- They should never see each other's data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import NamespaceManager\n", + "\n", + "# Create manager\n", + "manager = NamespaceManager()\n", + "\n", + "# Create separate spaces for each company\n", + "company_a = manager.create_namespace(\"company_a\", \"Company A's documents\")\n", + "company_b = manager.create_namespace(\"company_b\", \"Company B's documents\")\n", + "\n", + "# Add documents to Company A\n", + "for i in range(10):\n", + " manager.add_vector_to_namespace(f\"company_a_doc_{i}\", \"company_a\")\n", + "\n", + "# Add documents to Company B\n", + "for i in range(15):\n", + " manager.add_vector_to_namespace(f\"company_b_doc_{i}\", \"company_b\")\n", + "\n", + "# Get each company's documents\n", + "a_docs = manager.get_namespace_vectors(\"company_a\")\n", + "b_docs = manager.get_namespace_vectors(\"company_b\")\n", + "\n", + "print(f\"Company A has {len(a_docs)} documents\")\n", + "print(f\"Company B has {len(b_docs)} documents\")\n", + "\n", + "# Set permissions (who can access what)\n", + "company_a.set_access_control(\"admin@companya.com\", [\"read\", \"write\", \"delete\"])\n", + "company_a.set_access_control(\"user@companya.com\", [\"read\"]) # Read-only\n", + "\n", + "# Check permissions\n", + "print(f\"\\nAdmin can delete: {company_a.has_permission('admin@companya.com', 'delete')}\")\n", + "print(f\"User can delete: {company_a.has_permission('user@companya.com', 'delete')}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quick Reference Guide\n", + "\n", + "### Which Index Should I Use?\n", + "\n", + "```python\n", + "# Small dataset (< 10,000 items)\n", + "index = adapter.create_index(index_type=\"flat\", metric=\"L2\")\n", + "\n", + "# Medium dataset (10,000 - 1,000,000 items) \u2705 RECOMMENDED\n", + "index = adapter.create_index(index_type=\"hnsw\", metric=\"L2\", m=16)\n", + "\n", + "# Large dataset (> 1,000,000 items)\n", + "index = adapter.create_index(index_type=\"ivf\", metric=\"L2\", nlist=100)\n", + "```\n", + "\n", + "### How Do I Filter Results?\n", + "\n", + "```python\n", + "# Single condition\n", + "filter = MetadataFilter().eq(\"category\", \"Technology\")\n", + "\n", + "# Multiple conditions (AND)\n", + "filter = MetadataFilter() \\\n", + " .eq(\"category\", \"Technology\") \\\n", + " .eq(\"year\", 2024)\n", + "\n", + "# Greater than / Less than\n", + "filter = MetadataFilter().gt(\"year\", 2020)\n", + "```\n", + "\n", + "### How Do I Combine Results?\n", + "\n", + "```python\n", + "# Fair combination\n", + "ranker = SearchRanker(strategy=\"reciprocal_rank_fusion\")\n", + "combined = ranker.rank([results1, results2])\n", + "\n", + "# Weighted combination (prefer first source)\n", + "ranker = SearchRanker(strategy=\"weighted_average\")\n", + "combined = ranker.rank([results1, results2], weights=[0.7, 0.3])\n", + "```\n", + "\n", + "### How Do I Separate User Data?\n", + "\n", + "```python\n", + "# Create namespace for each user/company\n", + "manager = NamespaceManager()\n", + "user_space = manager.create_namespace(\"user_123\", \"User 123's data\")\n", + "\n", + "# Add data to namespace\n", + "manager.add_vector_to_namespace(\"doc_1\", \"user_123\")\n", + "\n", + "# Get user's data\n", + "user_docs = manager.get_namespace_vectors(\"user_123\")\n", + "```\n", + "\n", + "---\n", + "\n", + "## Summary\n", + "\n", + "You've learned:\n", + "\n", + "1. \u2705 **Index Selection**: Use HNSW for most cases\n", + "2. \u2705 **Smart Filtering**: Combine vector search with metadata\n", + "3. \u2705 **Result Fusion**: Merge searches from different sources\n", + "4. \u2705 **Data Isolation**: Keep users' data separate\n", + "\n", + "### Next Steps\n", + "\n", + "- Try these examples with your own data\n", + "- Experiment with different filters\n", + "- Build a multi-user application\n", + "- Explore the [introduction notebook](../introduction/13_Vector_Store.ipynb) for more basics\n", + "\n", + "**Need Help?** Check our [documentation](https://semantica.readthedocs.io) or ask on [GitHub](https://github.com/Hawksight-AI/semantica)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/cookbook/introduction/01_Welcome_to_Semantica.ipynb b/cookbook/introduction/01_Welcome_to_Semantica.ipynb index 0dc746bd..b17b3125 100644 --- a/cookbook/introduction/01_Welcome_to_Semantica.ipynb +++ b/cookbook/introduction/01_Welcome_to_Semantica.ipynb @@ -21,6 +21,15 @@ "**You should use this notebook to understand the big picture, not to learn every API in depth.**" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -86,73 +95,73 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 🧩 Semantica Modules Reference\n", + "## \ud83e\udde9 Semantica Modules Reference\n", "\n", "Semantica is modular by design. Here is a comprehensive guide to all available modules, grouped by functionality.\n", "\n", - "### 📥 Ingestion & Parsing\n", + "### \ud83d\udce5 Ingestion & Parsing\n", "Modules that handle raw data input and structure.\n", "\n", "| Module | Description | Key Capabilities |\n", "| :--- | :--- | :--- |\n", - "| **`ingest`** | **Data Ingestion**
Connects to data sources. | • File, Web, Feed, Stream ingestion
• DB, Email, Repo, MCP support |\n", - "| **`parse`** | **Document Parsing**
Parses raw content into structures. | • PDF, HTML, JSON, CSV, Excel
• Image & Code parsing |\n", + "| **`ingest`** | **Data Ingestion**
Connects to data sources. | \u2022 File, Web, Feed, Stream ingestion
\u2022 DB, Email, Repo, MCP support |\n", + "| **`parse`** | **Document Parsing**
Parses raw content into structures. | \u2022 PDF, HTML, JSON, CSV, Excel
\u2022 Image & Code parsing |\n", "\n", - "### ⚙️ Data Processing\n", + "### \u2699\ufe0f Data Processing\n", "Modules that clean, normalize, and split data.\n", "\n", "| Module | Description | Key Capabilities |\n", "| :--- | :--- | :--- |\n", - "| **`normalize`** | **Data Normalization**
Cleans and standardizes text. | • Text cleaning & Language detection
• Entity, Date, Number normalization |\n", - "| **`split`** | **Chunking**
Splits documents for RAG. | • Recursive character splitting
• Semantic & Token-based splitting |\n", + "| **`normalize`** | **Data Normalization**
Cleans and standardizes text. | \u2022 Text cleaning & Language detection
\u2022 Entity, Date, Number normalization |\n", + "| **`split`** | **Chunking**
Splits documents for RAG. | \u2022 Recursive character splitting
\u2022 Semantic & Token-based splitting |\n", "\n", - "### 🧠 Extraction & Enrichment\n", + "### \ud83e\udde0 Extraction & Enrichment\n", "Modules that extract meaning, structure, and vectors from raw data.\n", "\n", "| Module | Description | Key Capabilities |\n", "| :--- | :--- | :--- |\n", - "| **`semantic_extract`** | **Information Extraction**
Extracts entities and relations. | • NER & Relation Extraction
• Event & Semantic Network detection |\n", - "| **`context`** | **Agent Memory**
Manages state for AI agents. | • Long-term memory & history
• Context graph & RAG integration |\n", + "| **`semantic_extract`** | **Information Extraction**
Extracts entities and relations. | \u2022 NER & Relation Extraction
\u2022 Event & Semantic Network detection |\n", + "| **`context`** | **Agent Memory**
Manages state for AI agents. | \u2022 Long-term memory & history
\u2022 Context graph & RAG integration |\n", "\n", - "### 🕸️ Knowledge Graph Core\n", + "### \ud83d\udd78\ufe0f Knowledge Graph Core\n", "Modules for building, refining, and resolving knowledge graphs.\n", "\n", "| Module | Description | Key Capabilities |\n", "| :--- | :--- | :--- |\n", - "| **`kg`** | **Graph Construction**
Builds and analyzes graphs. | • Graph Building & Analysis
• Validation & Entity Resolution |\n", - "| **`conflicts`** | **Conflict Resolution**
Resolves data contradictions. | • Source reliability scoring
• Truth discovery algorithms |\n", - "| **`deduplication`** | **Entity Resolution**
Merges duplicate entities. | • Similarity-based blocking
• Clustering & Canonicalization |\n", + "| **`kg`** | **Graph Construction**
Builds and analyzes graphs. | \u2022 Graph Building & Analysis
\u2022 Validation & Entity Resolution |\n", + "| **`conflicts`** | **Conflict Resolution**
Resolves data contradictions. | \u2022 Source reliability scoring
\u2022 Truth discovery algorithms |\n", + "| **`deduplication`** | **Entity Resolution**
Merges duplicate entities. | \u2022 Similarity-based blocking
\u2022 Clustering & Canonicalization |\n", "\n", - "### 💾 Storage & Retrieval\n", + "### \ud83d\udcbe Storage & Retrieval\n", "Modules for persisting and querying data.\n", "\n", "| Module | Description | Key Capabilities |\n", "| :--- | :--- | :--- |\n", - "| **`embeddings`** | **Vector Embeddings**
Generates semantic vectors. | • Text & Graph embeddings
• Multi-provider support (OpenAI, etc.) |\n", - "| **`vector_store`** | **Vector Database**
Stores and searches vectors. | • Similarity search & Filtering
• Hybrid search (Vector + Keyword) |\n", - "| **`graph_store`** | **Property Graph Store**
Persists graph data. | • Neo4j, FalkorDB adapters
• Cypher query support |\n", - "| **`triplet_store`** | **RDF Store**
Persists semantic triples. | • SPARQL endpoints
• BlazeGraph, Jena, Virtuoso adapters |\n", + "| **`embeddings`** | **Vector Embeddings**
Generates semantic vectors. | \u2022 Text & Graph embeddings
\u2022 Multi-provider support (OpenAI, etc.) |\n", + "| **`vector_store`** | **Vector Database**
Stores and searches vectors. | \u2022 Similarity search & Filtering
\u2022 Hybrid search (Vector + Keyword) |\n", + "| **`graph_store`** | **Property Graph Store**
Persists graph data. | \u2022 Neo4j, FalkorDB adapters
\u2022 Cypher query support |\n", + "| **`triplet_store`** | **RDF Store**
Persists semantic triples. | \u2022 SPARQL endpoints
\u2022 BlazeGraph, Jena, Virtuoso adapters |\n", "\n", - "### 🔎 Reasoning & Analysis\n", + "### \ud83d\udd0e Reasoning & Analysis\n", "Modules for deriving new knowledge and evaluating quality.\n", "\n", "| Module | Description | Key Capabilities |\n", "| :--- | :--- | :--- |\n", - "| **`reasoning`** | **Inference Engine**
Derives new facts via rules. | • Datalog/Rule-based inference
• Forward/Backward chaining |\n", - "| **`ontology`** | **Ontology Management**
Manages schema and definitions. | • Ontology generation from data
• Validation & Evolution |\n", - "| **`visualization`** | **Visual Analytics**
Visualizes graphs and metrics. | • 2D/3D Graph visualization
• Interactive plots & dashboards |\n", - "| **`evals`** | **Evaluation**
Benchmarks pipeline quality. | • RAG & Graph quality metrics
• Ground truth comparison |\n", + "| **`reasoning`** | **Inference Engine**
Derives new facts via rules. | \u2022 Datalog/Rule-based inference
\u2022 Forward/Backward chaining |\n", + "| **`ontology`** | **Ontology Management**
Manages schema and definitions. | \u2022 Ontology generation from data
\u2022 Validation & Evolution |\n", + "| **`visualization`** | **Visual Analytics**
Visualizes graphs and metrics. | \u2022 2D/3D Graph visualization
\u2022 Interactive plots & dashboards |\n", + "| **`evals`** | **Evaluation**
Benchmarks pipeline quality. | \u2022 RAG & Graph quality metrics
\u2022 Ground truth comparison |\n", "\n", - "### 🛠️ Orchestration & Utils\n", + "### \ud83d\udee0\ufe0f Orchestration & Utils\n", "Modules for managing the framework and workflows.\n", "\n", "| Module | Description | Key Capabilities |\n", "| :--- | :--- | :--- |\n", - "| **`core`** | **Framework Core**
Main entry point and config. | • Lifecycle management
• Plugin system & Configuration |\n", - "| **`pipeline`** | **Workflow Orchestration**
Manages complex flows. | • DAG execution & Retries
• Error handling & Observability |\n", - "| **`seed`** | **Data Seeding**
Initializes knowledge bases. | • Taxonomy & Ontology seeding
• Reference data loading |\n", - "| **`export`** | **Data Export**
Exports data to files. | • JSON, CSV, RDF, GEXF export
• Report generation |\n", - "| **`utils`** | **Utilities**
Common helper functions. | • Logging, Async, Hashing
• Text processing helpers |" + "| **`core`** | **Framework Core**
Main entry point and config. | \u2022 Lifecycle management
\u2022 Plugin system & Configuration |\n", + "| **`pipeline`** | **Workflow Orchestration**
Manages complex flows. | \u2022 DAG execution & Retries
\u2022 Error handling & Observability |\n", + "| **`seed`** | **Data Seeding**
Initializes knowledge bases. | \u2022 Taxonomy & Ontology seeding
\u2022 Reference data loading |\n", + "| **`export`** | **Data Export**
Exports data to files. | \u2022 JSON, CSV, RDF, GEXF export
\u2022 Report generation |\n", + "| **`utils`** | **Utilities**
Common helper functions. | \u2022 Logging, Async, Hashing
\u2022 Text processing helpers |" ] }, { @@ -203,17 +212,6 @@ "```" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "semantica.__version__" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -522,4 +520,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/02_Data_Ingestion.ipynb b/cookbook/introduction/02_Data_Ingestion.ipynb index fe2acbcb..2b4ad7a8 100644 --- a/cookbook/introduction/02_Data_Ingestion.ipynb +++ b/cookbook/introduction/02_Data_Ingestion.ipynb @@ -1,441 +1,439 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)\n", - "\n", - "# Data Ingestion - Comprehensive Guide\n", - "\n", - "## Overview\n", - "\n", - "This notebook provides a comprehensive guide to Semantica's data ingestion capabilities. It covers all submodules, classes, and helper functions available in the `semantica.ingest` module.\n", - "\n", - "**Documentation**: [Ingest API Reference](https://semantica.readthedocs.io/reference/ingest/)\n", - "\n", - "### Table of Contents\n", - "\n", - "1. **Unified Ingestion**: `ingest` function\n", - "2. **File Ingestion**: `FileIngestor`, `FileTypeDetector`, `CloudStorageIngestor`\n", - "3. **Web Ingestion**: `WebIngestor`, `ContentExtractor`, `SitemapCrawler`, `RobotsChecker`\n", - "4. **Feed Ingestion**: `FeedIngestor`, `FeedMonitor`\n", - "5. **Stream Ingestion**: `StreamIngestor`, `StreamMonitor`\n", - "6. **Repository Ingestion**: `RepoIngestor`, `CodeExtractor`, `GitAnalyzer`\n", - "7. **Email Ingestion**: `EmailIngestor`, `AttachmentProcessor`\n", - "8. **Database Ingestion**: `DBIngestor`, `DatabaseConnector`\n", - "9. **MCP Ingestion**: `MCPIngestor`\n", - "10. **Configuration**: `IngestConfig`\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica with all dependencies:\n", - "\n", - "```bash\n", - "pip install semantica[all]\n", - "```\n", - "\n", - "---\n", - "\n", - "## 1. Unified Ingestion\n", - "\n", - "The `ingest` function is the main entry point for quick data loading. It automatically detects the source type.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import ingest\n", - "import tempfile\n", - "import os\n", - "import json\n", - "\n", - "# Setup temporary directory for examples\n", - "temp_dir = tempfile.mkdtemp()\n", - "sample_file = os.path.join(temp_dir, \"sample.txt\")\n", - "with open(sample_file, 'w') as f:\n", - " f.write(\"Semantica Unified Ingestion Example\")\n", - "\n", - "# Auto-detect file source\n", - "result = ingest(sample_file)\n", - "print(f\"Ingested: {result.name} (Type: {result.file_type})\")\n", - "\n", - "# Explicit source type\n", - "result_explicit = ingest(sample_file, source_type=\"file\")\n", - "print(f\"Explicit Ingest: {result_explicit.name}\")\n", - "\n", - "# Ingest web URL (auto-detected)\n", - "# Note: This will fail if no internet connection\n", - "try:\n", - " result_web = ingest(\"https://example.com\")\n", - " print(f\"Ingested Web: {result_web.title}\")\n", - "except Exception as e:\n", - " print(f\"Web ingestion skipped: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. File Ingestion\n", - "\n", - "Detailed control over file processing using `FileIngestor` and helper classes.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, FileTypeDetector, CloudStorageIngestor\n", - "\n", - "# --- FileTypeDetector ---\n", - "detector = FileTypeDetector()\n", - "detected_type = detector.detect_type(sample_file)\n", - "print(f\"Detected Type: {detected_type}\")\n", - "\n", - "# --- FileIngestor ---\n", - "file_ingestor = FileIngestor()\n", - "\n", - "# Ingest Directory\n", - "subdir = os.path.join(temp_dir, \"docs\")\n", - "os.makedirs(subdir, exist_ok=True)\n", - "with open(os.path.join(subdir, \"note.md\"), 'w') as f:\n", - " f.write(\"# Note\\nThis is a markdown file.\")\n", - "\n", - "files = file_ingestor.ingest_directory(temp_dir, recursive=True)\n", - "print(f\"Ingested {len(files)} files from directory\")\n", - "\n", - "# --- CloudStorageIngestor (Mock Config) ---\n", - "s3_config = {\n", - " \"aws_access_key_id\": \"mock_key\",\n", - " \"aws_secret_access_key\": \"mock_secret\",\n", - " \"region_name\": \"us-east-1\"\n", - "}\n", - "cloud_ingestor = CloudStorageIngestor(provider=\"s3\", **s3_config)\n", - "\n", - "# Example call (will raise error without real credentials)\n", - "try:\n", - " result = cloud_ingestor.ingest(\"s3://my-bucket/data.csv\")\n", - " print(f\"Cloud Ingest: {result.name}\")\n", - "except Exception as e:\n", - " print(f\"Cloud ingestion skipped (Mock Config): {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Web Ingestion\n", - "\n", - "Scraping and crawling with `WebIngestor`, `ContentExtractor`, and `SitemapCrawler`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import WebIngestor, ContentExtractor, SitemapCrawler, RobotsChecker\n", - "\n", - "# --- ContentExtractor ---\n", - "extractor = ContentExtractor()\n", - "html_content = \"

Hello World

This is a test.

Link\"\n", - "text = extractor.extract_text(html_content)\n", - "links = extractor.extract_links(html_content, base_url=\"https://example.com\")\n", - "print(f\"Extracted Text: {text}\")\n", - "print(f\"Extracted Links: {links}\")\n", - "\n", - "# --- RobotsChecker ---\n", - "checker = RobotsChecker()\n", - "can_fetch = checker.can_fetch(\"https://www.google.com/search\", \"MyBot\")\n", - "print(f\"Can fetch google search? {can_fetch}\")\n", - "\n", - "# --- WebIngestor ---\n", - "web_ingestor = WebIngestor(delay=1.0)\n", - "try:\n", - " web_content = web_ingestor.ingest_url(\"https://example.com\")\n", - " print(f\"Web Content Title: {web_content.title}\")\n", - "except Exception as e:\n", - " print(f\"Web ingest failed: {e}\")\n", - "\n", - "# --- SitemapCrawler ---\n", - "crawler = SitemapCrawler()\n", - "try:\n", - " urls = crawler.parse_sitemap(\"https://www.google.com/sitemap.xml\")\n", - " print(f\"Found {len(urls)} URLs in sitemap\")\n", - "except Exception as e:\n", - " print(f\"Sitemap crawl failed: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Feed Ingestion\n", - "\n", - "Consuming RSS/Atom feeds with `FeedIngestor` and monitoring with `FeedMonitor`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FeedIngestor, FeedMonitor\n", - "import time\n", - "\n", - "# --- FeedIngestor ---\n", - "feed_ingestor = FeedIngestor()\n", - "try:\n", - " feed_data = feed_ingestor.ingest_feed(\"https://feeds.feedburner.com/oreilly/radar\")\n", - " print(f\"Feed Title: {feed_data.title}\")\n", - "except Exception as e:\n", - " print(f\"Feed ingest failed: {e}\")\n", - "\n", - "# --- FeedMonitor ---\n", - "def feed_callback(feed_data):\n", - " print(f\"Feed Updated: {feed_data.title} with {len(feed_data.items)} items\")\n", - "\n", - "monitor = FeedMonitor(check_interval=5)\n", - "try:\n", - " monitor.monitor(\"https://feeds.feedburner.com/oreilly/radar\", callback=feed_callback)\n", - " time.sleep(2) # Let it run briefly\n", - " monitor.stop()\n", - "except Exception as e:\n", - " print(f\"Feed monitor failed: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Stream Ingestion\n", - "\n", - "Real-time processing with `StreamIngestor` and `StreamMonitor`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import StreamIngestor, StreamMonitor\n", - "\n", - "stream_ingestor = StreamIngestor()\n", - "\n", - "# --- Kafka Processor ---\n", - "kafka_config = {\"bootstrap_servers\": [\"localhost:9092\"]}\n", - "kafka_processor = stream_ingestor.ingest_kafka(\"my-topic\", **kafka_config)\n", - "\n", - "# --- RabbitMQ Processor ---\n", - "rabbitmq_processor = stream_ingestor.ingest_rabbitmq(\"my-queue\", \"amqp://guest:guest@localhost:5672/\")\n", - "\n", - "# --- Stream Monitor ---\n", - "monitor = stream_ingestor.monitor\n", - "health = monitor.check_health()\n", - "print(f\"Stream Health: {health['overall']}\")\n", - "print(f\"Processors: {list(health['processors'].keys())}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Repository Ingestion\n", - "\n", - "Analyzing codebases with `RepoIngestor`, `CodeExtractor`, and `GitAnalyzer`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import RepoIngestor, CodeExtractor, GitAnalyzer\n", - "\n", - "# --- CodeExtractor ---\n", - "code_extractor = CodeExtractor()\n", - "py_code = \"class MyClass:\\n def my_method(self):\\n pass\"\n", - "structure = code_extractor.extract_structure(py_code, language=\"python\")\n", - "print(f\"Classes: {structure.get('classes')}\")\n", - "print(f\"Functions: {structure.get('functions')}\")\n", - "\n", - "# --- RepoIngestor ---\n", - "repo_ingestor = RepoIngestor()\n", - "try:\n", - " repo_data = repo_ingestor.ingest_repository(\"https://github.com/Hawksight-AI/semantica.git\")\n", - " print(f\"Repo Name: {repo_data['name']}\")\n", - "except Exception as e:\n", - " print(f\"Repo ingest failed: {e}\")\n", - "\n", - "# --- GitAnalyzer ---\n", - "try:\n", - " analyzer = GitAnalyzer(\".\")\n", - " stats = analyzer.get_statistics()\n", - " print(f\"Commits in current repo: {stats.get('total_commits', 'N/A')}\")\n", - "except Exception as e:\n", - " print(f\"Git analysis failed: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Email Ingestion\n", - "\n", - "Processing emails with `EmailIngestor` and `AttachmentProcessor`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import EmailIngestor, AttachmentProcessor\n", - "\n", - "# --- AttachmentProcessor ---\n", - "att_processor = AttachmentProcessor()\n", - "dummy_content = b\"PDF Content\"\n", - "saved_path = att_processor.save_attachment(dummy_content, \"doc.pdf\", temp_dir)\n", - "print(f\"Saved attachment to: {saved_path}\")\n", - "\n", - "# --- EmailIngestor ---\n", - "email_ingestor = EmailIngestor()\n", - "try:\n", - " email_ingestor.connect_imap(\"imap.gmail.com\", \"user\", \"pass\")\n", - " emails = email_ingestor.ingest_mailbox(\"INBOX\", max_emails=5)\n", - " print(f\"Fetched {len(emails)} emails\")\n", - "except Exception as e:\n", - " print(f\"Email ingest failed (Auth required): {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 8. Database Ingestion\n", - "\n", - "Connecting to SQL databases with `DBIngestor` and `DatabaseConnector`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import DBIngestor, DatabaseConnector\n", - "import sqlite3\n", - "\n", - "# Setup SQLite DB\n", - "db_path = os.path.join(temp_dir, \"test.db\")\n", - "conn = sqlite3.connect(db_path)\n", - "conn.execute(\"CREATE TABLE items (id INT, name TEXT)\")\n", - "conn.execute(\"INSERT INTO items VALUES (1, 'Item 1'), (2, 'Item 2')\")\n", - "conn.commit()\n", - "conn.close()\n", - "\n", - "# --- DatabaseConnector ---\n", - "connector = DatabaseConnector()\n", - "engine = connector.create_engine(f\"sqlite:///{db_path}\")\n", - "print(f\"Connected to DB: {engine.name}\")\n", - "\n", - "# --- DBIngestor ---\n", - "db_ingestor = DBIngestor()\n", - "table_data = db_ingestor.ingest_database(f\"sqlite:///{db_path}\", table=\"items\")\n", - "print(f\"Table: {table_data.table_name}\")\n", - "print(f\"Rows: {table_data.row_count}\")\n", - "for row in table_data.rows:\n", - " print(f\" - {row}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 9. MCP Ingestion\n", - "\n", - "Integrating with Model Context Protocol servers using `MCPIngestor`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import MCPIngestor\n", - "\n", - "mcp_ingestor = MCPIngestor()\n", - "\n", - "try:\n", - " # Connect\n", - " mcp_ingestor.connect(\"weather_server\", url=\"http://localhost:8000/mcp\")\n", - "\n", - " # Ingest Resources\n", - " resources = mcp_ingestor.ingest_resources(\"weather_server\")\n", - " print(f\"Resources: {len(resources)}\")\n", - "\n", - " # Call Tool\n", - " result = mcp_ingestor.ingest_tool_output(\"weather_server\", \"get_forecast\", {\"city\": \"NYC\"})\n", - " print(f\"Tool Result: {result.content}\")\n", - "except Exception as e:\n", - " print(f\"MCP ingest failed (Server required): {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 10. Configuration\n", - "\n", - "Managing ingestion settings with `IngestConfig`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import IngestConfig, ingest_config\n", - "\n", - "# Global config\n", - "print(f\"Default Source Type: {ingest_config.get('default_source_type')}\")\n", - "\n", - "# Custom config instance\n", - "config = IngestConfig()\n", - "config.set(\"max_file_size\", 1024 * 1024) # 1MB\n", - "print(f\"Max File Size: {config.get('max_file_size')} bytes\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)\n", + "\n", + "# Data Ingestion - Comprehensive Guide\n", + "\n", + "## Overview\n", + "\n", + "This notebook provides a comprehensive guide to Semantica's data ingestion capabilities. It covers all submodules, classes, and helper functions available in the `semantica.ingest` module.\n", + "\n", + "**Documentation**: [Ingest API Reference](https://semantica.readthedocs.io/reference/ingest/)\n", + "\n", + "### Table of Contents\n", + "\n", + "1. **Unified Ingestion**: `ingest` function\n", + "2. **File Ingestion**: `FileIngestor`, `FileTypeDetector`, `CloudStorageIngestor`\n", + "3. **Web Ingestion**: `WebIngestor`, `ContentExtractor`, `SitemapCrawler`, `RobotsChecker`\n", + "4. **Feed Ingestion**: `FeedIngestor`, `FeedMonitor`\n", + "5. **Stream Ingestion**: `StreamIngestor`, `StreamMonitor`\n", + "6. **Repository Ingestion**: `RepoIngestor`, `CodeExtractor`, `GitAnalyzer`\n", + "7. **Email Ingestion**: `EmailIngestor`, `AttachmentProcessor`\n", + "8. **Database Ingestion**: `DBIngestor`, `DatabaseConnector`\n", + "9. **MCP Ingestion**: `MCPIngestor`\n", + "10. **Configuration**: `IngestConfig`\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica with all dependencies:\n", + "\n", + "```bash\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "---\n", + "\n", + "## 1. Unified Ingestion\n", + "\n", + "The `ingest` function is the main entry point for quick data loading. It automatically detects the source type.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import ingest\n", + "import tempfile\n", + "import os\n", + "import json\n", + "\n", + "# Setup temporary directory for examples\n", + "temp_dir = tempfile.mkdtemp()\n", + "sample_file = os.path.join(temp_dir, \"sample.txt\")\n", + "with open(sample_file, 'w') as f:\n", + " f.write(\"Semantica Unified Ingestion Example\")\n", + "\n", + "# Auto-detect file source\n", + "result = ingest(sample_file)\n", + "print(f\"Ingested: {result.name} (Type: {result.file_type})\")\n", + "\n", + "# Explicit source type\n", + "result_explicit = ingest(sample_file, source_type=\"file\")\n", + "print(f\"Explicit Ingest: {result_explicit.name}\")\n", + "\n", + "# Ingest web URL (auto-detected)\n", + "# Note: This will fail if no internet connection\n", + "try:\n", + " result_web = ingest(\"https://example.com\")\n", + " print(f\"Ingested Web: {result_web.title}\")\n", + "except Exception as e:\n", + " print(f\"Web ingestion skipped: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. File Ingestion\n", + "\n", + "Detailed control over file processing using `FileIngestor` and helper classes.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, FileTypeDetector, CloudStorageIngestor\n", + "\n", + "# --- FileTypeDetector ---\n", + "detector = FileTypeDetector()\n", + "detected_type = detector.detect_type(sample_file)\n", + "print(f\"Detected Type: {detected_type}\")\n", + "\n", + "# --- FileIngestor ---\n", + "file_ingestor = FileIngestor()\n", + "\n", + "# Ingest Directory\n", + "subdir = os.path.join(temp_dir, \"docs\")\n", + "os.makedirs(subdir, exist_ok=True)\n", + "with open(os.path.join(subdir, \"note.md\"), 'w') as f:\n", + " f.write(\"# Note\\nThis is a markdown file.\")\n", + "\n", + "files = file_ingestor.ingest_directory(temp_dir, recursive=True)\n", + "print(f\"Ingested {len(files)} files from directory\")\n", + "\n", + "# --- CloudStorageIngestor (Mock Config) ---\n", + "s3_config = {\n", + " \"aws_access_key_id\": \"mock_key\",\n", + " \"aws_secret_access_key\": \"mock_secret\",\n", + " \"region_name\": \"us-east-1\"\n", + "}\n", + "cloud_ingestor = CloudStorageIngestor(provider=\"s3\", **s3_config)\n", + "\n", + "# Example call (will raise error without real credentials)\n", + "try:\n", + " result = cloud_ingestor.ingest(\"s3://my-bucket/data.csv\")\n", + " print(f\"Cloud Ingest: {result.name}\")\n", + "except Exception as e:\n", + " print(f\"Cloud ingestion skipped (Mock Config): {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Web Ingestion\n", + "\n", + "Scraping and crawling with `WebIngestor`, `ContentExtractor`, and `SitemapCrawler`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, ContentExtractor, SitemapCrawler, RobotsChecker\n", + "\n", + "# --- ContentExtractor ---\n", + "extractor = ContentExtractor()\n", + "html_content = \"

Hello World

This is a test.

Link\"\n", + "text = extractor.extract_text(html_content)\n", + "links = extractor.extract_links(html_content, base_url=\"https://example.com\")\n", + "print(f\"Extracted Text: {text}\")\n", + "print(f\"Extracted Links: {links}\")\n", + "\n", + "# --- RobotsChecker ---\n", + "checker = RobotsChecker()\n", + "can_fetch = checker.can_fetch(\"https://www.google.com/search\", \"MyBot\")\n", + "print(f\"Can fetch google search? {can_fetch}\")\n", + "\n", + "# --- WebIngestor ---\n", + "web_ingestor = WebIngestor(delay=1.0)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(\"https://example.com\")\n", + " print(f\"Web Content Title: {web_content.title}\")\n", + "except Exception as e:\n", + " print(f\"Web ingest failed: {e}\")\n", + "\n", + "# --- SitemapCrawler ---\n", + "crawler = SitemapCrawler()\n", + "try:\n", + " urls = crawler.parse_sitemap(\"https://www.google.com/sitemap.xml\")\n", + " print(f\"Found {len(urls)} URLs in sitemap\")\n", + "except Exception as e:\n", + " print(f\"Sitemap crawl failed: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Feed Ingestion\n", + "\n", + "Consuming RSS/Atom feeds with `FeedIngestor` and monitoring with `FeedMonitor`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FeedIngestor, FeedMonitor\n", + "import time\n", + "\n", + "# --- FeedIngestor ---\n", + "feed_ingestor = FeedIngestor()\n", + "try:\n", + " feed_data = feed_ingestor.ingest_feed(\"https://feeds.feedburner.com/oreilly/radar\")\n", + " print(f\"Feed Title: {feed_data.title}\")\n", + "except Exception as e:\n", + " print(f\"Feed ingest failed: {e}\")\n", + "\n", + "# --- FeedMonitor ---\n", + "def feed_callback(feed_data):\n", + " print(f\"Feed Updated: {feed_data.title} with {len(feed_data.items)} items\")\n", + "\n", + "monitor = FeedMonitor(check_interval=5)\n", + "try:\n", + " monitor.monitor(\"https://feeds.feedburner.com/oreilly/radar\", callback=feed_callback)\n", + " time.sleep(2) # Let it run briefly\n", + " monitor.stop()\n", + "except Exception as e:\n", + " print(f\"Feed monitor failed: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Stream Ingestion\n", + "\n", + "Real-time processing with `StreamIngestor` and `StreamMonitor`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import StreamIngestor, StreamMonitor\n", + "\n", + "stream_ingestor = StreamIngestor()\n", + "\n", + "# --- Kafka Processor ---\n", + "kafka_config = {\"bootstrap_servers\": [\"localhost:9092\"]}\n", + "kafka_processor = stream_ingestor.ingest_kafka(\"my-topic\", **kafka_config)\n", + "\n", + "# --- RabbitMQ Processor ---\n", + "rabbitmq_processor = stream_ingestor.ingest_rabbitmq(\"my-queue\", \"amqp://guest:guest@localhost:5672/\")\n", + "\n", + "# --- Stream Monitor ---\n", + "monitor = stream_ingestor.monitor\n", + "health = monitor.check_health()\n", + "print(f\"Stream Health: {health['overall']}\")\n", + "print(f\"Processors: {list(health['processors'].keys())}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Repository Ingestion\n", + "\n", + "Analyzing codebases with `RepoIngestor`, `CodeExtractor`, and `GitAnalyzer`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import RepoIngestor, CodeExtractor, GitAnalyzer\n", + "\n", + "# --- CodeExtractor ---\n", + "code_extractor = CodeExtractor()\n", + "py_code = \"class MyClass:\\n def my_method(self):\\n pass\"\n", + "structure = code_extractor.extract_structure(py_code, language=\"python\")\n", + "print(f\"Classes: {structure.get('classes')}\")\n", + "print(f\"Functions: {structure.get('functions')}\")\n", + "\n", + "# --- RepoIngestor ---\n", + "repo_ingestor = RepoIngestor()\n", + "try:\n", + " repo_data = repo_ingestor.ingest_repository(\"https://github.com/Hawksight-AI/semantica.git\")\n", + " print(f\"Repo Name: {repo_data['name']}\")\n", + "except Exception as e:\n", + " print(f\"Repo ingest failed: {e}\")\n", + "\n", + "# --- GitAnalyzer ---\n", + "try:\n", + " analyzer = GitAnalyzer(\".\")\n", + " stats = analyzer.get_statistics()\n", + " print(f\"Commits in current repo: {stats.get('total_commits', 'N/A')}\")\n", + "except Exception as e:\n", + " print(f\"Git analysis failed: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Email Ingestion\n", + "\n", + "Processing emails with `EmailIngestor` and `AttachmentProcessor`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import EmailIngestor, AttachmentProcessor\n", + "\n", + "# --- AttachmentProcessor ---\n", + "att_processor = AttachmentProcessor()\n", + "dummy_content = b\"PDF Content\"\n", + "saved_path = att_processor.save_attachment(dummy_content, \"doc.pdf\", temp_dir)\n", + "print(f\"Saved attachment to: {saved_path}\")\n", + "\n", + "# --- EmailIngestor ---\n", + "email_ingestor = EmailIngestor()\n", + "try:\n", + " email_ingestor.connect_imap(\"imap.gmail.com\", \"user\", \"pass\")\n", + " emails = email_ingestor.ingest_mailbox(\"INBOX\", max_emails=5)\n", + " print(f\"Fetched {len(emails)} emails\")\n", + "except Exception as e:\n", + " print(f\"Email ingest failed (Auth required): {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Database Ingestion\n", + "\n", + "Connecting to SQL databases with `DBIngestor` and `DatabaseConnector`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import DBIngestor, DatabaseConnector\n", + "import sqlite3\n", + "\n", + "# Setup SQLite DB\n", + "db_path = os.path.join(temp_dir, \"test.db\")\n", + "conn = sqlite3.connect(db_path)\n", + "conn.execute(\"CREATE TABLE items (id INT, name TEXT)\")\n", + "conn.execute(\"INSERT INTO items VALUES (1, 'Item 1'), (2, 'Item 2')\")\n", + "conn.commit()\n", + "conn.close()\n", + "\n", + "# --- DatabaseConnector ---\n", + "connector = DatabaseConnector()\n", + "engine = connector.create_engine(f\"sqlite:///{db_path}\")\n", + "print(f\"Connected to DB: {engine.name}\")\n", + "\n", + "# --- DBIngestor ---\n", + "db_ingestor = DBIngestor()\n", + "table_data = db_ingestor.ingest_database(f\"sqlite:///{db_path}\", table=\"items\")\n", + "print(f\"Table: {table_data.table_name}\")\n", + "print(f\"Rows: {table_data.row_count}\")\n", + "for row in table_data.rows:\n", + " print(f\" - {row}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 9. MCP Ingestion\n", + "\n", + "Integrating with Model Context Protocol servers using `MCPIngestor`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import MCPIngestor\n", + "\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "try:\n", + " # Connect\n", + " mcp_ingestor.connect(\"weather_server\", url=\"http://localhost:8000/mcp\")\n", + "\n", + " # Ingest Resources\n", + " resources = mcp_ingestor.ingest_resources(\"weather_server\")\n", + " print(f\"Resources: {len(resources)}\")\n", + "\n", + " # Call Tool\n", + " result = mcp_ingestor.ingest_tool_output(\"weather_server\", \"get_forecast\", {\"city\": \"NYC\"})\n", + " print(f\"Tool Result: {result.content}\")\n", + "except Exception as e:\n", + " print(f\"MCP ingest failed (Server required): {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 10. Configuration\n", + "\n", + "Managing ingestion settings with `IngestConfig`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import IngestConfig, ingest_config\n", + "\n", + "# Global config\n", + "print(f\"Default Source Type: {ingest_config.get('default_source_type')}\")\n", + "\n", + "# Custom config instance\n", + "config = IngestConfig()\n", + "config.set(\"max_file_size\", 1024 * 1024) # 1MB\n", + "print(f\"Max File Size: {config.get('max_file_size')} bytes\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/cookbook/introduction/03_Document_Parsing.ipynb b/cookbook/introduction/03_Document_Parsing.ipynb index d0353b50..5f10da86 100644 --- a/cookbook/introduction/03_Document_Parsing.ipynb +++ b/cookbook/introduction/03_Document_Parsing.ipynb @@ -39,23 +39,21 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\\n", - "import semantica\\n", - "print(semantica.__version__)\\n" - ] + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.parse import DocumentParser\n", + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import DocumentParser\n", "import tempfile\n", "import os\n", "\n", @@ -268,4 +266,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/04_Data_Normalization.ipynb b/cookbook/introduction/04_Data_Normalization.ipynb index 94d3d683..6b92d71f 100644 --- a/cookbook/introduction/04_Data_Normalization.ipynb +++ b/cookbook/introduction/04_Data_Normalization.ipynb @@ -46,9 +46,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -202,8 +200,8 @@ "\n", "text_samples = [\n", " \"Hello, this is English text.\",\n", - " \"Bonjour, ceci est du texte français.\",\n", - " \"Hola, este es texto en español.\"\n", + " \"Bonjour, ceci est du texte fran\u00e7ais.\",\n", + " \"Hola, este es texto en espa\u00f1ol.\"\n", "]\n", "\n", "for text in text_samples:\n", @@ -242,4 +240,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/05_Entity_Extraction.ipynb b/cookbook/introduction/05_Entity_Extraction.ipynb index 4dc1da15..bc53f9fe 100644 --- a/cookbook/introduction/05_Entity_Extraction.ipynb +++ b/cookbook/introduction/05_Entity_Extraction.ipynb @@ -58,9 +58,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -180,7 +178,7 @@ " unique_entities = list(set(entity_list))\n", " print(f\"\\n{entity_type}:\")\n", " for ent in unique_entities:\n", - " print(f\" • {ent}\")\n", + " print(f\" \u2022 {ent}\")\n", " \n", " print(\"\\n\" + \"=\" * 80)\n", "\n", @@ -192,7 +190,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## ️ Step 3: Different Extraction Methods\n", + "## \ufe0f Step 3: Different Extraction Methods\n", "\n", "Semantica supports multiple extraction methods, each with different strengths:\n", "\n", @@ -200,11 +198,11 @@ "\n", "| Method | Speed | Accuracy | Use Case | Requires |\n", "|--------|-------|----------|----------|----------|\n", - "| **pattern** | | ⭐⭐ | Simple, predictable patterns | Nothing |\n", - "| **regex** | | ⭐⭐⭐ | Custom patterns, IDs, codes | Regex knowledge |\n", - "| **ml** (spaCy) | | ⭐⭐⭐⭐ | General text, multiple languages | spaCy model |\n", - "| **huggingface** | | ⭐⭐⭐⭐⭐ | Domain-specific, fine-tuned | HF model |\n", - "| **llm** | | ⭐⭐⭐⭐⭐ | Complex, custom types | API key |\n", + "| **pattern** | | \u2b50\u2b50 | Simple, predictable patterns | Nothing |\n", + "| **regex** | | \u2b50\u2b50\u2b50 | Custom patterns, IDs, codes | Regex knowledge |\n", + "| **ml** (spaCy) | | \u2b50\u2b50\u2b50\u2b50 | General text, multiple languages | spaCy model |\n", + "| **huggingface** | | \u2b50\u2b50\u2b50\u2b50\u2b50 | Domain-specific, fine-tuned | HF model |\n", + "| **llm** | | \u2b50\u2b50\u2b50\u2b50\u2b50 | Complex, custom types | API key |\n", "\n", "Let's try different methods:" ] @@ -237,10 +235,10 @@ " for entity in entities[:5]: # Show first 5\n", " entity_text = entity.get('text', entity.get('entity', '')) if isinstance(entity, dict) else entity.text\n", " entity_type = entity.get('type', entity.get('label', 'Unknown')) if isinstance(entity, dict) else entity.label\n", - " print(f\" • {entity_text} ({entity_type})\")\n", + " print(f\" \u2022 {entity_text} ({entity_type})\")\n", " \n", " except Exception as e:\n", - " print(f\" ️ Method '{method_name}' not available: {str(e)[:50]}\")\n", + " print(f\" \ufe0f Method '{method_name}' not available: {str(e)[:50]}\")\n", "\n", "print(\"\\n\" + \"=\" * 80)" ] @@ -300,7 +298,7 @@ " entity_text = entity.get('text', entity.get('entity', '')) if isinstance(entity, dict) else entity.text\n", " entity_type = entity.get('type', entity.get('label', 'Unknown')) if isinstance(entity, dict) else entity.label\n", " confidence = entity.get('confidence', 1.0) if isinstance(entity, dict) else getattr(entity, 'confidence', 1.0)\n", - " print(f\" • {entity_text:25s} | {entity_type:10s} | Confidence: {confidence:.2f}\")\n", + " print(f\" \u2022 {entity_text:25s} | {entity_type:10s} | Confidence: {confidence:.2f}\")\n", "\n", "print(f\"\\n Total entities extracted: {len(all_entities)}\")\n", "print(\"=\" * 80)" @@ -310,7 +308,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## ️ Step 5: Entity Classification\n", + "## \ufe0f Step 5: Entity Classification\n", "\n", "Use `EntityClassifier` to classify and group entities by type, and disambiguate similar entities.\n", "\n", @@ -337,7 +335,7 @@ "# Classify the entities we extracted earlier\n", "classified = classifier.classify_entities(all_entities)\n", "\n", - "print(\"️ Entity Classification Results:\\n\")\n", + "print(\"\ufe0f Entity Classification Results:\\n\")\n", "print(\"=\" * 80)\n", "\n", "for entity_type, entity_list in sorted(classified.items()):\n", @@ -351,7 +349,7 @@ " unique_entities.add(entity_text)\n", " \n", " for entity_text in sorted(unique_entities):\n", - " print(f\" • {entity_text}\")\n", + " print(f\" \u2022 {entity_text}\")\n", "\n", "print(\"\\n\" + \"=\" * 80)" ] @@ -405,8 +403,8 @@ " else:\n", " low_confidence.append(entity)\n", "\n", - "print(f\" High Confidence (≥0.8): {len(high_confidence)} entities\")\n", - "print(f\"️ Medium Confidence (0.5-0.8): {len(medium_confidence)} entities\")\n", + "print(f\" High Confidence (\u22650.8): {len(high_confidence)} entities\")\n", + "print(f\"\ufe0f Medium Confidence (0.5-0.8): {len(medium_confidence)} entities\")\n", "print(f\" Low Confidence (<0.5): {len(low_confidence)} entities\")\n", "\n", "print(\"\\n Confidence Distribution:\")\n", @@ -419,7 +417,7 @@ " entity_text = entity.get('text', entity.get('entity', '')) if isinstance(entity, dict) else entity.text\n", " entity_type = entity.get('type', entity.get('label', 'Unknown')) if isinstance(entity, dict) else entity.label\n", " confidence = entity.get('confidence', 1.0) if isinstance(entity, dict) else getattr(entity, 'confidence', 1.0)\n", - " print(f\" • {entity_text} ({entity_type}) - {confidence:.2f}\")\n", + " print(f\" \u2022 {entity_text} ({entity_type}) - {confidence:.2f}\")\n", "\n", "print(\"\\n\" + \"=\" * 80)" ] @@ -478,7 +476,7 @@ " print(f\"\\n{entity_type}:\")\n", " for entity in entities:\n", " entity_text = entity.get('text', entity.get('entity', '')) if isinstance(entity, dict) else entity.text\n", - " print(f\" • {entity_text}\")\n", + " print(f\" \u2022 {entity_text}\")\n", "\n", "print(\"\\n\" + \"=\" * 80)" ] @@ -561,7 +559,7 @@ "3. **Try HuggingFace** for domain-specific needs\n", "4. **Consider LLM** for complex, custom entity types\n", "\n", - "### ️ Optimizing Performance\n", + "### \ufe0f Optimizing Performance\n", "\n", "- **Set appropriate confidence thresholds** (0.7-0.8 for production)\n", "- **Use batch processing** for multiple documents\n", @@ -649,4 +647,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/06_Relation_Extraction.ipynb b/cookbook/introduction/06_Relation_Extraction.ipynb index a150513a..102b8883 100644 --- a/cookbook/introduction/06_Relation_Extraction.ipynb +++ b/cookbook/introduction/06_Relation_Extraction.ipynb @@ -25,7 +25,7 @@ "- Validate triples using `TripleValidator`\n", "- Serialize triples to RDF formats with `RDFSerializer`\n", "- Assess triple quality with `TripleQualityChecker`\n", - "- Build complete entity → relation → triple pipelines\n", + "- Build complete entity \u2192 relation \u2192 triple pipelines\n", "\n", "### What You'll Learn\n", "\n", @@ -58,9 +58,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -170,7 +168,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## ️ Step 2: Different Extraction Methods\n", + "## \ufe0f Step 2: Different Extraction Methods\n", "\n", "Semantica supports multiple relation extraction methods:\n", "\n", @@ -178,11 +176,11 @@ "\n", "| Method | Speed | Accuracy | Use Case | Requires |\n", "|--------|-------|----------|----------|----------|\n", - "| **pattern** | | ⭐⭐⭐ | Common relations | Nothing |\n", - "| **dependency** | | ⭐⭐⭐⭐ | Grammatical relations | spaCy |\n", - "| **cooccurrence** | | ⭐⭐ | Proximity-based | Nothing |\n", - "| **huggingface** | | ⭐⭐⭐⭐⭐ | Domain-specific | HF model |\n", - "| **llm** | | ⭐⭐⭐⭐⭐ | Complex, custom | API key |" + "| **pattern** | | \u2b50\u2b50\u2b50 | Common relations | Nothing |\n", + "| **dependency** | | \u2b50\u2b50\u2b50\u2b50 | Grammatical relations | spaCy |\n", + "| **cooccurrence** | | \u2b50\u2b50 | Proximity-based | Nothing |\n", + "| **huggingface** | | \u2b50\u2b50\u2b50\u2b50\u2b50 | Domain-specific | HF model |\n", + "| **llm** | | \u2b50\u2b50\u2b50\u2b50\u2b50 | Complex, custom | API key |" ] }, { @@ -220,10 +218,10 @@ " source_text = source.get('text', str(source)) if isinstance(source, dict) else getattr(source, 'text', str(source))\n", " target_text = target.get('text', str(target)) if isinstance(target, dict) else getattr(target, 'text', str(target))\n", " \n", - " print(f\" • {source_text} --[{rel_type}]--> {target_text}\")\n", + " print(f\" \u2022 {source_text} --[{rel_type}]--> {target_text}\")\n", " \n", " except Exception as e:\n", - " print(f\" ️ Method '{method_name}' not available: {str(e)[:50]}\")\n", + " print(f\" \ufe0f Method '{method_name}' not available: {str(e)[:50]}\")\n", "\n", "print(\"\\n\" + \"=\" * 80)" ] @@ -285,7 +283,7 @@ " source_text = source.get('text', str(source)) if isinstance(source, dict) else getattr(source, 'text', str(source))\n", " target_text = target.get('text', str(target)) if isinstance(target, dict) else getattr(target, 'text', str(target))\n", " \n", - " print(f\" • {source_text} --[{rel_type}]--> {target_text} (conf: {confidence:.2f})\")\n", + " print(f\" \u2022 {source_text} --[{rel_type}]--> {target_text} (conf: {confidence:.2f})\")\n", "\n", "print(\"\\n\" + \"=\" * 80)" ] @@ -294,7 +292,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## ️ Step 4: Relation Classification\n", + "## \ufe0f Step 4: Relation Classification\n", "\n", "Group and classify extracted relations by their predicate type." ] @@ -315,7 +313,7 @@ "# Classify relations\n", "classified_relations = advanced_extractor.classify_relations(all_relations)\n", "\n", - "print(\"️ Relation Classification:\\n\")\n", + "print(\"\ufe0f Relation Classification:\\n\")\n", "print(\"=\" * 80)\n", "\n", "for rel_type, rel_list in sorted(classified_relations.items()):\n", @@ -329,7 +327,7 @@ " source_text = source.get('text', str(source)) if isinstance(source, dict) else getattr(source, 'text', str(source))\n", " target_text = target.get('text', str(target)) if isinstance(target, dict) else getattr(target, 'text', str(target))\n", " \n", - " print(f\" • {source_text} → {target_text}\")\n", + " print(f\" \u2022 {source_text} \u2192 {target_text}\")\n", "\n", "print(\"\\n\" + \"=\" * 80)" ] @@ -521,7 +519,7 @@ "source": [ "## Step 8: Complete Extraction Pipeline\n", "\n", - "Let's build a complete pipeline: **Entities → Relations → Triples**\n", + "Let's build a complete pipeline: **Entities \u2192 Relations \u2192 Triples**\n", "\n", "This demonstrates the full workflow for knowledge graph construction." ] @@ -603,7 +601,7 @@ "3. **Try co-occurrence** for exploratory analysis\n", "4. **Consider LLM** for complex, domain-specific relations\n", "\n", - "### ️ Optimizing Extraction\n", + "### \ufe0f Optimizing Extraction\n", "\n", "- **Set confidence thresholds** (0.6-0.7 for production)\n", "- **Specify relation_types** to focus extraction\n", @@ -662,7 +660,7 @@ "3. **Configuration is powerful**: Tune parameters for your domain\n", "4. **Triples are standardized**: Use RDF for interoperability\n", "5. **Validation is essential**: Ensure quality before using triples\n", - "6. **Pipelines are efficient**: Extract entities → relations → triples in sequence\n", + "6. **Pipelines are efficient**: Extract entities \u2192 relations \u2192 triples in sequence\n", "\n", "### Next Steps\n", "\n", @@ -701,4 +699,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb b/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb index 1d0cf9e3..f1874344 100644 --- a/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb +++ b/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb @@ -44,9 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -73,7 +71,7 @@ " \"id\": f\"e{i}\",\n", " \"type\": entity.get(\"type\", \"Entity\"),\n", " \"name\": entity.get(\"text\", entity.get(\"entity\", \"\")),\n", - " \"properties\": {}\n", + " \"properties\": {}\n", " })\n", "\n", "relationships = []\n", @@ -199,4 +197,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb b/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb index 1544d021..224b22a1 100644 --- a/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb +++ b/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb @@ -1,300 +1,298 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Your_First_Knowledge_Graph.ipynb)\n", - "\n", - "# 🚀 Your First Knowledge Graph\n", - "\n", - "## Overview\n", - "\n", - "This notebook walks you through creating your first knowledge graph from a simple document. You'll learn the complete end-to-end workflow from ingesting a file to visualizing the resulting knowledge graph.\n", - "\n", - "> [!TIP]\n", - "> This is the perfect starting point if you are new to Semantica. No prior knowledge of knowledge graphs is required!\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/kg/)\n", - "\n", - "### 🎯 Learning Objectives\n", - "\n", - "- **Understand the Workflow**: Learn the `File → Parse → Extract → Graph` pipeline\n", - "- **Ingest Data**: Load documents using `FileIngestor`\n", - "- **Parse Content**: Extract text using `DocumentParser`\n", - "- **Extract Knowledge**: Identify entities using `NERExtractor`\n", - "- **Build Graph**: Construct a graph using `GraphBuilder`\n", - "- **Visualize**: See your graph come to life with `KGVisualizer`\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica from PyPI:\n", - "\n", - "```bash\n", - "pip install semantica\n", - "# Or with all optional dependencies:\n", - "pip install semantica[all]\n", - "```\n", - "\n", - "---\n", - "\n", - "## 🔄 Simple End-to-End Workflow\n", - "\n", - "The complete workflow consists of four main steps:\n", - "\n", - "1. **📥 Ingest** - Load data from files or other sources\n", - "2. **📄 Parse** - Extract and structure content from documents\n", - "3. **⛏️ Extract** - Identify entities and relationships\n", - "4. **🕸️ Build Graph** - Construct the knowledge graph\n", - "\n", - "Each step is demonstrated in the code cells below.\n", - "\n", - "> [!TIP]\n", - "> **Alternative: Using Semantica Framework**\n", - "> \n", - "> For a simpler, high-level approach, you can use the `Semantica` framework class which orchestrates all these steps:\n", - "> \n", - "> ```python\n", - "> from semantica.core import Semantica\n", - "> \n", - "> framework = Semantica()\n", - "> framework.initialize()\n", - "> \n", - "> result = framework.build_knowledge_base(\n", - "> sources=[\"sample_document.txt\"],\n", - "> embeddings=True,\n", - "> graph=True\n", - "> )\n", - "> \n", - "> framework.shutdown()\n", - "> ```\n", - "> \n", - "> This notebook shows the step-by-step approach for learning. See [Core Module Usage Guide](../../../semantica/core/core_usage.md) for more details.\n", - "\n", - "---\n", - "\n", - "## 📂 Step 1: Ingest a File\n", - "\n", - "In this step, we'll use `FileIngestor` to load a document. The ingestor supports various file formats including PDF, DOCX, TXT, and more.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor\n", - "from pathlib import Path\n", - "\n", - "# Initialize the ingestor\n", - "ingestor = FileIngestor()\n", - "\n", - "# Create a sample document for demonstration\n", - "sample_text = \"\"\"\n", - "Apple Inc. is a technology company founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976.\n", - "The company is headquartered in Cupertino, California.\n", - "Tim Cook is the current CEO of Apple Inc.\n", - "Apple designs and manufactures consumer electronics, software, and online services.\n", - "\"\"\"\n", - "\n", - "sample_file = Path(\"sample_document.txt\")\n", - "sample_file.write_text(sample_text)\n", - "\n", - "print(f\"File: {sample_file}\")\n", - "print(f\"Content length: {len(sample_text)} characters\")\n", - "\n", - "# Ingest the file\n", - "file_object = ingestor.ingest_file(sample_file, read_content=True)\n", - "print(f\" File name: {file_object.name}\")\n", - "print(f\" File type: {file_object.file_type}\")\n", - "print(f\" Content available: {file_object.content is not None}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 📄 Step 2: Parse the Document\n", - "\n", - "After ingesting the file, we need to parse it to extract the text content. The `DocumentParser` handles various file formats and extracts structured content.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.parse import DocumentParser\n", - "\n", - "parser = DocumentParser()\n", - "\n", - "# Parse the document to extract text\n", - "parsed_content = parser.parse_document(str(sample_file))\n", - "print(f\" Parsed content length: {len(parsed_content) if parsed_content else 0} characters\")\n", - "print(f\" Preview: {parsed_content[:200] if parsed_content else 'N/A'}...\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ⛏️ Step 3: Extract Entities\n", - "\n", - "Now we'll extract entities from the parsed text using Named Entity Recognition (NER). This identifies people, organizations, locations, dates, and other entities in the text.\n", - "\n", - "> [!NOTE]\n", - "> In a real scenario, you would use `NERExtractor` with an LLM or model backend. Here we simulate the output for demonstration purposes.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import NamedEntityRecognizer, NERExtractor\n", - "\n", - "ner = NamedEntityRecognizer()\n", - "extractor = NERExtractor()\n", - "\n", - "print(f\"\\nText: {parsed_content[:100]}...\")\n", - "\n", - "# Simulated extraction results\n", - "expected_entities = [\n", - " {\"text\": \"Apple Inc.\", \"type\": \"Organization\", \"start\": 0, \"end\": 10},\n", - " {\"text\": \"Steve Jobs\", \"type\": \"Person\", \"start\": 50, \"end\": 60},\n", - " {\"text\": \"Steve Wozniak\", \"type\": \"Person\", \"start\": 62, \"end\": 75},\n", - " {\"text\": \"Ronald Wayne\", \"type\": \"Person\", \"start\": 81, \"end\": 93},\n", - " {\"text\": \"1976\", \"type\": \"Date\", \"start\": 97, \"end\": 101},\n", - " {\"text\": \"Cupertino, California\", \"type\": \"Location\", \"start\": 130, \"end\": 151},\n", - " {\"text\": \"Tim Cook\", \"type\": \"Person\", \"start\": 153, \"end\": 161},\n", - "]\n", - "\n", - "for entity in expected_entities:\n", - " print(f\" - {entity['text']} ({entity['type']})\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 🕸️ Step 4: Build the Knowledge Graph\n", - "\n", - "Using the extracted entities and relationships, we'll construct a knowledge graph. The graph represents entities as nodes and relationships as edges.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphBuilder\n", - "import networkx as nx\n", - "\n", - "builder = GraphBuilder()\n", - "\n", - "# Prepare data for graph construction\n", - "entities_data = [\n", - " {\"id\": f\"entity_{i}\", \"name\": entity[\"text\"], \"type\": entity[\"type\"]}\n", - " for i, entity in enumerate(expected_entities)\n", - "]\n", - "\n", - "relationships_data = [\n", - " {\"source\": \"entity_0\", \"target\": \"entity_1\", \"type\": \"founded_by\"},\n", - " {\"source\": \"entity_0\", \"target\": \"entity_2\", \"type\": \"founded_by\"},\n", - " {\"source\": \"entity_0\", \"target\": \"entity_3\", \"type\": \"founded_by\"},\n", - " {\"source\": \"entity_0\", \"target\": \"entity_4\", \"type\": \"founded_in\"},\n", - " {\"source\": \"entity_0\", \"target\": \"entity_5\", \"type\": \"located_in\"},\n", - " {\"source\": \"entity_6\", \"target\": \"entity_0\", \"type\": \"ceo_of\"},\n", - "]\n", - "\n", - "# Build the graph using NetworkX\n", - "kg = nx.DiGraph()\n", - "\n", - "for entity in entities_data:\n", - " kg.add_node(entity[\"id\"], name=entity[\"name\"], type=entity[\"type\"])\n", - "\n", - "for rel in relationships_data:\n", - " source_name = entities_data[int(rel[\"source\"].split(\"_\")[1])][\"name\"]\n", - " target_name = entities_data[int(rel[\"target\"].split(\"_\")[1])][\"name\"]\n", - " kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"])\n", - "\n", - "print(f\" Nodes (entities): {len(kg.nodes)}\")\n", - "print(f\" Edges (relationships): {len(kg.edges)}\")\n", - "\n", - "for node_id in kg.nodes():\n", - " node_data = kg.nodes[node_id]\n", - " print(f\" Node: {node_data['name']} ({node_data['type']})\")\n", - "\n", - "for source, target, data in kg.edges(data=True):\n", - " source_name = kg.nodes[source]['name']\n", - " target_name = kg.nodes[target]['name']\n", - " print(f\" {source_name} --[{data['type']}]--> {target_name}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 📊 Step 5: Visualize and Analyze\n", - "\n", - "Finally, we'll visualize the knowledge graph and analyze its structure. This helps you understand the relationships and entities in your data.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.visualization import KGVisualizer\n", - "\n", - "visualizer = KGVisualizer()\n", - "\n", - "print(f\" Total entities: {len(kg.nodes)}\")\n", - "print(f\" Total relationships: {len(kg.edges)}\")\n", - "\n", - "entity_types = {}\n", - "for node_id in kg.nodes():\n", - " entity_type = kg.nodes[node_id]['type']\n", - " entity_types[entity_type] = entity_types.get(entity_type, 0) + 1\n", - "\n", - "for etype, count in entity_types.items():\n", - " print(f\" - {etype}: {count}\")\n", - "\n", - "rel_types = {}\n", - "for _, _, data in kg.edges(data=True):\n", - " rel_type = data.get('type', 'unknown')\n", - " rel_types[rel_type] = rel_types.get(rel_type, 0) + 1\n", - "\n", - "for rtype, count in rel_types.items():\n", - " print(f\" - {rtype}: {count}\")\n", - "\n", - "# Cleanup\n", - "if sample_file.exists():\n", - " sample_file.unlink()\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Your_First_Knowledge_Graph.ipynb)\n", + "\n", + "# \ud83d\ude80 Your First Knowledge Graph\n", + "\n", + "## Overview\n", + "\n", + "This notebook walks you through creating your first knowledge graph from a simple document. You'll learn the complete end-to-end workflow from ingesting a file to visualizing the resulting knowledge graph.\n", + "\n", + "> [!TIP]\n", + "> This is the perfect starting point if you are new to Semantica. No prior knowledge of knowledge graphs is required!\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/kg/)\n", + "\n", + "### \ud83c\udfaf Learning Objectives\n", + "\n", + "- **Understand the Workflow**: Learn the `File \u2192 Parse \u2192 Extract \u2192 Graph` pipeline\n", + "- **Ingest Data**: Load documents using `FileIngestor`\n", + "- **Parse Content**: Extract text using `DocumentParser`\n", + "- **Extract Knowledge**: Identify entities using `NERExtractor`\n", + "- **Build Graph**: Construct a graph using `GraphBuilder`\n", + "- **Visualize**: See your graph come to life with `KGVisualizer`\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica from PyPI:\n", + "\n", + "```bash\n", + "pip install semantica\n", + "# Or with all optional dependencies:\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "---\n", + "\n", + "## \ud83d\udd04 Simple End-to-End Workflow\n", + "\n", + "The complete workflow consists of four main steps:\n", + "\n", + "1. **\ud83d\udce5 Ingest** - Load data from files or other sources\n", + "2. **\ud83d\udcc4 Parse** - Extract and structure content from documents\n", + "3. **\u26cf\ufe0f Extract** - Identify entities and relationships\n", + "4. **\ud83d\udd78\ufe0f Build Graph** - Construct the knowledge graph\n", + "\n", + "Each step is demonstrated in the code cells below.\n", + "\n", + "> [!TIP]\n", + "> **Alternative: Using Semantica Framework**\n", + "> \n", + "> For a simpler, high-level approach, you can use the `Semantica` framework class which orchestrates all these steps:\n", + "> \n", + "> ```python\n", + "> from semantica.core import Semantica\n", + "> \n", + "> framework = Semantica()\n", + "> framework.initialize()\n", + "> \n", + "> result = framework.build_knowledge_base(\n", + "> sources=[\"sample_document.txt\"],\n", + "> embeddings=True,\n", + "> graph=True\n", + "> )\n", + "> \n", + "> framework.shutdown()\n", + "> ```\n", + "> \n", + "> This notebook shows the step-by-step approach for learning. See [Core Module Usage Guide](../../../semantica/core/core_usage.md) for more details.\n", + "\n", + "---\n", + "\n", + "## \ud83d\udcc2 Step 1: Ingest a File\n", + "\n", + "In this step, we'll use `FileIngestor` to load a document. The ingestor supports various file formats including PDF, DOCX, TXT, and more.\n" + ] }, - "nbformat": 4, - "nbformat_minor": 2 -} + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor\n", + "from pathlib import Path\n", + "\n", + "# Initialize the ingestor\n", + "ingestor = FileIngestor()\n", + "\n", + "# Create a sample document for demonstration\n", + "sample_text = \"\"\"\n", + "Apple Inc. is a technology company founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976.\n", + "The company is headquartered in Cupertino, California.\n", + "Tim Cook is the current CEO of Apple Inc.\n", + "Apple designs and manufactures consumer electronics, software, and online services.\n", + "\"\"\"\n", + "\n", + "sample_file = Path(\"sample_document.txt\")\n", + "sample_file.write_text(sample_text)\n", + "\n", + "print(f\"File: {sample_file}\")\n", + "print(f\"Content length: {len(sample_text)} characters\")\n", + "\n", + "# Ingest the file\n", + "file_object = ingestor.ingest_file(sample_file, read_content=True)\n", + "print(f\" File name: {file_object.name}\")\n", + "print(f\" File type: {file_object.file_type}\")\n", + "print(f\" Content available: {file_object.content is not None}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## \ud83d\udcc4 Step 2: Parse the Document\n", + "\n", + "After ingesting the file, we need to parse it to extract the text content. The `DocumentParser` handles various file formats and extracts structured content.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import DocumentParser\n", + "\n", + "parser = DocumentParser()\n", + "\n", + "# Parse the document to extract text\n", + "parsed_content = parser.parse_document(str(sample_file))\n", + "print(f\" Parsed content length: {len(parsed_content) if parsed_content else 0} characters\")\n", + "print(f\" Preview: {parsed_content[:200] if parsed_content else 'N/A'}...\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## \u26cf\ufe0f Step 3: Extract Entities\n", + "\n", + "Now we'll extract entities from the parsed text using Named Entity Recognition (NER). This identifies people, organizations, locations, dates, and other entities in the text.\n", + "\n", + "> [!NOTE]\n", + "> In a real scenario, you would use `NERExtractor` with an LLM or model backend. Here we simulate the output for demonstration purposes.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NamedEntityRecognizer, NERExtractor\n", + "\n", + "ner = NamedEntityRecognizer()\n", + "extractor = NERExtractor()\n", + "\n", + "print(f\"\\nText: {parsed_content[:100]}...\")\n", + "\n", + "# Simulated extraction results\n", + "expected_entities = [\n", + " {\"text\": \"Apple Inc.\", \"type\": \"Organization\", \"start\": 0, \"end\": 10},\n", + " {\"text\": \"Steve Jobs\", \"type\": \"Person\", \"start\": 50, \"end\": 60},\n", + " {\"text\": \"Steve Wozniak\", \"type\": \"Person\", \"start\": 62, \"end\": 75},\n", + " {\"text\": \"Ronald Wayne\", \"type\": \"Person\", \"start\": 81, \"end\": 93},\n", + " {\"text\": \"1976\", \"type\": \"Date\", \"start\": 97, \"end\": 101},\n", + " {\"text\": \"Cupertino, California\", \"type\": \"Location\", \"start\": 130, \"end\": 151},\n", + " {\"text\": \"Tim Cook\", \"type\": \"Person\", \"start\": 153, \"end\": 161},\n", + "]\n", + "\n", + "for entity in expected_entities:\n", + " print(f\" - {entity['text']} ({entity['type']})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## \ud83d\udd78\ufe0f Step 4: Build the Knowledge Graph\n", + "\n", + "Using the extracted entities and relationships, we'll construct a knowledge graph. The graph represents entities as nodes and relationships as edges.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "import networkx as nx\n", + "\n", + "builder = GraphBuilder()\n", + "\n", + "# Prepare data for graph construction\n", + "entities_data = [\n", + " {\"id\": f\"entity_{i}\", \"name\": entity[\"text\"], \"type\": entity[\"type\"]}\n", + " for i, entity in enumerate(expected_entities)\n", + "]\n", + "\n", + "relationships_data = [\n", + " {\"source\": \"entity_0\", \"target\": \"entity_1\", \"type\": \"founded_by\"},\n", + " {\"source\": \"entity_0\", \"target\": \"entity_2\", \"type\": \"founded_by\"},\n", + " {\"source\": \"entity_0\", \"target\": \"entity_3\", \"type\": \"founded_by\"},\n", + " {\"source\": \"entity_0\", \"target\": \"entity_4\", \"type\": \"founded_in\"},\n", + " {\"source\": \"entity_0\", \"target\": \"entity_5\", \"type\": \"located_in\"},\n", + " {\"source\": \"entity_6\", \"target\": \"entity_0\", \"type\": \"ceo_of\"},\n", + "]\n", + "\n", + "# Build the graph using NetworkX\n", + "kg = nx.DiGraph()\n", + "\n", + "for entity in entities_data:\n", + " kg.add_node(entity[\"id\"], name=entity[\"name\"], type=entity[\"type\"])\n", + "\n", + "for rel in relationships_data:\n", + " source_name = entities_data[int(rel[\"source\"].split(\"_\")[1])][\"name\"]\n", + " target_name = entities_data[int(rel[\"target\"].split(\"_\")[1])][\"name\"]\n", + " kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"])\n", + "\n", + "print(f\" Nodes (entities): {len(kg.nodes)}\")\n", + "print(f\" Edges (relationships): {len(kg.edges)}\")\n", + "\n", + "for node_id in kg.nodes():\n", + " node_data = kg.nodes[node_id]\n", + " print(f\" Node: {node_data['name']} ({node_data['type']})\")\n", + "\n", + "for source, target, data in kg.edges(data=True):\n", + " source_name = kg.nodes[source]['name']\n", + " target_name = kg.nodes[target]['name']\n", + " print(f\" {source_name} --[{data['type']}]--> {target_name}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## \ud83d\udcca Step 5: Visualize and Analyze\n", + "\n", + "Finally, we'll visualize the knowledge graph and analyze its structure. This helps you understand the relationships and entities in your data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import KGVisualizer\n", + "\n", + "visualizer = KGVisualizer()\n", + "\n", + "print(f\" Total entities: {len(kg.nodes)}\")\n", + "print(f\" Total relationships: {len(kg.edges)}\")\n", + "\n", + "entity_types = {}\n", + "for node_id in kg.nodes():\n", + " entity_type = kg.nodes[node_id]['type']\n", + " entity_types[entity_type] = entity_types.get(entity_type, 0) + 1\n", + "\n", + "for etype, count in entity_types.items():\n", + " print(f\" - {etype}: {count}\")\n", + "\n", + "rel_types = {}\n", + "for _, _, data in kg.edges(data=True):\n", + " rel_type = data.get('type', 'unknown')\n", + " rel_types[rel_type] = rel_types.get(rel_type, 0) + 1\n", + "\n", + "for rtype, count in rel_types.items():\n", + " print(f\" - {rtype}: {count}\")\n", + "\n", + "# Cleanup\n", + "if sample_file.exists():\n", + " sample_file.unlink()\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/cookbook/introduction/09_Graph_Store.ipynb b/cookbook/introduction/09_Graph_Store.ipynb index 6818670e..70b32367 100644 --- a/cookbook/introduction/09_Graph_Store.ipynb +++ b/cookbook/introduction/09_Graph_Store.ipynb @@ -54,7 +54,6 @@ "# For Neo4j (requires Neo4j server)\n", "pip install neo4j\n", "\n", - "# For FalkorDB (requires Redis/FalkorDB server)\n", "pip install falkordb\n", "```\n", @@ -87,9 +86,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -609,4 +606,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/10_Graph_Analytics.ipynb b/cookbook/introduction/10_Graph_Analytics.ipynb index 1fc2b5bc..2fa07804 100644 --- a/cookbook/introduction/10_Graph_Analytics.ipynb +++ b/cookbook/introduction/10_Graph_Analytics.ipynb @@ -45,9 +45,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -186,4 +184,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/11_Chunking_and_Splitting.ipynb b/cookbook/introduction/11_Chunking_and_Splitting.ipynb index e2b3d9b9..d6ab4d8f 100644 --- a/cookbook/introduction/11_Chunking_and_Splitting.ipynb +++ b/cookbook/introduction/11_Chunking_and_Splitting.ipynb @@ -1,864 +1,862 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n", - "\n", - "# Chunking and Splitting - Comprehensive Guide\n", - "\n", - "## Overview\n", - "\n", - "This notebook provides a **comprehensive walkthrough** of Semantica's split module, demonstrating all chunking strategies and methods for optimal document processing. You'll learn to use 15+ splitting methods including standard, semantic, and knowledge graph-aware approaches.\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/split/)\n", - "\n", - "### Learning Objectives\n", - "\n", - "By the end of this notebook, you will be able to:\n", - "\n", - "- Use `TextSplitter` with multiple methods\n", - "- Apply standard splitting methods (recursive, token, sentence, paragraph)\n", - "- Use semantic chunking for topic coherence\n", - "- Apply KG-aware chunking (entity-aware, relation-aware, graph-based)\n", - "- Use specialized chunkers (structural, sliding window, table, hierarchical)\n", - "- Validate chunk quality with `ChunkValidator`\n", - "- Track provenance with `ProvenanceTracker`\n", - "- Choose the right method for your use case\n", - "\n", - "### What You'll Learn\n", - "\n", - "| Component | Purpose | When to Use |\n", - "|-----------|---------|-------------|\n", - "| `TextSplitter` | Unified splitter | All chunking needs |\n", - "| `SemanticChunker` | Semantic boundaries | Topic-based chunks |\n", - "| `EntityAwareChunker` | Preserve entities | GraphRAG workflows |\n", - "| `RelationAwareChunker` | Preserve triples | KG construction |\n", - "| `StructuralChunker` | Document structure | Formatted documents |\n", - "| `HierarchicalChunker` | Multi-level chunks | Large documents |\n", - "\n", - "---\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica from PyPI:\n", - "\n", - "```bash\n", - "pip install semantica\n", - "# Or with all optional dependencies:\n", - "pip install semantica[all]\n", - "```\n", - "\n", - "---" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Basic Chunking with TextSplitter\n", - "\n", - "Let's start with the unified `TextSplitter` interface, which provides access to all chunking methods.\n", - "\n", - "### What is TextSplitter?\n", - "\n", - "`TextSplitter` is a unified interface that supports 15+ chunking methods:\n", - "- **Standard**: recursive, token, sentence, paragraph, character, word\n", - "- **Semantic**: semantic_transformer, llm, huggingface, nltk\n", - "- **KG/Ontology**: entity_aware, relation_aware, graph_based, ontology_aware\n", - "- **Advanced**: hierarchical, structural, sliding_window, table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import TextSplitter\n", - "\n", - "# Sample long text\n", - "text = \"\"\"\n", - "Apple Inc. is a technology company founded by Steve Jobs, Steve Wozniak, and Ronald Wayne \n", - "in Cupertino, California on April 1, 1976. The company's current CEO is Tim Cook, who took \n", - "over from Steve Jobs in August 2011. Apple is headquartered at One Apple Park Way in Cupertino.\n", - "\n", - "Apple develops and sells consumer electronics, computer software, and online services. The company's \n", - "hardware products include the iPhone smartphone, the iPad tablet computer, the Mac personal computer, \n", - "the iPod portable media player, the Apple Watch smartwatch, the Apple TV digital media player, and the \n", - "HomePod smart speaker.\n", - "\n", - "Apple's software includes the macOS and iOS operating systems, the iTunes media player, the Safari web \n", - "browser, and the iLife and iWork creativity and productivity suites. Its online services include the \n", - "iTunes Store, the iOS App Store and Mac App Store, Apple Music, and iCloud.\n", - "\"\"\"\n", - "\n", - "# Basic recursive splitting\n", - "splitter = TextSplitter(\n", - " method=\"recursive\",\n", - " chunk_size=200,\n", - " chunk_overlap=50\n", - ")\n", - "\n", - "chunks = splitter.split(text)\n", - "\n", - "print(f\"Split into {len(chunks)} chunks using recursive method\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " print(f\"\\nChunk {i}:\")\n", - " print(f\" Length: {len(chunk.text)} characters\")\n", - " print(f\" Start: {chunk.start}, End: {chunk.end}\")\n", - " print(f\" Text: {chunk.text[:100]}...\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Standard Splitting Methods\n", - "\n", - "Let's compare different standard splitting methods.\n", - "\n", - "### Method Comparison\n", - "\n", - "| Method | Best For | Speed | Accuracy |\n", - "|--------|----------|-------|----------|\n", - "| **recursive** | General text | Fast | Good |\n", - "| **sentence** | Coherent chunks | Medium | Very Good |\n", - "| **token** | LLM context | Medium | Excellent |\n", - "| **paragraph** | Natural breaks | Fast | Good |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Compare different methods\n", - "methods = [\"recursive\", \"sentence\", \"paragraph\"]\n", - "\n", - "print(\"Comparing Standard Splitting Methods:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for method in methods:\n", - " splitter = TextSplitter(\n", - " method=method,\n", - " chunk_size=200,\n", - " chunk_overlap=50\n", - " )\n", - " \n", - " chunks = splitter.split(text)\n", - " \n", - " print(f\"\\nMethod: {method.upper()}\")\n", - " print(\"-\" * 40)\n", - " print(f\" Chunks created: {len(chunks)}\")\n", - " print(f\" Avg chunk size: {sum(len(c.text) for c in chunks) / len(chunks):.0f} chars\")\n", - " print(f\" First chunk: {chunks[0].text[:80]}...\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Token-Based Splitting\n", - "\n", - "Token-based splitting is crucial for LLM applications where you need to respect token limits.\n", - "\n", - "### Why Token-Based?\n", - "\n", - "- **LLM Context Windows**: GPT-4 has 8K/32K token limits\n", - "- **Accurate Counting**: Character count ≠ token count\n", - "- **Cost Optimization**: Tokens determine API costs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import split_by_tokens\n", - "\n", - "# Token-based splitting\n", - "chunks = split_by_tokens(\n", - " text,\n", - " chunk_size=100, # 100 tokens\n", - " chunk_overlap=20,\n", - " tokenizer=\"tiktoken\",\n", - " model=\"gpt-4\"\n", - ")\n", - "\n", - "print(\"Token-Based Splitting Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " token_count = chunk.metadata.get('token_count', 'N/A')\n", - " print(f\"\\nChunk {i}:\")\n", - " print(f\" Tokens: {token_count}\")\n", - " print(f\" Characters: {len(chunk.text)}\")\n", - " print(f\" Ratio: {len(chunk.text)/token_count if token_count != 'N/A' else 'N/A':.2f} chars/token\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Semantic Chunking\n", - "\n", - "Semantic chunking creates chunks based on semantic boundaries using embeddings.\n", - "\n", - "### How It Works\n", - "\n", - "1. Split text into sentences\n", - "2. Generate embeddings for each sentence\n", - "3. Calculate similarity between consecutive sentences\n", - "4. Create boundaries where similarity drops below threshold" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import SemanticChunker\n", - "\n", - "# Semantic chunking\n", - "semantic_chunker = SemanticChunker(\n", - " chunk_size=200,\n", - " chunk_overlap=50,\n", - " embedding_model=\"all-MiniLM-L6-v2\",\n", - " similarity_threshold=0.7\n", - ")\n", - "\n", - "chunks = semantic_chunker.chunk(text)\n", - "\n", - "print(\"Semantic Chunking Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " coherence = chunk.metadata.get('coherence_score', 'N/A')\n", - " print(f\"\\nChunk {i}:\")\n", - " print(f\" Length: {len(chunk.text)} chars\")\n", - " print(f\" Coherence: {coherence}\")\n", - " print(f\" Text: {chunk.text[:100]}...\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Entity-Aware Chunking for GraphRAG\n", - "\n", - "Entity-aware chunking preserves entity boundaries, crucial for GraphRAG workflows.\n", - "\n", - "### Why Entity-Aware?\n", - "\n", - "- **Preserve Entities**: Don't split \"Steve Jobs\" across chunks\n", - "- **Better Extraction**: Complete entities improve NER accuracy\n", - "- **GraphRAG**: Essential for knowledge graph construction" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import EntityAwareChunker\n", - "\n", - "# Entity-aware chunking\n", - "entity_chunker = EntityAwareChunker(\n", - " chunk_size=200,\n", - " chunk_overlap=50,\n", - " ner_method=\"ml\", # \"ml\" (spaCy), \"pattern\", or \"llm\"\n", - " preserve_entities=True\n", - ")\n", - "\n", - "chunks = entity_chunker.chunk(text)\n", - "\n", - "print(\"Entity-Aware Chunking Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " entities = chunk.metadata.get('entities', [])\n", - " print(f\"\\nChunk {i}:\")\n", - " print(f\" Length: {len(chunk.text)} chars\")\n", - " print(f\" Entities: {len(entities)}\")\n", - " \n", - " if entities:\n", - " entity_texts = [e.get('text', e.get('entity', '')) if isinstance(e, dict) else str(e) for e in entities[:3]]\n", - " print(f\" Sample entities: {entity_texts}\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 6: Relation-Aware Chunking\n", - "\n", - "Relation-aware chunking preserves relationship triples within chunks.\n", - "\n", - "### Why Relation-Aware?\n", - "\n", - "- **Preserve Triples**: Keep (subject, predicate, object) together\n", - "- **KG Construction**: Better for building knowledge graphs\n", - "- **Context**: Relationships need complete context" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import RelationAwareChunker\n", - "\n", - "# Relation-aware chunking\n", - "relation_chunker = RelationAwareChunker(\n", - " chunk_size=200,\n", - " chunk_overlap=50,\n", - " preserve_triples=True\n", - ")\n", - "\n", - "chunks = relation_chunker.chunk(text)\n", - "\n", - "print(\"Relation-Aware Chunking Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " triples = chunk.metadata.get('triples', [])\n", - " relationships = chunk.metadata.get('relationships', [])\n", - " \n", - " print(f\"\\nChunk {i}:\")\n", - " print(f\" Length: {len(chunk.text)} chars\")\n", - " print(f\" Triples: {len(triples)}\")\n", - " print(f\" Relationships: {len(relationships)}\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 7: Structural Chunking\n", - "\n", - "Structural chunking respects document structure like headings, paragraphs, and lists.\n", - "\n", - "### When to Use?\n", - "\n", - "- **Formatted Documents**: Markdown, HTML, structured text\n", - "- **Preserve Hierarchy**: Keep sections together\n", - "- **Better Context**: Headings provide context" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import StructuralChunker\n", - "\n", - "# Markdown text with structure\n", - "markdown_text = \"\"\"\n", - "# Apple Inc.\n", - "\n", - "## History\n", - "\n", - "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976.\n", - "\n", - "## Products\n", - "\n", - "### Hardware\n", - "- iPhone\n", - "- iPad\n", - "- Mac\n", - "\n", - "### Software\n", - "- macOS\n", - "- iOS\n", - "- Safari\n", - "\"\"\"\n", - "\n", - "# Structural chunking\n", - "structural_chunker = StructuralChunker(\n", - " respect_headings=True,\n", - " respect_paragraphs=True,\n", - " respect_lists=True,\n", - " max_chunk_size=500\n", - ")\n", - "\n", - "chunks = structural_chunker.chunk(markdown_text)\n", - "\n", - "print(\"Structural Chunking Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " section = chunk.metadata.get('section_title', 'N/A')\n", - " level = chunk.metadata.get('heading_level', 'N/A')\n", - " \n", - " print(f\"\\nChunk {i}:\")\n", - " print(f\" Section: {section}\")\n", - " print(f\" Level: {level}\")\n", - " print(f\" Text: {chunk.text[:80]}...\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 8: Hierarchical Chunking\n", - "\n", - "Hierarchical chunking creates multi-level chunks for large documents.\n", - "\n", - "### Benefits\n", - "\n", - "- **Multiple Granularities**: Document → Section → Paragraph\n", - "- **Better Navigation**: Parent-child relationships\n", - "- **Flexible Retrieval**: Query at different levels" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import HierarchicalChunker\n", - "\n", - "# Hierarchical chunking\n", - "hierarchical_chunker = HierarchicalChunker(\n", - " chunk_sizes=[400, 200, 100], # 3 levels\n", - " chunk_overlaps=[80, 40, 20],\n", - " create_parent_chunks=True\n", - ")\n", - "\n", - "chunks = hierarchical_chunker.chunk(text)\n", - "\n", - "print(\"Hierarchical Chunking Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " level = chunk.metadata.get('level', 'N/A')\n", - " parent_id = chunk.metadata.get('parent_id', None)\n", - " child_ids = chunk.metadata.get('child_ids', [])\n", - " \n", - " print(f\"\\nChunk {i}:\")\n", - " print(f\" Level: {level}\")\n", - " print(f\" Length: {len(chunk.text)} chars\")\n", - " print(f\" Parent: {parent_id if parent_id else 'None (root)'}\")\n", - " print(f\" Children: {len(child_ids)}\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 9: Sliding Window Chunking\n", - "\n", - "Sliding window creates overlapping fixed-size chunks.\n", - "\n", - "### Use Cases\n", - "\n", - "- **Dense Retrieval**: Ensure no information is missed\n", - "- **Fixed Context**: Consistent chunk sizes\n", - "- **Overlap Control**: Precise overlap management" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import SlidingWindowChunker\n", - "\n", - "# Sliding window chunking\n", - "sliding_chunker = SlidingWindowChunker(\n", - " window_size=150,\n", - " step_size=100, # 50 char overlap\n", - " min_chunk_size=50\n", - ")\n", - "\n", - "chunks = sliding_chunker.chunk(text)\n", - "\n", - "print(\"Sliding Window Chunking Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " overlap = chunk.metadata.get('overlap_chars', 0)\n", - " \n", - " print(f\"\\nWindow {i}:\")\n", - " print(f\" Position: {chunk.start}-{chunk.end}\")\n", - " print(f\" Length: {len(chunk.text)} chars\")\n", - " print(f\" Overlap with previous: {overlap} chars\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 10: Table Chunking\n", - "\n", - "Table chunking preserves table structure while splitting large tables.\n", - "\n", - "### Features\n", - "\n", - "- **Preserve Headers**: Keep column headers in each chunk\n", - "- **Row-Based Splitting**: Split by rows, not characters\n", - "- **Context Inclusion**: Include surrounding text" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import TableChunker\n", - "\n", - "# Text with table\n", - "text_with_table = \"\"\"\n", - "Apple's product lineup includes:\n", - "\n", - "| Product | Category | Release Year |\n", - "|---------|----------|-------------|\n", - "| iPhone | Smartphone | 2007 |\n", - "| iPad | Tablet | 2010 |\n", - "| Mac | Computer | 1984 |\n", - "| Apple Watch | Wearable | 2015 |\n", - "| AirPods | Audio | 2016 |\n", - "\n", - "These products have revolutionized their respective categories.\n", - "\"\"\"\n", - "\n", - "# Table chunking\n", - "table_chunker = TableChunker(\n", - " preserve_headers=True,\n", - " max_rows_per_chunk=3,\n", - " include_context=True,\n", - " table_format=\"markdown\"\n", - ")\n", - "\n", - "chunks = table_chunker.chunk(text_with_table)\n", - "\n", - "print(\"Table Chunking Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "for i, chunk in enumerate(chunks, 1):\n", - " is_table = chunk.metadata.get('is_table', False)\n", - " \n", - " print(f\"\\nChunk {i}:\")\n", - " print(f\" Type: {'Table' if is_table else 'Text'}\")\n", - " \n", - " if is_table:\n", - " rows = chunk.metadata.get('row_count', 'N/A')\n", - " cols = chunk.metadata.get('column_count', 'N/A')\n", - " print(f\" Rows: {rows}, Columns: {cols}\")\n", - " \n", - " print(f\" Content: {chunk.text[:100]}...\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 11: Chunk Validation\n", - "\n", - "Validate chunk quality to ensure optimal processing.\n", - "\n", - "### Validation Checks\n", - "\n", - "- **Size Constraints**: Min/max chunk size\n", - "- **Overlap**: Appropriate overlap percentage\n", - "- **Completeness**: Full text coverage\n", - "- **Quality Score**: Overall quality metric" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import ChunkValidator\n", - "\n", - "# Create chunks\n", - "splitter = TextSplitter(method=\"recursive\", chunk_size=200, chunk_overlap=50)\n", - "chunks = splitter.split(text)\n", - "\n", - "# Validate chunks\n", - "validator = ChunkValidator(\n", - " min_chunk_size=50,\n", - " max_chunk_size=300,\n", - " min_overlap=20,\n", - " max_overlap=100\n", - ")\n", - "\n", - "validation_result = validator.validate(chunks)\n", - "\n", - "print(\"Chunk Validation Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "print(f\"\\nOverall Valid: {validation_result.get('valid', False)}\")\n", - "print(f\"Quality Score: {validation_result.get('quality_score', 0):.2f}\")\n", - "\n", - "issues = validation_result.get('issues', [])\n", - "if issues:\n", - " print(f\"\\nIssues Found: {len(issues)}\")\n", - " for issue in issues[:3]:\n", - " print(f\" - {issue}\")\n", - "else:\n", - " print(\"\\nNo issues found!\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 12: Provenance Tracking\n", - "\n", - "Track chunk origins for data lineage and debugging.\n", - "\n", - "### Why Track Provenance?\n", - "\n", - "- **Data Lineage**: Know where chunks came from\n", - "- **Debugging**: Trace issues back to source\n", - "- **Compliance**: Required for some use cases" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.split import ProvenanceTracker\n", - "\n", - "# Create chunks\n", - "splitter = TextSplitter(method=\"recursive\", chunk_size=200, chunk_overlap=50)\n", - "chunks = splitter.split(text)\n", - "\n", - "# Track provenance\n", - "tracker = ProvenanceTracker()\n", - "\n", - "for chunk in chunks:\n", - " tracker.track(\n", - " chunk=chunk,\n", - " source={\n", - " \"document_id\": \"apple_doc_001\",\n", - " \"file_path\": \"data/apple.txt\",\n", - " \"timestamp\": \"2024-01-01T00:00:00Z\",\n", - " \"method\": \"recursive\"\n", - " }\n", - " )\n", - "\n", - "print(\"Provenance Tracking Results:\\n\")\n", - "print(\"=\" * 80)\n", - "\n", - "# Get lineage for first chunk\n", - "if chunks:\n", - " lineage = tracker.get_lineage(chunks[0].id)\n", - " \n", - " print(f\"\\nLineage for Chunk 1:\")\n", - " print(f\" Source Document: {lineage.get('source', {}).get('document_id')}\")\n", - " print(f\" File Path: {lineage.get('source', {}).get('file_path')}\")\n", - " print(f\" Method: {lineage.get('source', {}).get('method')}\")\n", - " print(f\" Timestamp: {lineage.get('source', {}).get('timestamp')}\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 13: Method Comparison\n", - "\n", - "Let's compare all methods side-by-side to help you choose the right one.\n", - "\n", - "### Comparison Criteria\n", - "\n", - "- **Chunk Count**: Number of chunks created\n", - "- **Average Size**: Average chunk size\n", - "- **Processing Time**: Speed of chunking" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "# Methods to compare\n", - "methods_to_compare = [\n", - " (\"recursive\", {}),\n", - " (\"sentence\", {}),\n", - " (\"paragraph\", {}),\n", - " (\"token\", {\"tokenizer\": \"tiktoken\"}),\n", - "]\n", - "\n", - "print(\"Method Comparison:\\n\")\n", - "print(\"=\" * 80)\n", - "print(f\"{'Method':<15} {'Chunks':<10} {'Avg Size':<12} {'Time (ms)':<12}\")\n", - "print(\"-\" * 80)\n", - "\n", - "for method, kwargs in methods_to_compare:\n", - " try:\n", - " start_time = time.time()\n", - " \n", - " splitter = TextSplitter(\n", - " method=method,\n", - " chunk_size=200,\n", - " chunk_overlap=50,\n", - " **kwargs\n", - " )\n", - " \n", - " chunks = splitter.split(text)\n", - " \n", - " elapsed = (time.time() - start_time) * 1000\n", - " avg_size = sum(len(c.text) for c in chunks) / len(chunks) if chunks else 0\n", - " \n", - " print(f\"{method:<15} {len(chunks):<10} {avg_size:<12.0f} {elapsed:<12.2f}\")\n", - " \n", - " except Exception as e:\n", - " print(f\"{method:<15} Error: {str(e)[:40]}\")\n", - "\n", - "print(\"=\" * 80)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 14: Best Practices\n", - "\n", - "### Choosing the Right Method\n", - "\n", - "1. **General Documents**: Use `recursive` for speed and simplicity\n", - "2. **LLM Applications**: Use `token` to respect context windows\n", - "3. **Semantic Search**: Use `semantic_transformer` for topic coherence\n", - "4. **GraphRAG**: Use `entity_aware` or `relation_aware`\n", - "5. **Structured Docs**: Use `structural` for formatted documents\n", - "6. **Large Documents**: Use `hierarchical` for multi-level access\n", - "\n", - "### Chunk Size Guidelines\n", - "\n", - "| Use Case | Recommended Size | Overlap |\n", - "|----------|------------------|----------|\n", - "| Semantic Search | 512-1024 chars | 20% |\n", - "| LLM Context | 2000-4000 chars | 10-20% |\n", - "| Entity Extraction | 500-1500 chars | 15-25% |\n", - "| Question Answering | 1000-2000 chars | 20% |\n", - "\n", - "### Overlap Recommendations\n", - "\n", - "- **10-15%**: Fast processing, less redundancy\n", - "- **20-25%**: Balanced (recommended)\n", - "- **30-40%**: Maximum context preservation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "### What You've Learned\n", - "\n", - "In this notebook, you've learned how to:\n", - "\n", - "- Use `TextSplitter` with multiple methods\n", - "- Apply standard splitting (recursive, token, sentence, paragraph)\n", - "- Use semantic chunking for topic coherence\n", - "- Apply KG-aware chunking (entity-aware, relation-aware)\n", - "- Use specialized chunkers (structural, hierarchical, sliding window, table)\n", - "- Validate chunk quality\n", - "- Track provenance\n", - "- Choose the right method for your use case\n", - "\n", - "### Key Takeaways\n", - "\n", - "1. **Method Selection Matters**: Different methods for different needs\n", - "2. **Chunk Size is Critical**: Balance between context and processing\n", - "3. **Overlap Helps**: 20% overlap is a good default\n", - "4. **Validate Quality**: Always validate chunks before use\n", - "5. **Track Provenance**: Important for debugging and compliance\n", - "6. **KG-Aware for GraphRAG**: Use entity/relation-aware for knowledge graphs\n", - "\n", - "### Next Steps\n", - "\n", - "**Next Notebook**: [12_Embedding_Generation.ipynb](./12_Embedding_Generation.ipynb) \n", - "Learn how to generate embeddings for your chunks!\n", - "\n", - "**Further Reading**:\n", - "- [Split Module API Reference](https://semantica.readthedocs.io/reference/split/)\n", - "- [Advanced Chunking Strategies](../advanced/11_Text_Chunking_Strategies.ipynb)\n", - "- [GraphRAG Pipeline](../use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n", - "\n", - "---\n", - "\n", - "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n", + "\n", + "# Chunking and Splitting - Comprehensive Guide\n", + "\n", + "## Overview\n", + "\n", + "This notebook provides a **comprehensive walkthrough** of Semantica's split module, demonstrating all chunking strategies and methods for optimal document processing. You'll learn to use 15+ splitting methods including standard, semantic, and knowledge graph-aware approaches.\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/split/)\n", + "\n", + "### Learning Objectives\n", + "\n", + "By the end of this notebook, you will be able to:\n", + "\n", + "- Use `TextSplitter` with multiple methods\n", + "- Apply standard splitting methods (recursive, token, sentence, paragraph)\n", + "- Use semantic chunking for topic coherence\n", + "- Apply KG-aware chunking (entity-aware, relation-aware, graph-based)\n", + "- Use specialized chunkers (structural, sliding window, table, hierarchical)\n", + "- Validate chunk quality with `ChunkValidator`\n", + "- Track provenance with `ProvenanceTracker`\n", + "- Choose the right method for your use case\n", + "\n", + "### What You'll Learn\n", + "\n", + "| Component | Purpose | When to Use |\n", + "|-----------|---------|-------------|\n", + "| `TextSplitter` | Unified splitter | All chunking needs |\n", + "| `SemanticChunker` | Semantic boundaries | Topic-based chunks |\n", + "| `EntityAwareChunker` | Preserve entities | GraphRAG workflows |\n", + "| `RelationAwareChunker` | Preserve triples | KG construction |\n", + "| `StructuralChunker` | Document structure | Formatted documents |\n", + "| `HierarchicalChunker` | Multi-level chunks | Large documents |\n", + "\n", + "---\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica from PyPI:\n", + "\n", + "```bash\n", + "pip install semantica\n", + "# Or with all optional dependencies:\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Basic Chunking with TextSplitter\n", + "\n", + "Let's start with the unified `TextSplitter` interface, which provides access to all chunking methods.\n", + "\n", + "### What is TextSplitter?\n", + "\n", + "`TextSplitter` is a unified interface that supports 15+ chunking methods:\n", + "- **Standard**: recursive, token, sentence, paragraph, character, word\n", + "- **Semantic**: semantic_transformer, llm, huggingface, nltk\n", + "- **KG/Ontology**: entity_aware, relation_aware, graph_based, ontology_aware\n", + "- **Advanced**: hierarchical, structural, sliding_window, table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import TextSplitter\n", + "\n", + "# Sample long text\n", + "text = \"\"\"\n", + "Apple Inc. is a technology company founded by Steve Jobs, Steve Wozniak, and Ronald Wayne \n", + "in Cupertino, California on April 1, 1976. The company's current CEO is Tim Cook, who took \n", + "over from Steve Jobs in August 2011. Apple is headquartered at One Apple Park Way in Cupertino.\n", + "\n", + "Apple develops and sells consumer electronics, computer software, and online services. The company's \n", + "hardware products include the iPhone smartphone, the iPad tablet computer, the Mac personal computer, \n", + "the iPod portable media player, the Apple Watch smartwatch, the Apple TV digital media player, and the \n", + "HomePod smart speaker.\n", + "\n", + "Apple's software includes the macOS and iOS operating systems, the iTunes media player, the Safari web \n", + "browser, and the iLife and iWork creativity and productivity suites. Its online services include the \n", + "iTunes Store, the iOS App Store and Mac App Store, Apple Music, and iCloud.\n", + "\"\"\"\n", + "\n", + "# Basic recursive splitting\n", + "splitter = TextSplitter(\n", + " method=\"recursive\",\n", + " chunk_size=200,\n", + " chunk_overlap=50\n", + ")\n", + "\n", + "chunks = splitter.split(text)\n", + "\n", + "print(f\"Split into {len(chunks)} chunks using recursive method\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Length: {len(chunk.text)} characters\")\n", + " print(f\" Start: {chunk.start}, End: {chunk.end}\")\n", + " print(f\" Text: {chunk.text[:100]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Standard Splitting Methods\n", + "\n", + "Let's compare different standard splitting methods.\n", + "\n", + "### Method Comparison\n", + "\n", + "| Method | Best For | Speed | Accuracy |\n", + "|--------|----------|-------|----------|\n", + "| **recursive** | General text | Fast | Good |\n", + "| **sentence** | Coherent chunks | Medium | Very Good |\n", + "| **token** | LLM context | Medium | Excellent |\n", + "| **paragraph** | Natural breaks | Fast | Good |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Compare different methods\n", + "methods = [\"recursive\", \"sentence\", \"paragraph\"]\n", + "\n", + "print(\"Comparing Standard Splitting Methods:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for method in methods:\n", + " splitter = TextSplitter(\n", + " method=method,\n", + " chunk_size=200,\n", + " chunk_overlap=50\n", + " )\n", + " \n", + " chunks = splitter.split(text)\n", + " \n", + " print(f\"\\nMethod: {method.upper()}\")\n", + " print(\"-\" * 40)\n", + " print(f\" Chunks created: {len(chunks)}\")\n", + " print(f\" Avg chunk size: {sum(len(c.text) for c in chunks) / len(chunks):.0f} chars\")\n", + " print(f\" First chunk: {chunks[0].text[:80]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Token-Based Splitting\n", + "\n", + "Token-based splitting is crucial for LLM applications where you need to respect token limits.\n", + "\n", + "### Why Token-Based?\n", + "\n", + "- **LLM Context Windows**: GPT-4 has 8K/32K token limits\n", + "- **Accurate Counting**: Character count \u2260 token count\n", + "- **Cost Optimization**: Tokens determine API costs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import split_by_tokens\n", + "\n", + "# Token-based splitting\n", + "chunks = split_by_tokens(\n", + " text,\n", + " chunk_size=100, # 100 tokens\n", + " chunk_overlap=20,\n", + " tokenizer=\"tiktoken\",\n", + " model=\"gpt-4\"\n", + ")\n", + "\n", + "print(\"Token-Based Splitting Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " token_count = chunk.metadata.get('token_count', 'N/A')\n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Tokens: {token_count}\")\n", + " print(f\" Characters: {len(chunk.text)}\")\n", + " print(f\" Ratio: {len(chunk.text)/token_count if token_count != 'N/A' else 'N/A':.2f} chars/token\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Semantic Chunking\n", + "\n", + "Semantic chunking creates chunks based on semantic boundaries using embeddings.\n", + "\n", + "### How It Works\n", + "\n", + "1. Split text into sentences\n", + "2. Generate embeddings for each sentence\n", + "3. Calculate similarity between consecutive sentences\n", + "4. Create boundaries where similarity drops below threshold" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import SemanticChunker\n", + "\n", + "# Semantic chunking\n", + "semantic_chunker = SemanticChunker(\n", + " chunk_size=200,\n", + " chunk_overlap=50,\n", + " embedding_model=\"all-MiniLM-L6-v2\",\n", + " similarity_threshold=0.7\n", + ")\n", + "\n", + "chunks = semantic_chunker.chunk(text)\n", + "\n", + "print(\"Semantic Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " coherence = chunk.metadata.get('coherence_score', 'N/A')\n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Coherence: {coherence}\")\n", + " print(f\" Text: {chunk.text[:100]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Entity-Aware Chunking for GraphRAG\n", + "\n", + "Entity-aware chunking preserves entity boundaries, crucial for GraphRAG workflows.\n", + "\n", + "### Why Entity-Aware?\n", + "\n", + "- **Preserve Entities**: Don't split \"Steve Jobs\" across chunks\n", + "- **Better Extraction**: Complete entities improve NER accuracy\n", + "- **GraphRAG**: Essential for knowledge graph construction" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import EntityAwareChunker\n", + "\n", + "# Entity-aware chunking\n", + "entity_chunker = EntityAwareChunker(\n", + " chunk_size=200,\n", + " chunk_overlap=50,\n", + " ner_method=\"ml\", # \"ml\" (spaCy), \"pattern\", or \"llm\"\n", + " preserve_entities=True\n", + ")\n", + "\n", + "chunks = entity_chunker.chunk(text)\n", + "\n", + "print(\"Entity-Aware Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " entities = chunk.metadata.get('entities', [])\n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Entities: {len(entities)}\")\n", + " \n", + " if entities:\n", + " entity_texts = [e.get('text', e.get('entity', '')) if isinstance(e, dict) else str(e) for e in entities[:3]]\n", + " print(f\" Sample entities: {entity_texts}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Relation-Aware Chunking\n", + "\n", + "Relation-aware chunking preserves relationship triples within chunks.\n", + "\n", + "### Why Relation-Aware?\n", + "\n", + "- **Preserve Triples**: Keep (subject, predicate, object) together\n", + "- **KG Construction**: Better for building knowledge graphs\n", + "- **Context**: Relationships need complete context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import RelationAwareChunker\n", + "\n", + "# Relation-aware chunking\n", + "relation_chunker = RelationAwareChunker(\n", + " chunk_size=200,\n", + " chunk_overlap=50,\n", + " preserve_triples=True\n", + ")\n", + "\n", + "chunks = relation_chunker.chunk(text)\n", + "\n", + "print(\"Relation-Aware Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " triples = chunk.metadata.get('triples', [])\n", + " relationships = chunk.metadata.get('relationships', [])\n", + " \n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Triples: {len(triples)}\")\n", + " print(f\" Relationships: {len(relationships)}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Structural Chunking\n", + "\n", + "Structural chunking respects document structure like headings, paragraphs, and lists.\n", + "\n", + "### When to Use?\n", + "\n", + "- **Formatted Documents**: Markdown, HTML, structured text\n", + "- **Preserve Hierarchy**: Keep sections together\n", + "- **Better Context**: Headings provide context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import StructuralChunker\n", + "\n", + "# Markdown text with structure\n", + "markdown_text = \"\"\"\n", + "# Apple Inc.\n", + "\n", + "## History\n", + "\n", + "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976.\n", + "\n", + "## Products\n", + "\n", + "### Hardware\n", + "- iPhone\n", + "- iPad\n", + "- Mac\n", + "\n", + "### Software\n", + "- macOS\n", + "- iOS\n", + "- Safari\n", + "\"\"\"\n", + "\n", + "# Structural chunking\n", + "structural_chunker = StructuralChunker(\n", + " respect_headings=True,\n", + " respect_paragraphs=True,\n", + " respect_lists=True,\n", + " max_chunk_size=500\n", + ")\n", + "\n", + "chunks = structural_chunker.chunk(markdown_text)\n", + "\n", + "print(\"Structural Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " section = chunk.metadata.get('section_title', 'N/A')\n", + " level = chunk.metadata.get('heading_level', 'N/A')\n", + " \n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Section: {section}\")\n", + " print(f\" Level: {level}\")\n", + " print(f\" Text: {chunk.text[:80]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Hierarchical Chunking\n", + "\n", + "Hierarchical chunking creates multi-level chunks for large documents.\n", + "\n", + "### Benefits\n", + "\n", + "- **Multiple Granularities**: Document \u2192 Section \u2192 Paragraph\n", + "- **Better Navigation**: Parent-child relationships\n", + "- **Flexible Retrieval**: Query at different levels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import HierarchicalChunker\n", + "\n", + "# Hierarchical chunking\n", + "hierarchical_chunker = HierarchicalChunker(\n", + " chunk_sizes=[400, 200, 100], # 3 levels\n", + " chunk_overlaps=[80, 40, 20],\n", + " create_parent_chunks=True\n", + ")\n", + "\n", + "chunks = hierarchical_chunker.chunk(text)\n", + "\n", + "print(\"Hierarchical Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " level = chunk.metadata.get('level', 'N/A')\n", + " parent_id = chunk.metadata.get('parent_id', None)\n", + " child_ids = chunk.metadata.get('child_ids', [])\n", + " \n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Level: {level}\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Parent: {parent_id if parent_id else 'None (root)'}\")\n", + " print(f\" Children: {len(child_ids)}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Sliding Window Chunking\n", + "\n", + "Sliding window creates overlapping fixed-size chunks.\n", + "\n", + "### Use Cases\n", + "\n", + "- **Dense Retrieval**: Ensure no information is missed\n", + "- **Fixed Context**: Consistent chunk sizes\n", + "- **Overlap Control**: Precise overlap management" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import SlidingWindowChunker\n", + "\n", + "# Sliding window chunking\n", + "sliding_chunker = SlidingWindowChunker(\n", + " window_size=150,\n", + " step_size=100, # 50 char overlap\n", + " min_chunk_size=50\n", + ")\n", + "\n", + "chunks = sliding_chunker.chunk(text)\n", + "\n", + "print(\"Sliding Window Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " overlap = chunk.metadata.get('overlap_chars', 0)\n", + " \n", + " print(f\"\\nWindow {i}:\")\n", + " print(f\" Position: {chunk.start}-{chunk.end}\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Overlap with previous: {overlap} chars\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: Table Chunking\n", + "\n", + "Table chunking preserves table structure while splitting large tables.\n", + "\n", + "### Features\n", + "\n", + "- **Preserve Headers**: Keep column headers in each chunk\n", + "- **Row-Based Splitting**: Split by rows, not characters\n", + "- **Context Inclusion**: Include surrounding text" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import TableChunker\n", + "\n", + "# Text with table\n", + "text_with_table = \"\"\"\n", + "Apple's product lineup includes:\n", + "\n", + "| Product | Category | Release Year |\n", + "|---------|----------|-------------|\n", + "| iPhone | Smartphone | 2007 |\n", + "| iPad | Tablet | 2010 |\n", + "| Mac | Computer | 1984 |\n", + "| Apple Watch | Wearable | 2015 |\n", + "| AirPods | Audio | 2016 |\n", + "\n", + "These products have revolutionized their respective categories.\n", + "\"\"\"\n", + "\n", + "# Table chunking\n", + "table_chunker = TableChunker(\n", + " preserve_headers=True,\n", + " max_rows_per_chunk=3,\n", + " include_context=True,\n", + " table_format=\"markdown\"\n", + ")\n", + "\n", + "chunks = table_chunker.chunk(text_with_table)\n", + "\n", + "print(\"Table Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " is_table = chunk.metadata.get('is_table', False)\n", + " \n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Type: {'Table' if is_table else 'Text'}\")\n", + " \n", + " if is_table:\n", + " rows = chunk.metadata.get('row_count', 'N/A')\n", + " cols = chunk.metadata.get('column_count', 'N/A')\n", + " print(f\" Rows: {rows}, Columns: {cols}\")\n", + " \n", + " print(f\" Content: {chunk.text[:100]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 11: Chunk Validation\n", + "\n", + "Validate chunk quality to ensure optimal processing.\n", + "\n", + "### Validation Checks\n", + "\n", + "- **Size Constraints**: Min/max chunk size\n", + "- **Overlap**: Appropriate overlap percentage\n", + "- **Completeness**: Full text coverage\n", + "- **Quality Score**: Overall quality metric" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import ChunkValidator\n", + "\n", + "# Create chunks\n", + "splitter = TextSplitter(method=\"recursive\", chunk_size=200, chunk_overlap=50)\n", + "chunks = splitter.split(text)\n", + "\n", + "# Validate chunks\n", + "validator = ChunkValidator(\n", + " min_chunk_size=50,\n", + " max_chunk_size=300,\n", + " min_overlap=20,\n", + " max_overlap=100\n", + ")\n", + "\n", + "validation_result = validator.validate(chunks)\n", + "\n", + "print(\"Chunk Validation Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "print(f\"\\nOverall Valid: {validation_result.get('valid', False)}\")\n", + "print(f\"Quality Score: {validation_result.get('quality_score', 0):.2f}\")\n", + "\n", + "issues = validation_result.get('issues', [])\n", + "if issues:\n", + " print(f\"\\nIssues Found: {len(issues)}\")\n", + " for issue in issues[:3]:\n", + " print(f\" - {issue}\")\n", + "else:\n", + " print(\"\\nNo issues found!\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 12: Provenance Tracking\n", + "\n", + "Track chunk origins for data lineage and debugging.\n", + "\n", + "### Why Track Provenance?\n", + "\n", + "- **Data Lineage**: Know where chunks came from\n", + "- **Debugging**: Trace issues back to source\n", + "- **Compliance**: Required for some use cases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import ProvenanceTracker\n", + "\n", + "# Create chunks\n", + "splitter = TextSplitter(method=\"recursive\", chunk_size=200, chunk_overlap=50)\n", + "chunks = splitter.split(text)\n", + "\n", + "# Track provenance\n", + "tracker = ProvenanceTracker()\n", + "\n", + "for chunk in chunks:\n", + " tracker.track(\n", + " chunk=chunk,\n", + " source={\n", + " \"document_id\": \"apple_doc_001\",\n", + " \"file_path\": \"data/apple.txt\",\n", + " \"timestamp\": \"2024-01-01T00:00:00Z\",\n", + " \"method\": \"recursive\"\n", + " }\n", + " )\n", + "\n", + "print(\"Provenance Tracking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "# Get lineage for first chunk\n", + "if chunks:\n", + " lineage = tracker.get_lineage(chunks[0].id)\n", + " \n", + " print(f\"\\nLineage for Chunk 1:\")\n", + " print(f\" Source Document: {lineage.get('source', {}).get('document_id')}\")\n", + " print(f\" File Path: {lineage.get('source', {}).get('file_path')}\")\n", + " print(f\" Method: {lineage.get('source', {}).get('method')}\")\n", + " print(f\" Timestamp: {lineage.get('source', {}).get('timestamp')}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 13: Method Comparison\n", + "\n", + "Let's compare all methods side-by-side to help you choose the right one.\n", + "\n", + "### Comparison Criteria\n", + "\n", + "- **Chunk Count**: Number of chunks created\n", + "- **Average Size**: Average chunk size\n", + "- **Processing Time**: Speed of chunking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "# Methods to compare\n", + "methods_to_compare = [\n", + " (\"recursive\", {}),\n", + " (\"sentence\", {}),\n", + " (\"paragraph\", {}),\n", + " (\"token\", {\"tokenizer\": \"tiktoken\"}),\n", + "]\n", + "\n", + "print(\"Method Comparison:\\n\")\n", + "print(\"=\" * 80)\n", + "print(f\"{'Method':<15} {'Chunks':<10} {'Avg Size':<12} {'Time (ms)':<12}\")\n", + "print(\"-\" * 80)\n", + "\n", + "for method, kwargs in methods_to_compare:\n", + " try:\n", + " start_time = time.time()\n", + " \n", + " splitter = TextSplitter(\n", + " method=method,\n", + " chunk_size=200,\n", + " chunk_overlap=50,\n", + " **kwargs\n", + " )\n", + " \n", + " chunks = splitter.split(text)\n", + " \n", + " elapsed = (time.time() - start_time) * 1000\n", + " avg_size = sum(len(c.text) for c in chunks) / len(chunks) if chunks else 0\n", + " \n", + " print(f\"{method:<15} {len(chunks):<10} {avg_size:<12.0f} {elapsed:<12.2f}\")\n", + " \n", + " except Exception as e:\n", + " print(f\"{method:<15} Error: {str(e)[:40]}\")\n", + "\n", + "print(\"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 14: Best Practices\n", + "\n", + "### Choosing the Right Method\n", + "\n", + "1. **General Documents**: Use `recursive` for speed and simplicity\n", + "2. **LLM Applications**: Use `token` to respect context windows\n", + "3. **Semantic Search**: Use `semantic_transformer` for topic coherence\n", + "4. **GraphRAG**: Use `entity_aware` or `relation_aware`\n", + "5. **Structured Docs**: Use `structural` for formatted documents\n", + "6. **Large Documents**: Use `hierarchical` for multi-level access\n", + "\n", + "### Chunk Size Guidelines\n", + "\n", + "| Use Case | Recommended Size | Overlap |\n", + "|----------|------------------|----------|\n", + "| Semantic Search | 512-1024 chars | 20% |\n", + "| LLM Context | 2000-4000 chars | 10-20% |\n", + "| Entity Extraction | 500-1500 chars | 15-25% |\n", + "| Question Answering | 1000-2000 chars | 20% |\n", + "\n", + "### Overlap Recommendations\n", + "\n", + "- **10-15%**: Fast processing, less redundancy\n", + "- **20-25%**: Balanced (recommended)\n", + "- **30-40%**: Maximum context preservation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### What You've Learned\n", + "\n", + "In this notebook, you've learned how to:\n", + "\n", + "- Use `TextSplitter` with multiple methods\n", + "- Apply standard splitting (recursive, token, sentence, paragraph)\n", + "- Use semantic chunking for topic coherence\n", + "- Apply KG-aware chunking (entity-aware, relation-aware)\n", + "- Use specialized chunkers (structural, hierarchical, sliding window, table)\n", + "- Validate chunk quality\n", + "- Track provenance\n", + "- Choose the right method for your use case\n", + "\n", + "### Key Takeaways\n", + "\n", + "1. **Method Selection Matters**: Different methods for different needs\n", + "2. **Chunk Size is Critical**: Balance between context and processing\n", + "3. **Overlap Helps**: 20% overlap is a good default\n", + "4. **Validate Quality**: Always validate chunks before use\n", + "5. **Track Provenance**: Important for debugging and compliance\n", + "6. **KG-Aware for GraphRAG**: Use entity/relation-aware for knowledge graphs\n", + "\n", + "### Next Steps\n", + "\n", + "**Next Notebook**: [12_Embedding_Generation.ipynb](./12_Embedding_Generation.ipynb) \n", + "Learn how to generate embeddings for your chunks!\n", + "\n", + "**Further Reading**:\n", + "- [Split Module API Reference](https://semantica.readthedocs.io/reference/split/)\n", + "- [Advanced Chunking Strategies](../advanced/11_Text_Chunking_Strategies.ipynb)\n", + "- [GraphRAG Pipeline](../use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n", + "\n", + "---\n", + "\n", + "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/cookbook/introduction/11_Graph_Quality.ipynb b/cookbook/introduction/11_Graph_Quality.ipynb index eb7b05c1..b3db3107 100644 --- a/cookbook/introduction/11_Graph_Quality.ipynb +++ b/cookbook/introduction/11_Graph_Quality.ipynb @@ -40,27 +40,23 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\\n", - "import semantica\\n", - "print(semantica.__version__)\\n" - ] + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "from semantica.kg import GraphBuilder\n", "\n", "builder = GraphBuilder()\n", - "\n", "entities = [\n", " {\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\", \"properties\": {}}\n", @@ -92,7 +88,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "consistency_checker = ConsistencyChecker()\n", "\n", @@ -118,7 +113,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "completeness_validator = CompletenessValidator()\n", "\n", @@ -144,7 +138,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "quality_metrics = QualityMetrics()\n", "\n", @@ -179,4 +172,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/12_Embedding_Generation.ipynb b/cookbook/introduction/12_Embedding_Generation.ipynb index dbb30097..105a3aa3 100644 --- a/cookbook/introduction/12_Embedding_Generation.ipynb +++ b/cookbook/introduction/12_Embedding_Generation.ipynb @@ -40,23 +40,21 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\\n", - "import semantica\\n", - "print(semantica.__version__)\\n" - ] + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import EmbeddingGenerator\n", + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.embeddings import EmbeddingGenerator\n", "\n", "generator = EmbeddingGenerator()\n", "\n", @@ -150,4 +148,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/13_Vector_Store.ipynb b/cookbook/introduction/13_Vector_Store.ipynb index d5be0570..43ed0579 100644 --- a/cookbook/introduction/13_Vector_Store.ipynb +++ b/cookbook/introduction/13_Vector_Store.ipynb @@ -59,9 +59,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -582,4 +580,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/14_Ontology.ipynb b/cookbook/introduction/14_Ontology.ipynb index 444b8164..0f88ade5 100644 --- a/cookbook/introduction/14_Ontology.ipynb +++ b/cookbook/introduction/14_Ontology.ipynb @@ -1,554 +1,543 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)\n", - "\n", - "# Mastering Ontology Generation with Semantica\n", - "\n", - "Welcome to the comprehensive guide on Semantica's Ontology Module. This module is the powerhouse for structuring your data into meaningful knowledge graphs, providing a complete 6-stage pipeline from raw data to validated OWL ontologies.\n", - "\n", - "In this notebook, we will dive deep into:\n", - "1. **The 6-Stage Generation Pipeline**: Understanding how Semantica transforms data into knowledge.\n", - "2. **Core Components in Focus**: Detailed usage of `ClassInferrer`, `PropertyGenerator`, and `OntologyOptimizer`.\n", - "3. **Validation & Quality**: ensuring your ontology is consistent and structurally sound.\n", - "4. **Visualize**: exploring your ontology with interactive charts and hierarchies.\n", - "5. **Advanced Usage**: Text-to-Ontology (LLM), Competency Questions, and Lifecycle Management.\n", - "6. **Exporting & Interoperability**: Saving your work in standard formats like Turtle and RDF/XML.\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/ontology/)\n", - "\n", - "## Getting Started\n", - "\n", - "First, let's setup our environment and initialize the `OntologyEngine`. This engine is the unified entry point for all ontology operations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Install semantica if not already installed\n", - "# !pip install semantica\n", - "# !pip install plotly # Required for visualization\n", - "\n", - "from semantica.ontology import OntologyEngine, OntologyGenerator\n", - "from semantica.utils.logging import get_logger\n", - "\n", - "# Initialize logger for visibility\n", - "logger = get_logger(\"ontology_guide\")\n", - "\n", - "# Initialize the Engine\n", - "# base_uri defines the namespace root for your ontology\n", - "engine = OntologyEngine(base_uri=\"https://docs.semantica.dev/ontology/\")\n", - "\n", - "print(\"Ontology Engine initialized successfully!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## The 6-Stage Generation Pipeline\n", - "\n", - "Semantica uses a sophisticated 6-stage pipeline to robustly generate ontologies. This automated process takes raw entity and relationship data and produces a high-quality OWL ontology.\n", - "\n", - "### The Stages:\n", - "1. **Semantic Network Parsing**: Extracts raw concepts and connections from your inputs.\n", - "2. **YAML-to-Definition**: Transforms concepts into structured class definitions.\n", - "3. **Definition-to-Types**: Maps definitions to formal OWL types (e.g., `owl:Class`, `owl:ObjectProperty`).\n", - "4. **Hierarchy Generation**: Builds a taxonomic structure (parent-child relationships) using `associatedWith` or linguistic patterns.\n", - "5. **TTL Generation**: Serializes the in-memory structure into Turtle format logic.\n", - "6. **Symbolic Validation**: Validates the result using reasoners like HermiT (if available) or structural checks.\n", - "\n", - "Let's see this in action with some sample data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Sample Data: A simple corporate structure\n", - "entities = [\n", - " {\"id\": \"e1\", \"type\": \"Company\", \"name\": \"TechCorp\", \"founded\": \"2010\"},\n", - " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Alice\", \"role\": \"CEO\"},\n", - " {\"id\": \"e3\", \"type\": \"Person\", \"name\": \"Bob\", \"role\": \"CTO\"},\n", - " {\"id\": \"e4\", \"type\": \"Department\", \"name\": \"Engineering\"},\n", - " {\"id\": \"e5\", \"type\": \"Project\", \"name\": \"Project Phoenix\"}\n", - "]\n", - "\n", - "relationships = [\n", - " {\"source\": \"e2\", \"target\": \"e1\", \"type\": \"leads\"},\n", - " {\"source\": \"e3\", \"target\": \"e4\", \"type\": \"manages\"},\n", - " {\"source\": \"e4\", \"target\": \"e1\", \"type\": \"part_of\"},\n", - " {\"source\": \"e3\", \"target\": \"e5\", \"type\": \"works_on\"}\n", - "]\n", - "\n", - "data = {\n", - " \"entities\": entities,\n", - " \"relationships\": relationships\n", - "}\n", - "\n", - "# Run the full pipeline\n", - "ontology = engine.from_data(data, name=\"CorporateOntology\")\n", - "\n", - "print(f\"Generated Ontology: {ontology['name']}\")\n", - "print(f\"Classes Found: {len(ontology['classes'])}\")\n", - "print(f\"Properties Found: {len(ontology['properties'])}\")\n", - "print(f\"Validation Status: Valid={ontology.get('validation_result', {}).get('valid', 'Unknown')}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Inspecting the Results\n", - "\n", - "The generated `ontology` object is a rich dictionary containing all the inferred structure. Let's peek inside to see what Classes and Properties were created." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Inspect Classes\n", - "print(\"--- Inferred Classes ---\")\n", - "for cls in ontology['classes']:\n", - " print(f\"Class: {cls['name']}\")\n", - " print(f\" URI: {cls.get('uri')}\")\n", - " # Check if a hierarchy was inferred\n", - " if cls.get('subClassOf'):\n", - " print(f\" Parent: {cls['subClassOf']}\")\n", - " print(\"\")\n", - "\n", - "# Inspect Properties\n", - "print(\"--- Inferred Properties ---\")\n", - "for prop in ontology['properties']:\n", - " type_label = \"Object Property\" if prop['type'] == 'object' else \"Data Property\"\n", - " print(f\"{prop['name']} [{type_label}]\")\n", - " print(f\" Domain: {prop.get('domain')}\")\n", - " print(f\" Range: {prop.get('range')}\")\n", - " print(\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Deep Dive: Component by Component\n", - "\n", - "While `OntologyEngine` is great for one-shot generation, you often need fine-grained control. Let's look at the individual tools that power the engine.\n", - "\n", - "### 1. `ClassInferrer`: Mastering Class Discovery\n", - "\n", - "The `ClassInferrer` analyzes entities to find patterns. It can handle noise and only creates classes for types that appear frequently enough.\n", - "\n", - "* **`min_occurrences`**: Ignores types with fewer entities than this count.\n", - "* **`build_class_hierarchy`**: Toggles automatic parent-child detection.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import ClassInferrer\n", - "\n", - "# Initialize inferrer with a threshold\n", - "# We set min_occurrences=1 here to capture everything in our small example\n", - "inferrer = ClassInferrer(min_occurrences=1)\n", - "\n", - "raw_entities = [\n", - " {\"type\": \"Manager\", \"name\": \"Dave\", \"level\": 5},\n", - " {\"type\": \"Manager\", \"name\": \"Eve\", \"level\": 4},\n", - " {\"type\": \"Employee\", \"name\": \"Frank\"}, # Only 1 employee\n", - " {\"type\": \"TemporaryWorker\", \"name\": \"Grace\"} \n", - "]\n", - "\n", - "# Infer classes\n", - "classes = inferrer.infer_classes(raw_entities, build_hierarchy=True)\n", - "\n", - "print(f\"Inferred {len(classes)} classes from raw entities.\")\n", - "for c in classes:\n", - " print(f\"- {c['name']} (Count: {c['entity_count']})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2. `PropertyGenerator`: The Glue of the Ontology\n", - "\n", - "Properties define relationships. Semantica distinguishes between:\n", - "* **Object Properties**: Links between two entities (e.g., `leads` between Person and Company).\n", - "* **Data Properties**: Attributes of an entity (e.g., `founded` year of a Company).\n", - "\n", - "The `PropertyGenerator` automatically detects this distinction." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import PropertyGenerator\n", - "\n", - "prop_gen = PropertyGenerator()\n", - "\n", - "# We need the classes first to help property generation context\n", - "context_classes = classes # reusing from previous step\n", - "\n", - "# Let's define some relationships and attributes implicitly via entities\n", - "# Note: 'level' in Manager entities is a potential data property\n", - "complex_entities = [\n", - " {\"id\": \"m1\", \"type\": \"Manager\", \"name\": \"Dave\", \"level\": 5},\n", - " {\"id\": \"e1\", \"type\": \"Employee\", \"name\": \"Frank\"}\n", - "]\n", - "complex_relationships = [\n", - " {\"source\": \"m1\", \"target\": \"e1\", \"type\": \"supervises\"} # Object property\n", - "]\n", - "\n", - "properties = prop_gen.infer_properties(\n", - " entities=complex_entities,\n", - " relationships=complex_relationships,\n", - " classes=context_classes\n", - ")\n", - "\n", - "print(\"--- Property Types Identified ---\")\n", - "for p in properties:\n", - " print(f\"Property: {p['name']}\")\n", - " print(f\" Type: {p['type']}\")\n", - " print(f\" Domain: {p['domain']} -> Range: {p['range']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3. `OntologyOptimizer`: Refining the Structure\n", - "\n", - "Before finalizing, it's good practice to optimize. The optimizer removes redundancies and improves coherence, such as ensuring all classes have proper labels and valid URIs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import OntologyOptimizer\n", - "\n", - "optimizer = OntologyOptimizer()\n", - "\n", - "# Let's pretend we have a messy ontology dict\n", - "messy_ontology = {\n", - " \"classes\": [\n", - " {\"name\": \"Person\", \"uri\": \"...Person\"},\n", - " {\"name\": \"Person\", \"uri\": \"...Person\"} # Duplicate!\n", - " ],\n", - " \"properties\": []\n", - "}\n", - "\n", - "clean_ontology = optimizer.optimize_ontology(messy_ontology, remove_redundancy=True)\n", - "\n", - "print(f\"Original Classes: {len(messy_ontology['classes'])}\")\n", - "print(f\"Optimized Classes: {len(clean_ontology['classes'])}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Validation and Quality Control\n", - "\n", - "Semantica includes a robust `OntologyValidator`. It checks for:\n", - "1. **Structure**: Missing fields, malformed URIs.\n", - "2. **Consistency**: Circular hierarchies, contradictory definitions.\n", - "3. **Metrics**: Depth of hierarchy, property usage.\n", - "\n", - "If you have `Owlready2` installed, it can even run a reasoner (HermiT or Pellet) to prove logical consistency." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import OntologyValidator\n", - "\n", - "validator = OntologyValidator(\n", - " check_consistency=True,\n", - " check_satisfiability=True\n", - ")\n", - "\n", - "# Validate our previously generated 'ontology'\n", - "result = validator.validate_ontology(ontology)\n", - "\n", - "print(f\"Is Valid? {result.valid}\")\n", - "print(f\"Is Consistent? {result.consistent}\")\n", - "\n", - "if result.errors:\n", - " print(\"Errors Found:\", result.errors)\n", - "if result.warnings:\n", - " print(\"Warnings:\", result.warnings)\n", - " \n", - "# Check Metrics\n", - "print(\"Metrics:\", result.metrics)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Visualization\n", - "\n", - "A picture is worth a thousand triples! The `OntologyVisualizer` lets you explore your ontology's structure interactively.\n", - "\n", - "We can visualize:\n", - "* **Class Hierarchies**: Tree diagrams of class inheritance.\n", - "* **Structure Networks**: The full graph of classes and properties.\n", - "* **Metrics Dashboards**: High-level stats at a glance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.visualization import OntologyVisualizer\n", - "\n", - "viz = OntologyVisualizer()\n", - "\n", - "# 1. Interactive Class Hierarchy\n", - "# Returns a Plotly figure you can interact with\n", - "fig_hierarchy = viz.visualize_hierarchy(ontology, output=\"interactive\")\n", - "if fig_hierarchy:\n", - " fig_hierarchy.show()\n", - "\n", - "# 2. Ontology Structure Network\n", - "# See how classes and properties connect\n", - "fig_structure = viz.visualize_structure(ontology, output=\"interactive\")\n", - "if fig_structure:\n", - " fig_structure.show()\n", - "\n", - "# 3. Metrics Dashboard\n", - "# View counts, depths, and statistics\n", - "fig_metrics = viz.visualize_metrics(ontology, output=\"interactive\")\n", - "if fig_metrics:\n", - " fig_metrics.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Advanced Usage: Lifecycle & AI\n", - "\n", - "Enterprise ontologies are living artifacts. Semantica provides tools to manage their entire lifecycle and accelerate creation with AI.\n", - "\n", - "### 1. Text-to-Ontology (LLM Integration)\n", - "\n", - "Instead of manually creating entities, use the `LLMOntologyGenerator` to extract an ontology directly from text requirements or documents." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import LLMOntologyGenerator\n", - "\n", - "try:\n", - " # Note: Requires an API key in your environment variables\n", - " llm_gen = LLMOntologyGenerator(provider=\"openai\", model=\"gpt-4\")\n", - "\n", - " text_description = \"\"\"\n", - " A University has many Departments. Each Department offers several Courses.\n", - " Professors teach Courses and belong to a Department.\n", - " Students enroll in Courses.\n", - " \"\"\"\n", - "\n", - " llm_ontology = llm_gen.generate_ontology_from_text(\n", - " text=text_description,\n", - " name=\"UniversityOntology\"\n", - " )\n", - "\n", - " print(\"Generated Classes:\", [c['name'] for c in llm_ontology['classes']])\n", - "except Exception:\n", - " print(\"Skipping LLM generation: No API key or provider configured in this environment.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2. Test-Driven Design (Competency Questions)\n", - "\n", - "Formalize your requirements as \"Competency Questions\" (CQs). The `CompetencyQuestionsManager` can check if your ontology contains the necessary terms to answer them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import CompetencyQuestionsManager\n", - "\n", - "cq_manager = CompetencyQuestionsManager()\n", - "\n", - "# Define what our ontology SHOULD answer\n", - "cq_manager.add_question(\"Who leads TechCorp?\", category=\"organizational\")\n", - "cq_manager.add_question(\"Which projects does Bob manage?\", category=\"operational\")\n", - "\n", - "# Validate our 'ontology' against these questions\n", - "validation_results = cq_manager.validate_ontology(ontology)\n", - "\n", - "print(f\"Answerable Questions: {validation_results['answerable']} / {validation_results['total_questions']}\")\n", - "for q in cq_manager.questions:\n", - " status = \"✅\" if q.answerable else \"❌\"\n", - " print(f\"{status} {q.question}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3. Lifecycle Management (Versioning & Reuse)\n", - "\n", - "Manage iterations with `VersionManager` and import external standards like FOAF or Dublin Core with `ReuseManager`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ontology import VersionManager, ReuseManager\n", - "\n", - "# --- Versioning ---\n", - "v_manager = VersionManager(base_uri=\"https://example.org/ontology/\")\n", - "v1 = v_manager.create_version(\"1.0\", ontology, changes=[\"Initial creation\"])\n", - "print(f\"Created Version: {v1.version} at {v1.ontology_iri}\")\n", - "\n", - "# --- Reuse ---\n", - "reuse_manager = ReuseManager()\n", - "\n", - "# Check if we can reuse FOAF\n", - "foaf_info = reuse_manager.research_ontology(\"http://xmlns.com/foaf/0.1/\")\n", - "if foaf_info:\n", - " print(f\"Found standard ontology: {foaf_info['name']}\")\n", - " # We could now import this into our ontology\n", - " ontology['imports'].append(foaf_info['uri'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Exporting Your Ontology\n", - "\n", - "Once your ontology is built and validated, you'll want to save it. Semantica focuses on **Turtle (`.ttl`)** as the primary format, but supports others via `rdflib`.\n", - "\n", - "You can export to a string or directly to a file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Get Turtle string representation\n", - "ttl_output = engine.to_owl(ontology, format=\"turtle\")\n", - "\n", - "print(\"--- Turtle Preview (First 500 chars) ---\")\n", - "print(ttl_output[:500])\n", - "print(\"...\")\n", - "\n", - "# Save to file\n", - "output_path = \"corporate_ontology.ttl\"\n", - "engine.export_owl(ontology, path=output_path, format=\"turtle\")\n", - "print(f\"Successfully saved ontology to {output_path}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "You have now mastered the essentials of Semantica's Ontology Module!\n", - "\n", - "* **Automated Generation**: Used the 6-stage pipeline to go from raw data to a structured ontology.\n", - "* **Component Control**: Used `ClassInferrer` and `PropertyGenerator` for fine-tuned modeling.\n", - "* **Quality Assurance**: Validated your model against strict standards.\n", - "* **Visualization**: Explored the ontology structure interactively.\n", - "* **Advanced Lifecycle**: Used AI generation, competency questions, and versioning.\n", - "* **Export**: Serialized your knowledge graph for use in other semantic web tools.\n", - "\n", - "**Next Steps**:\n", - "* Try customizing the `NamespaceManager` to use your organization's URL.\n", - "* Explore `OntologyEvaluator` for deeper quality metrics.\n", - "* Feed the generated ontology into the **Knowledge Graph** module to start reasoning over your data!" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)\n", + "\n", + "# Mastering Ontology Generation with Semantica\n", + "\n", + "Welcome to the comprehensive guide on Semantica's Ontology Module. This module is the powerhouse for structuring your data into meaningful knowledge graphs, providing a complete 6-stage pipeline from raw data to validated OWL ontologies.\n", + "\n", + "In this notebook, we will dive deep into:\n", + "1. **The 6-Stage Generation Pipeline**: Understanding how Semantica transforms data into knowledge.\n", + "2. **Core Components in Focus**: Detailed usage of `ClassInferrer`, `PropertyGenerator`, and `OntologyOptimizer`.\n", + "3. **Validation & Quality**: ensuring your ontology is consistent and structurally sound.\n", + "4. **Visualize**: exploring your ontology with interactive charts and hierarchies.\n", + "5. **Advanced Usage**: Text-to-Ontology (LLM), Competency Questions, and Lifecycle Management.\n", + "6. **Exporting & Interoperability**: Saving your work in standard formats like Turtle and RDF/XML.\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/ontology/)\n", + "\n", + "## Getting Started\n", + "\n", + "First, let's setup our environment and initialize the `OntologyEngine`. This engine is the unified entry point for all ontology operations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install semantica if not already installed\n", + "# !pip install semantica\n", + "# !pip install plotly # Required for visualization\n", + "\n", + "from semantica.ontology import OntologyEngine, OntologyGenerator\n", + "from semantica.utils.logging import get_logger\n", + "\n", + "# Initialize logger for visibility\n", + "logger = get_logger(\"ontology_guide\")\n", + "\n", + "# Initialize the Engine\n", + "# base_uri defines the namespace root for your ontology\n", + "engine = OntologyEngine(base_uri=\"https://docs.semantica.dev/ontology/\")\n", + "\n", + "print(\"Ontology Engine initialized successfully!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## The 6-Stage Generation Pipeline\n", + "\n", + "Semantica uses a sophisticated 6-stage pipeline to robustly generate ontologies. This automated process takes raw entity and relationship data and produces a high-quality OWL ontology.\n", + "\n", + "### The Stages:\n", + "1. **Semantic Network Parsing**: Extracts raw concepts and connections from your inputs.\n", + "2. **YAML-to-Definition**: Transforms concepts into structured class definitions.\n", + "3. **Definition-to-Types**: Maps definitions to formal OWL types (e.g., `owl:Class`, `owl:ObjectProperty`).\n", + "4. **Hierarchy Generation**: Builds a taxonomic structure (parent-child relationships) using `associatedWith` or linguistic patterns.\n", + "5. **TTL Generation**: Serializes the in-memory structure into Turtle format logic.\n", + "6. **Symbolic Validation**: Validates the result using reasoners like HermiT (if available) or structural checks.\n", + "\n", + "Let's see this in action with some sample data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample Data: A simple corporate structure\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Company\", \"name\": \"TechCorp\", \"founded\": \"2010\"},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Alice\", \"role\": \"CEO\"},\n", + " {\"id\": \"e3\", \"type\": \"Person\", \"name\": \"Bob\", \"role\": \"CTO\"},\n", + " {\"id\": \"e4\", \"type\": \"Department\", \"name\": \"Engineering\"},\n", + " {\"id\": \"e5\", \"type\": \"Project\", \"name\": \"Project Phoenix\"}\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e2\", \"target\": \"e1\", \"type\": \"leads\"},\n", + " {\"source\": \"e3\", \"target\": \"e4\", \"type\": \"manages\"},\n", + " {\"source\": \"e4\", \"target\": \"e1\", \"type\": \"part_of\"},\n", + " {\"source\": \"e3\", \"target\": \"e5\", \"type\": \"works_on\"}\n", + "]\n", + "\n", + "data = {\n", + " \"entities\": entities,\n", + " \"relationships\": relationships\n", + "}\n", + "\n", + "# Run the full pipeline\n", + "ontology = engine.from_data(data, name=\"CorporateOntology\")\n", + "\n", + "print(f\"Generated Ontology: {ontology['name']}\")\n", + "print(f\"Classes Found: {len(ontology['classes'])}\")\n", + "print(f\"Properties Found: {len(ontology['properties'])}\")\n", + "print(f\"Validation Status: Valid={ontology.get('validation_result', {}).get('valid', 'Unknown')}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Inspecting the Results\n", + "\n", + "The generated `ontology` object is a rich dictionary containing all the inferred structure. Let's peek inside to see what Classes and Properties were created." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect Classes\n", + "print(\"--- Inferred Classes ---\")\n", + "for cls in ontology['classes']:\n", + " print(f\"Class: {cls['name']}\")\n", + " print(f\" URI: {cls.get('uri')}\")\n", + " # Check if a hierarchy was inferred\n", + " if cls.get('subClassOf'):\n", + " print(f\" Parent: {cls['subClassOf']}\")\n", + " print(\"\")\n", + "\n", + "# Inspect Properties\n", + "print(\"--- Inferred Properties ---\")\n", + "for prop in ontology['properties']:\n", + " type_label = \"Object Property\" if prop['type'] == 'object' else \"Data Property\"\n", + " print(f\"{prop['name']} [{type_label}]\")\n", + " print(f\" Domain: {prop.get('domain')}\")\n", + " print(f\" Range: {prop.get('range')}\")\n", + " print(\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Deep Dive: Component by Component\n", + "\n", + "While `OntologyEngine` is great for one-shot generation, you often need fine-grained control. Let's look at the individual tools that power the engine.\n", + "\n", + "### 1. `ClassInferrer`: Mastering Class Discovery\n", + "\n", + "The `ClassInferrer` analyzes entities to find patterns. It can handle noise and only creates classes for types that appear frequently enough.\n", + "\n", + "* **`min_occurrences`**: Ignores types with fewer entities than this count.\n", + "* **`build_class_hierarchy`**: Toggles automatic parent-child detection.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import ClassInferrer\n", + "\n", + "# Initialize inferrer with a threshold\n", + "# We set min_occurrences=1 here to capture everything in our small example\n", + "inferrer = ClassInferrer(min_occurrences=1)\n", + "\n", + "raw_entities = [\n", + " {\"type\": \"Manager\", \"name\": \"Dave\", \"level\": 5},\n", + " {\"type\": \"Manager\", \"name\": \"Eve\", \"level\": 4},\n", + " {\"type\": \"Employee\", \"name\": \"Frank\"}, # Only 1 employee\n", + " {\"type\": \"TemporaryWorker\", \"name\": \"Grace\"} \n", + "]\n", + "\n", + "# Infer classes\n", + "classes = inferrer.infer_classes(raw_entities, build_hierarchy=True)\n", + "\n", + "print(f\"Inferred {len(classes)} classes from raw entities.\")\n", + "for c in classes:\n", + " print(f\"- {c['name']} (Count: {c['entity_count']})\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. `PropertyGenerator`: The Glue of the Ontology\n", + "\n", + "Properties define relationships. Semantica distinguishes between:\n", + "* **Object Properties**: Links between two entities (e.g., `leads` between Person and Company).\n", + "* **Data Properties**: Attributes of an entity (e.g., `founded` year of a Company).\n", + "\n", + "The `PropertyGenerator` automatically detects this distinction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import PropertyGenerator\n", + "\n", + "prop_gen = PropertyGenerator()\n", + "\n", + "# We need the classes first to help property generation context\n", + "context_classes = classes # reusing from previous step\n", + "\n", + "# Let's define some relationships and attributes implicitly via entities\n", + "# Note: 'level' in Manager entities is a potential data property\n", + "complex_entities = [\n", + " {\"id\": \"m1\", \"type\": \"Manager\", \"name\": \"Dave\", \"level\": 5},\n", + " {\"id\": \"e1\", \"type\": \"Employee\", \"name\": \"Frank\"}\n", + "]\n", + "complex_relationships = [\n", + " {\"source\": \"m1\", \"target\": \"e1\", \"type\": \"supervises\"} # Object property\n", + "]\n", + "\n", + "properties = prop_gen.infer_properties(\n", + " entities=complex_entities,\n", + " relationships=complex_relationships,\n", + " classes=context_classes\n", + ")\n", + "\n", + "print(\"--- Property Types Identified ---\")\n", + "for p in properties:\n", + " print(f\"Property: {p['name']}\")\n", + " print(f\" Type: {p['type']}\")\n", + " print(f\" Domain: {p['domain']} -> Range: {p['range']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. `OntologyOptimizer`: Refining the Structure\n", + "\n", + "Before finalizing, it's good practice to optimize. The optimizer removes redundancies and improves coherence, such as ensuring all classes have proper labels and valid URIs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import OntologyOptimizer\n", + "\n", + "optimizer = OntologyOptimizer()\n", + "\n", + "# Let's pretend we have a messy ontology dict\n", + "messy_ontology = {\n", + " \"classes\": [\n", + " {\"name\": \"Person\", \"uri\": \"...Person\"},\n", + " {\"name\": \"Person\", \"uri\": \"...Person\"} # Duplicate!\n", + " ],\n", + " \"properties\": []\n", + "}\n", + "\n", + "clean_ontology = optimizer.optimize_ontology(messy_ontology, remove_redundancy=True)\n", + "\n", + "print(f\"Original Classes: {len(messy_ontology['classes'])}\")\n", + "print(f\"Optimized Classes: {len(clean_ontology['classes'])}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Validation and Quality Control\n", + "\n", + "Semantica includes a robust `OntologyValidator`. It checks for:\n", + "1. **Structure**: Missing fields, malformed URIs.\n", + "2. **Consistency**: Circular hierarchies, contradictory definitions.\n", + "3. **Metrics**: Depth of hierarchy, property usage.\n", + "\n", + "If you have `Owlready2` installed, it can even run a reasoner (HermiT or Pellet) to prove logical consistency." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import OntologyValidator\n", + "\n", + "validator = OntologyValidator(\n", + " check_consistency=True,\n", + " check_satisfiability=True\n", + ")\n", + "\n", + "# Validate our previously generated 'ontology'\n", + "result = validator.validate_ontology(ontology)\n", + "\n", + "print(f\"Is Valid? {result.valid}\")\n", + "print(f\"Is Consistent? {result.consistent}\")\n", + "\n", + "if result.errors:\n", + " print(\"Errors Found:\", result.errors)\n", + "if result.warnings:\n", + " print(\"Warnings:\", result.warnings)\n", + " \n", + "# Check Metrics\n", + "print(\"Metrics:\", result.metrics)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Visualization\n", + "\n", + "A picture is worth a thousand triples! The `OntologyVisualizer` lets you explore your ontology's structure interactively.\n", + "\n", + "We can visualize:\n", + "* **Class Hierarchies**: Tree diagrams of class inheritance.\n", + "* **Structure Networks**: The full graph of classes and properties.\n", + "* **Metrics Dashboards**: High-level stats at a glance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import OntologyVisualizer\n", + "\n", + "viz = OntologyVisualizer()\n", + "\n", + "# 1. Interactive Class Hierarchy\n", + "# Returns a Plotly figure you can interact with\n", + "fig_hierarchy = viz.visualize_hierarchy(ontology, output=\"interactive\")\n", + "if fig_hierarchy:\n", + " fig_hierarchy.show()\n", + "\n", + "# 2. Ontology Structure Network\n", + "# See how classes and properties connect\n", + "fig_structure = viz.visualize_structure(ontology, output=\"interactive\")\n", + "if fig_structure:\n", + " fig_structure.show()\n", + "\n", + "# 3. Metrics Dashboard\n", + "# View counts, depths, and statistics\n", + "fig_metrics = viz.visualize_metrics(ontology, output=\"interactive\")\n", + "if fig_metrics:\n", + " fig_metrics.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Advanced Usage: Lifecycle & AI\n", + "\n", + "Enterprise ontologies are living artifacts. Semantica provides tools to manage their entire lifecycle and accelerate creation with AI.\n", + "\n", + "### 1. Text-to-Ontology (LLM Integration)\n", + "\n", + "Instead of manually creating entities, use the `LLMOntologyGenerator` to extract an ontology directly from text requirements or documents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import LLMOntologyGenerator\n", + "\n", + "try:\n", + " # Note: Requires an API key in your environment variables\n", + " llm_gen = LLMOntologyGenerator(provider=\"openai\", model=\"gpt-4\")\n", + "\n", + " text_description = \"\"\"\n", + " A University has many Departments. Each Department offers several Courses.\n", + " Professors teach Courses and belong to a Department.\n", + " Students enroll in Courses.\n", + " \"\"\"\n", + "\n", + " llm_ontology = llm_gen.generate_ontology_from_text(\n", + " text=text_description,\n", + " name=\"UniversityOntology\"\n", + " )\n", + "\n", + " print(\"Generated Classes:\", [c['name'] for c in llm_ontology['classes']])\n", + "except Exception:\n", + " print(\"Skipping LLM generation: No API key or provider configured in this environment.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Test-Driven Design (Competency Questions)\n", + "\n", + "Formalize your requirements as \"Competency Questions\" (CQs). The `CompetencyQuestionsManager` can check if your ontology contains the necessary terms to answer them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import CompetencyQuestionsManager\n", + "\n", + "cq_manager = CompetencyQuestionsManager()\n", + "\n", + "# Define what our ontology SHOULD answer\n", + "cq_manager.add_question(\"Who leads TechCorp?\", category=\"organizational\")\n", + "cq_manager.add_question(\"Which projects does Bob manage?\", category=\"operational\")\n", + "\n", + "# Validate our 'ontology' against these questions\n", + "validation_results = cq_manager.validate_ontology(ontology)\n", + "\n", + "print(f\"Answerable Questions: {validation_results['answerable']} / {validation_results['total_questions']}\")\n", + "for q in cq_manager.questions:\n", + " status = \"\u2705\" if q.answerable else \"\u274c\"\n", + " print(f\"{status} {q.question}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Lifecycle Management (Versioning & Reuse)\n", + "\n", + "Manage iterations with `VersionManager` and import external standards like FOAF or Dublin Core with `ReuseManager`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import VersionManager, ReuseManager\n", + "\n", + "# --- Versioning ---\n", + "v_manager = VersionManager(base_uri=\"https://example.org/ontology/\")\n", + "v1 = v_manager.create_version(\"1.0\", ontology, changes=[\"Initial creation\"])\n", + "print(f\"Created Version: {v1.version} at {v1.ontology_iri}\")\n", + "\n", + "# --- Reuse ---\n", + "reuse_manager = ReuseManager()\n", + "\n", + "# Check if we can reuse FOAF\n", + "foaf_info = reuse_manager.research_ontology(\"http://xmlns.com/foaf/0.1/\")\n", + "if foaf_info:\n", + " print(f\"Found standard ontology: {foaf_info['name']}\")\n", + " # We could now import this into our ontology\n", + " ontology['imports'].append(foaf_info['uri'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## Exporting Your Ontology\n", + "\n", + "Once your ontology is built and validated, you'll want to save it. Semantica focuses on **Turtle (`.ttl`)** as the primary format, but supports others via `rdflib`.\n", + "\n", + "You can export to a string or directly to a file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get Turtle string representation\n", + "ttl_output = engine.to_owl(ontology, format=\"turtle\")\n", + "\n", + "print(\"--- Turtle Preview (First 500 chars) ---\")\n", + "print(ttl_output[:500])\n", + "print(\"...\")\n", + "\n", + "# Save to file\n", + "output_path = \"corporate_ontology.ttl\"\n", + "engine.export_owl(ontology, path=output_path, format=\"turtle\")\n", + "print(f\"Successfully saved ontology to {output_path}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You have now mastered the essentials of Semantica's Ontology Module!\n", + "\n", + "* **Automated Generation**: Used the 6-stage pipeline to go from raw data to a structured ontology.\n", + "* **Component Control**: Used `ClassInferrer` and `PropertyGenerator` for fine-tuned modeling.\n", + "* **Quality Assurance**: Validated your model against strict standards.\n", + "* **Visualization**: Explored the ontology structure interactively.\n", + "* **Advanced Lifecycle**: Used AI generation, competency questions, and versioning.\n", + "* **Export**: Serialized your knowledge graph for use in other semantic web tools.\n", + "\n", + "**Next Steps**:\n", + "* Try customizing the `NamespaceManager` to use your organization's URL.\n", + "* Explore `OntologyEvaluator` for deeper quality metrics.\n", + "* Feed the generated ontology into the **Knowledge Graph** module to start reasoning over your data!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/cookbook/introduction/15_Export.ipynb b/cookbook/introduction/15_Export.ipynb index 3f85667c..1ea63410 100644 --- a/cookbook/introduction/15_Export.ipynb +++ b/cookbook/introduction/15_Export.ipynb @@ -75,24 +75,22 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\\n", - "import semantica\\n", - "print(semantica.__version__)\\n" - ] + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.export import JSONExporter\n", - "from semantica.kg import GraphBuilder\n", + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import JSONExporter\n", + "from semantica.kg import GraphBuilder\n", "\n", "# Create exporter and builder\n", "json_exporter = JSONExporter()\n", @@ -365,4 +363,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/16_Visualization.ipynb b/cookbook/introduction/16_Visualization.ipynb index 8b06ad4f..12e5902e 100644 --- a/cookbook/introduction/16_Visualization.ipynb +++ b/cookbook/introduction/16_Visualization.ipynb @@ -43,9 +43,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -216,4 +214,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/17_Conflict_Detection.ipynb b/cookbook/introduction/17_Conflict_Detection.ipynb index f08b78d5..67e664d0 100644 --- a/cookbook/introduction/17_Conflict_Detection.ipynb +++ b/cookbook/introduction/17_Conflict_Detection.ipynb @@ -38,24 +38,22 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\\n", - "import semantica\\n", - "print(semantica.__version__)\\n" - ] + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.conflicts import ConflictDetector\n", - "from datetime import datetime\n", + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.conflicts import ConflictDetector\n", + "from datetime import datetime\n", "\n", "# Initialize detector with configuration\n", "detector = ConflictDetector(\n", @@ -449,7 +447,7 @@ "\n", "**Method-Specific:** Voting (`min_sources`, `tie_breaker`), credibility-weighted (`min_credibility`), most_recent (`time_field`), etc.\n", "\n", - "**Priority:** Method-specific → Global → Environment variables → Defaults\n", + "**Priority:** Method-specific \u2192 Global \u2192 Environment variables \u2192 Defaults\n", "\n", "**Best Practices:** Set source credibility early, configure conflict fields, use method-specific configs\n" ] @@ -487,7 +485,7 @@ "\n", "End-to-end example: integrating company data from multiple sources.\n", "\n", - "**Workflow:** Initialize → Track Sources → Detect → Resolve → Analyze → Generate Guides → Build Final Entity\n", + "**Workflow:** Initialize \u2192 Track Sources \u2192 Detect \u2192 Resolve \u2192 Analyze \u2192 Generate Guides \u2192 Build Final Entity\n", "\n", "**Scenario:** Three sources (Wikipedia, Official Site, Financial DB) with conflicts in name, founding year, and type classifications.\n" ] @@ -571,14 +569,14 @@ "\n", "### Key Features\n", "\n", - "✅ **Detection**: Value, type, temporal, logical, relationship conflicts \n", - "✅ **Resolution**: 6 strategies (voting, credibility-weighted, most_recent, first_seen, highest_confidence, manual_review) \n", - "✅ **Source Tracking**: Provenance, credibility, traceability chains \n", - "✅ **Analysis**: Patterns, trends, recommendations \n", - "✅ **Investigation Guides**: Automated guides and checklists \n", - "✅ **Methods Module**: Convenience functions for all operations \n", - "✅ **Method Registry**: Custom method registration \n", - "✅ **Configuration**: Global and method-specific settings\n", + "\u2705 **Detection**: Value, type, temporal, logical, relationship conflicts \n", + "\u2705 **Resolution**: 6 strategies (voting, credibility-weighted, most_recent, first_seen, highest_confidence, manual_review) \n", + "\u2705 **Source Tracking**: Provenance, credibility, traceability chains \n", + "\u2705 **Analysis**: Patterns, trends, recommendations \n", + "\u2705 **Investigation Guides**: Automated guides and checklists \n", + "\u2705 **Methods Module**: Convenience functions for all operations \n", + "\u2705 **Method Registry**: Custom method registration \n", + "\u2705 **Configuration**: Global and method-specific settings\n", "\n", "### Best Practices\n", "\n", @@ -591,10 +589,10 @@ "\n", "### Common Patterns\n", "\n", - "- **Integration**: Track → Detect → Resolve → Analyze\n", - "- **QA**: Detect → Analyze → Generate guides → Review\n", - "- **Auto**: Detect → Resolve → Analyze\n", - "- **Assessment**: Track → Analyze → Adjust credibility\n", + "- **Integration**: Track \u2192 Detect \u2192 Resolve \u2192 Analyze\n", + "- **QA**: Detect \u2192 Analyze \u2192 Generate guides \u2192 Review\n", + "- **Auto**: Detect \u2192 Resolve \u2192 Analyze\n", + "- **Assessment**: Track \u2192 Analyze \u2192 Adjust credibility\n", "\n", "### Next Steps\n", "\n", @@ -612,4 +610,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/18_Deduplication.ipynb b/cookbook/introduction/18_Deduplication.ipynb index 812b0a1f..5d99b1ae 100644 --- a/cookbook/introduction/18_Deduplication.ipynb +++ b/cookbook/introduction/18_Deduplication.ipynb @@ -94,10 +94,10 @@ "- Multi-factor Aggregation: Weighted sum of similarity components\n", "\n", "**Duplicate Detection:**\n", - "- Pairwise Comparison: O(n²) all-pairs similarity calculation\n", + "- Pairwise Comparison: O(n\u00b2) all-pairs similarity calculation\n", "- Union-Find Algorithm: Disjoint set union for group formation\n", "- Confidence Scoring: Multi-factor confidence calculation\n", - "- Incremental Processing: O(n×m) efficient new vs existing comparison\n", + "- Incremental Processing: O(n\u00d7m) efficient new vs existing comparison\n", "\n", "**Clustering:**\n", "- Union-Find (DSU): Connected component detection\n", @@ -137,24 +137,22 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\\n", - "import semantica\\n", - "print(semantica.__version__)\\n" - ] + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import all deduplication classes\n", - "from semantica.deduplication import (\n", + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import all deduplication classes\n", + "from semantica.deduplication import (\n", " # Main Classes\n", " DuplicateDetector,\n", " EntityMerger,\n", @@ -299,9 +297,9 @@ "\n", "### Detection Methods\n", "\n", - "1. **Pairwise Detection**: Compare all entity pairs (O(n²) complexity)\n", + "1. **Pairwise Detection**: Compare all entity pairs (O(n\u00b2) complexity)\n", "2. **Group Detection**: Find clusters of duplicates using Union-Find algorithm\n", - "3. **Incremental Detection**: Efficiently detect duplicates between new and existing entities (O(n×m))\n", + "3. **Incremental Detection**: Efficiently detect duplicates between new and existing entities (O(n\u00d7m))\n", "4. **Relationship Detection**: Identify duplicate relationships\n", "\n", "### Confidence Scoring\n", @@ -412,7 +410,7 @@ "print(f\"Original entities: {len(entities)}\")\n", "print(f\"Merge operations: {len(merge_operations)}\")\n", "for i, op in enumerate(merge_operations, 1):\n", - " print(f\" Operation {i}: Merged {len(op.source_entities)} entities → {op.merged_entity.get('name')}\")\n", + " print(f\" Operation {i}: Merged {len(op.source_entities)} entities \u2192 {op.merged_entity.get('name')}\")\n", " if op.merge_result.conflicts:\n", " print(f\" Conflicts: {len(op.merge_result.conflicts)}\")\n", "\n", @@ -423,7 +421,7 @@ "# Merge specific group\n", "duplicate_entities = [entities[0], entities[1]]\n", "operation = merger.merge_entity_group(duplicate_entities, strategy=MergeStrategy.KEEP_FIRST)\n", - "print(f\"\\nMerged group: {[e['name'] for e in operation.source_entities]} → {operation.merged_entity['name']}\")\n", + "print(f\"\\nMerged group: {[e['name'] for e in operation.source_entities]} \u2192 {operation.merged_entity['name']}\")\n", "\n", "# Get merge history\n", "history = merger.get_merge_history()\n", @@ -766,4 +764,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/introduction/19_Context_Module.ipynb b/cookbook/introduction/19_Context_Module.ipynb index 926ea76a..f9d8b789 100644 --- a/cookbook/introduction/19_Context_Module.ipynb +++ b/cookbook/introduction/19_Context_Module.ipynb @@ -32,25 +32,23 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\\n", - "import semantica\\n", - "print(semantica.__version__)\\n" - ] + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Setup: Create a mock vector store for demonstration\n", - "from typing import List, Dict, Any, Optional\n", - "from semantica.context import VectorStore\n", + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Setup: Create a mock vector store for demonstration\n", + "from typing import List, Dict, Any, Optional\n", + "from semantica.context import VectorStore\n", "\n", "class MockVectorStore(VectorStore):\n", " def __init__(self):\n", @@ -287,4 +285,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/cookbook/introduction/20_Triplet_Store.ipynb b/cookbook/introduction/20_Triplet_Store.ipynb index 6dfcdded..81bbb4b1 100644 --- a/cookbook/introduction/20_Triplet_Store.ipynb +++ b/cookbook/introduction/20_Triplet_Store.ipynb @@ -1,785 +1,783 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/20_Triple_Store.ipynb)\n", - "\n", - "# Triplet Store - Comprehensive Guide\n", - "\n", - "## Overview\n", - "\n", - "This notebook provides a **comprehensive walkthrough** of Semantica's triplet_store module, demonstrating RDF triplet storage, SPARQL querying, and multi-backend support for knowledge graph persistence.\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/triplet_store/)\n", - "\n", - "### Learning Objectives\n", - "\n", - "By the end of this notebook, you will be able to:\n", - "\n", - "- Register and manage triplet stores (Blazegraph, Jena, RDF4J, Virtuoso)\n", - "- Perform CRUD operations on RDF triplets\n", - "- Execute SPARQL queries with optimization\n", - "- Use bulk loading for large datasets\n", - "- Work with multiple store backends\n", - "- Validate and track triplet operations\n", - "- Choose the right backend for your use case\n", - "\n", - "### What You'll Learn\n", - "\n", - "| Component | Purpose | When to Use |\n", - "|-----------|---------|-------------|\n", - "| `TripletManager` | Store coordination | All triplet operations |\n", - "| `QueryEngine` | SPARQL execution | Query optimization |\n", - "| `BulkLoader` | High-volume loading | Large datasets |\n", - "| `BlazegraphAdapter` | Blazegraph backend | High performance |\n", - "| `JenaAdapter` | Jena backend | Java integration |\n", - "| `RDF4JAdapter` | RDF4J backend | Transaction support |\n", - "| `VirtuosoAdapter` | Virtuoso backend | Enterprise scale |\n", - "\n", - "---\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica from PyPI:\n", - "\n", - "```bash\n", - "pip install semantica\n", - "# Or with all optional dependencies:\n", - "pip install semantica[all]\n", - "```\n", - "\n", - "---" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Basic Triplet Store Operations\n", - "\n", - "Let's start with the `TripletManager` for basic triplet store operations.\n", - "\n", - "### What is TripletManager?\n", - "\n", - "`TripletManager` is the main coordinator for triplet store operations:\n", - "- **Store Registration**: Register multiple backends\n", - "- **CRUD Operations**: Add, get, update, delete triples\n", - "- **Multi-Store**: Manage multiple stores simultaneously" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import TripletManager\n", - "from semantica.semantic_extract.triple_extractor import Triple\n", - "\n", - "# Create triple manager\n", - "manager = TripletManager()\n", - "\n", - "# Register a Blazegraph store (in-memory for demo)\n", - "store = manager.register_store(\n", - " store_id=\"demo\",\n", - " store_type=\"blazegraph\",\n", - " endpoint=\"http://localhost:9999/blazegraph/sparql\"\n", - ")\n", - "\n", - "print(f\"Registered store: {store.store_id}\")\n", - "print(f\"Store type: {store.store_type}\")\n", - "print(f\"Endpoint: {store.endpoint}\")\n", - "\n", - "# Create a triple\n", - "triple = Triple(\n", - " subject=\"http://example.org/Alice\",\n", - " predicate=\"http://example.org/knows\",\n", - " object=\"http://example.org/Bob\",\n", - " confidence=0.95\n", - ")\n", - "\n", - "# Add triple to store\n", - "result = manager.add_triple(triple, store_id=\"demo\")\n", - "print(f\"\\nTriple added: {result['success']}\")\n", - "print(f\"Triple: {triple.subject} -> {triple.predicate} -> {triple.object}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Store Registration and Management\n", - "\n", - "Register multiple stores and manage them.\n", - "\n", - "### Supported Backends\n", - "\n", - "| Backend | Best For | Performance | Features |\n", - "|---------|----------|-------------|----------|\n", - "| **Blazegraph** | Large datasets | Excellent | GPU acceleration, full-text |\n", - "| **Jena** | Java apps | Good | SHACL, inference |\n", - "| **RDF4J** | Transactions | Good | ACID, federation |\n", - "| **Virtuoso** | Enterprise | Excellent | SQL integration, clustering |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import register_store\n", - "\n", - "# Register multiple stores using convenience function\n", - "blazegraph_store = register_store(\n", - " \"blazegraph_main\",\n", - " \"blazegraph\",\n", - " \"http://localhost:9999/blazegraph/sparql\"\n", - ")\n", - "\n", - "jena_store = register_store(\n", - " \"jena_backup\",\n", - " \"jena\",\n", - " \"http://localhost:3030/ds\"\n", - ")\n", - "\n", - "# List all registered stores\n", - "stores = manager.list_stores()\n", - "print(f\"Registered stores: {stores}\")\n", - "\n", - "# Get specific store\n", - "store = manager.get_store(\"blazegraph_main\")\n", - "print(f\"\\nStore details:\")\n", - "print(f\" ID: {store.store_id}\")\n", - "print(f\" Type: {store.store_type}\")\n", - "print(f\" Endpoint: {store.endpoint}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: CRUD Operations\n", - "\n", - "Perform Create, Read, Update, Delete operations on triples.\n", - "\n", - "### Operations Overview\n", - "\n", - "- **Create**: `add_triple()`, `add_triples()`\n", - "- **Read**: `get_triple()`\n", - "- **Update**: `update_triple()`\n", - "- **Delete**: `delete_triple()`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import add_triple, add_triples, get_triples, update_triple, delete_triple\n", - "\n", - "# Create - Add single triple\n", - "triple1 = Triple(\n", - " subject=\"http://example.org/Alice\",\n", - " predicate=\"http://example.org/hasAge\",\n", - " object=\"30\"\n", - ")\n", - "result = add_triple(triple1, store_id=\"demo\")\n", - "print(f\"Added single triple: {result['success']}\")\n", - "\n", - "# Create - Add multiple triples\n", - "triples = [\n", - " Triple(\"http://example.org/Alice\", \"http://example.org/hasCity\", \"New York\"),\n", - " Triple(\"http://example.org/Bob\", \"http://example.org/hasAge\", \"25\"),\n", - " Triple(\"http://example.org/Bob\", \"http://example.org/hasCity\", \"Boston\")\n", - "]\n", - "result = add_triples(triples, store_id=\"demo\")\n", - "print(f\"\\nAdded {result['total_triples']} triples in {result['batches']} batches\")\n", - "\n", - "# Read - Get triples for a subject\n", - "alice_triples = get_triples(\n", - " subject=\"http://example.org/Alice\",\n", - " store_id=\"demo\"\n", - ")\n", - "print(f\"\\nFound {len(alice_triples)} triples for Alice\")\n", - "\n", - "# Update - Change Alice's age\n", - "old_triple = Triple(\"http://example.org/Alice\", \"http://example.org/hasAge\", \"30\")\n", - "new_triple = Triple(\"http://example.org/Alice\", \"http://example.org/hasAge\", \"31\")\n", - "result = update_triple(old_triple, new_triple, store_id=\"demo\")\n", - "print(f\"\\nUpdated triple: {result['success']}\")\n", - "\n", - "# Delete - Remove a triple\n", - "triple_to_delete = Triple(\"http://example.org/Bob\", \"http://example.org/hasCity\", \"Boston\")\n", - "result = delete_triple(triple_to_delete, store_id=\"demo\")\n", - "print(f\"Deleted triple: {result['success']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: SPARQL Query Execution\n", - "\n", - "Execute SPARQL queries with the QueryEngine.\n", - "\n", - "### Query Types\n", - "\n", - "- **SELECT**: Retrieve variable bindings\n", - "- **ASK**: Boolean queries\n", - "- **CONSTRUCT**: Build RDF graphs\n", - "- **DESCRIBE**: Describe resources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import QueryEngine, BlazegraphAdapter\n", - "\n", - "# Create query engine with caching\n", - "engine = QueryEngine(enable_caching=True, enable_optimization=True)\n", - "\n", - "# Create adapter\n", - "adapter = BlazegraphAdapter(endpoint=\"http://localhost:9999/blazegraph/sparql\")\n", - "\n", - "# SELECT query\n", - "select_query = \"\"\"\n", - "PREFIX ex: \n", - "\n", - "SELECT ?person ?age ?city\n", - "WHERE {\n", - " ?person ex:hasAge ?age .\n", - " ?person ex:hasCity ?city .\n", - "}\n", - "ORDER BY DESC(?age)\n", - "LIMIT 10\n", - "\"\"\"\n", - "\n", - "result = engine.execute_query(select_query, adapter)\n", - "\n", - "print(f\"Query Results:\")\n", - "print(f\" Variables: {result.variables}\")\n", - "print(f\" Results: {len(result.bindings)}\")\n", - "print(f\" Execution time: {result.execution_time:.2f}s\")\n", - "print(f\" Cached: {result.metadata.get('cached', False)}\")\n", - "\n", - "print(\"\\nResults:\")\n", - "for binding in result.bindings:\n", - " person = binding.get('person', {}).get('value', '')\n", - " age = binding.get('age', {}).get('value', '')\n", - " city = binding.get('city', {}).get('value', '')\n", - " print(f\" {person}: Age {age}, City {city}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Query Optimization\n", - "\n", - "Optimize SPARQL queries for better performance.\n", - "\n", - "### Optimization Features\n", - "\n", - "- **Query Planning**: Analyze execution steps\n", - "- **Cost Estimation**: Estimate query cost\n", - "- **Query Rewriting**: Optimize query structure\n", - "- **Caching**: Cache query results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import optimize_query, plan_query\n", - "\n", - "# Original query\n", - "query = \"\"\"\n", - "SELECT ?s ?p ?o\n", - "WHERE {\n", - " ?s ?p ?o .\n", - "}\n", - "\"\"\"\n", - "\n", - "# Optimize query (adds LIMIT if missing)\n", - "optimized = optimize_query(query, add_limit=True, default_limit=1000)\n", - "print(\"Optimized Query:\")\n", - "print(optimized)\n", - "\n", - "# Create query plan\n", - "plan = plan_query(query)\n", - "print(f\"\\nQuery Plan:\")\n", - "print(f\" Original length: {len(plan.query)}\")\n", - "print(f\" Optimized length: {len(plan.optimized_query)}\")\n", - "print(f\" Estimated cost: {plan.estimated_cost}\")\n", - "print(f\" Execution steps:\")\n", - "for i, step in enumerate(plan.execution_steps, 1):\n", - " print(f\" {i}. {step}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 6: Bulk Loading\n", - "\n", - "Load large datasets efficiently with progress tracking.\n", - "\n", - "### Bulk Loading Features\n", - "\n", - "- **Batch Processing**: Process in configurable batches\n", - "- **Progress Tracking**: Monitor loading progress\n", - "- **Retry Mechanism**: Handle failures gracefully\n", - "- **Validation**: Validate before loading" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import BulkLoader, LoadProgress\n", - "\n", - "# Create bulk loader\n", - "loader = BulkLoader(\n", - " batch_size=1000,\n", - " max_retries=3\n", - ")\n", - "\n", - "# Generate sample triples\n", - "large_dataset = [\n", - " Triple(\n", - " f\"http://example.org/entity{i}\",\n", - " \"http://example.org/hasName\",\n", - " f\"Entity {i}\"\n", - " )\n", - " for i in range(5000)\n", - "]\n", - "\n", - "# Progress callback\n", - "def progress_callback(progress: LoadProgress):\n", - " print(f\"Progress: {progress.progress_percentage:.1f}% \"\n", - " f\"({progress.loaded_triples}/{progress.total_triples}) \"\n", - " f\"Batch {progress.current_batch}/{progress.total_batches}\")\n", - "\n", - "# Load triples with progress tracking\n", - "adapter = BlazegraphAdapter(endpoint=\"http://localhost:9999/blazegraph/sparql\")\n", - "progress = loader.load_triples(\n", - " large_dataset,\n", - " adapter,\n", - " progress_callback=progress_callback\n", - ")\n", - "\n", - "print(f\"\\nLoading Complete:\")\n", - "print(f\" Loaded: {progress.loaded_triples}/{progress.total_triples}\")\n", - "print(f\" Failed: {progress.failed_triples}\")\n", - "print(f\" Elapsed time: {progress.elapsed_time:.2f}s\")\n", - "print(f\" Throughput: {progress.metadata.get('throughput', 0):.0f} triples/sec\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 7: Store Adapters\n", - "\n", - "Work with different triplet store backends.\n", - "\n", - "### Blazegraph Adapter\n", - "\n", - "High-performance triplet store with GPU acceleration." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import BlazegraphAdapter\n", - "\n", - "# Create Blazegraph adapter\n", - "blazegraph = BlazegraphAdapter(\n", - " endpoint=\"http://localhost:9999/blazegraph/sparql\",\n", - " namespace=\"kb\",\n", - " timeout=30\n", - ")\n", - "\n", - "# Add triples\n", - "triples = [\n", - " Triple(\"http://example.org/Alice\", \"http://example.org/hasSkill\", \"Python\")\n", - "]\n", - "result = blazegraph.add_triples(triples)\n", - "print(f\"Blazegraph - Added: {result['success']}\")\n", - "\n", - "# Execute SPARQL query\n", - "query = \"SELECT ?s ?p ?o WHERE { ?s ?p ?o } LIMIT 5\"\n", - "result = blazegraph.execute_sparql(query)\n", - "print(f\"Blazegraph - Found {len(result['bindings'])} results\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Jena Adapter\n", - "\n", - "Full-featured RDF framework with inference support." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import JenaAdapter\n", - "\n", - "# Create Jena adapter (in-memory)\n", - "jena = JenaAdapter()\n", - "\n", - "# Or connect to Fuseki endpoint\n", - "# jena = JenaAdapter(\n", - "# endpoint=\"http://localhost:3030/ds\",\n", - "# dataset=\"default\",\n", - "# enable_inference=True\n", - "# )\n", - "\n", - "# Add triples with inference\n", - "triples = [\n", - " Triple(\n", - " \"http://example.org/Dog\",\n", - " \"http://www.w3.org/2000/01/rdf-schema#subClassOf\",\n", - " \"http://example.org/Animal\"\n", - " ),\n", - " Triple(\n", - " \"http://example.org/Fido\",\n", - " \"http://www.w3.org/1999/02/22-rdf-syntax-ns#type\",\n", - " \"http://example.org/Dog\"\n", - " )\n", - "]\n", - "result = jena.add_triples(triples)\n", - "print(f\"Jena - Added: {result['success']}\")\n", - "\n", - "# Query with inference (Fido is inferred to be an Animal)\n", - "query = \"\"\"\n", - "PREFIX rdf: \n", - "PREFIX ex: \n", - "SELECT ?animal WHERE {\n", - " ?animal rdf:type ex:Animal .\n", - "}\n", - "\"\"\"\n", - "result = jena.query(query)\n", - "print(f\"Jena - Found {len(result)} animals (with inference)\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### RDF4J Adapter\n", - "\n", - "Java-based RDF framework with transaction support." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import RDF4JAdapter\n", - "\n", - "# Create RDF4J adapter\n", - "rdf4j = RDF4JAdapter(\n", - " server_url=\"http://localhost:8080/rdf4j-server\",\n", - " repository_id=\"test\"\n", - ")\n", - "\n", - "# Add triples with transaction\n", - "rdf4j.begin_transaction()\n", - "try:\n", - " triple = Triple(\n", - " \"http://example.org/Alice\",\n", - " \"http://example.org/hasEmail\",\n", - " \"alice@example.org\"\n", - " )\n", - " rdf4j.add_triple(\n", - " subject=triple.subject,\n", - " predicate=triple.predicate,\n", - " object_literal=triple.object\n", - " )\n", - " rdf4j.commit_transaction()\n", - " print(\"RDF4J - Transaction committed\")\n", - "except Exception as e:\n", - " rdf4j.rollback_transaction()\n", - " print(f\"RDF4J - Transaction rolled back: {e}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Virtuoso Adapter\n", - "\n", - "Enterprise-grade RDF store with SQL integration." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import VirtuosoAdapter\n", - "\n", - "# Create Virtuoso adapter\n", - "virtuoso = VirtuosoAdapter(\n", - " host=\"localhost\",\n", - " port=1111,\n", - " user=\"dba\",\n", - " password=\"dba\"\n", - ")\n", - "\n", - "# Create named graph\n", - "graph_uri = \"http://example.org/graph1\"\n", - "virtuoso.create_graph(graph_uri)\n", - "\n", - "# Add triples to named graph\n", - "triple = Triple(\n", - " \"http://example.org/Alice\",\n", - " \"http://example.org/worksAt\",\n", - " \"http://example.org/Company1\"\n", - ")\n", - "virtuoso.add_triple(\n", - " subject=triple.subject,\n", - " predicate=triple.predicate,\n", - " object=triple.object,\n", - " graph=graph_uri\n", - ")\n", - "\n", - "print(f\"Virtuoso - Added triple to graph: {graph_uri}\")\n", - "\n", - "# Query specific graph\n", - "query = f\"\"\"\n", - "PREFIX ex: \n", - "SELECT ?person ?company\n", - "FROM <{graph_uri}>\n", - "WHERE {{\n", - " ?person ex:worksAt ?company .\n", - "}}\n", - "\"\"\"\n", - "result = virtuoso.query(query)\n", - "print(f\"Virtuoso - Found {len(result)} results in graph\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 8: Triple Validation\n", - "\n", - "Validate triples before adding them to the store.\n", - "\n", - "### Validation Checks\n", - "\n", - "- **Required Fields**: Subject, predicate, object\n", - "- **Confidence Range**: 0-1 if provided\n", - "- **URI Format**: Valid URIs\n", - "- **Empty Components**: No empty values" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.triplet_store import validate_triples\n", - "\n", - "# Create triples (some invalid)\n", - "triples_to_validate = [\n", - " Triple(\"http://example.org/Alice\", \"http://example.org/knows\", \"http://example.org/Bob\"), # Valid\n", - " Triple(\"\", \"http://example.org/knows\", \"http://example.org/Charlie\"), # Invalid (empty subject)\n", - " Triple(\"http://example.org/Dave\", \"\", \"http://example.org/Eve\"), # Invalid (empty predicate)\n", - " Triple(\"http://example.org/Frank\", \"http://example.org/knows\", \"http://example.org/Grace\", confidence=1.5), # Invalid (confidence > 1)\n", - "]\n", - "\n", - "# Validate triples\n", - "validation = validate_triples(triples_to_validate)\n", - "\n", - "print(\"Validation Results:\")\n", - "print(f\" Valid: {validation['valid']}\")\n", - "print(f\" Valid triples: {validation['valid_triples']}/{validation['total_triples']}\")\n", - "print(f\"\\nErrors: {len(validation['errors'])}\")\n", - "for error in validation['errors']:\n", - " print(f\" - {error}\")\n", - "print(f\"\\nWarnings: {len(validation['warnings'])}\")\n", - "for warning in validation['warnings']:\n", - " print(f\" - {warning}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 9: Multi-Store Operations\n", - "\n", - "Work with multiple stores simultaneously.\n", - "\n", - "### Use Cases\n", - "\n", - "- **Primary/Backup**: Replicate to backup store\n", - "- **Read/Write Split**: Separate read and write stores\n", - "- **Multi-Tenant**: Different stores for different tenants" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Register multiple stores\n", - "manager = TripletManager()\n", - "\n", - "primary = manager.register_store(\n", - " \"primary\",\n", - " \"blazegraph\",\n", - " \"http://localhost:9999/blazegraph/sparql\"\n", - ")\n", - "\n", - "backup = manager.register_store(\n", - " \"backup\",\n", - " \"jena\",\n", - " \"http://localhost:3030/ds\"\n", - ")\n", - "\n", - "# Add to primary store\n", - "triple = Triple(\n", - " \"http://example.org/Document1\",\n", - " \"http://example.org/hasAuthor\",\n", - " \"http://example.org/Alice\"\n", - ")\n", - "manager.add_triple(triple, store_id=\"primary\")\n", - "print(\"Added to primary store\")\n", - "\n", - "# Replicate to backup store\n", - "manager.add_triple(triple, store_id=\"backup\")\n", - "print(\"Replicated to backup store\")\n", - "\n", - "# List all stores\n", - "stores = manager.list_stores()\n", - "print(f\"\\nActive stores: {stores}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 10: Best Practices\n", - "\n", - "### Choosing the Right Backend\n", - "\n", - "1. **Blazegraph**: High-performance, large datasets, GPU acceleration\n", - "2. **Jena**: Java integration, SHACL validation, inference\n", - "3. **RDF4J**: Transaction support, ACID guarantees, federation\n", - "4. **Virtuoso**: Enterprise scale, SQL integration, clustering\n", - "\n", - "### Performance Tips\n", - "\n", - "- **Batch Operations**: Use `add_triples()` for multiple triples\n", - "- **Query Optimization**: Enable optimization and caching\n", - "- **Bulk Loading**: Use `BulkLoader` for large datasets\n", - "- **Validation**: Validate before loading to avoid errors\n", - "\n", - "### Configuration\n", - "\n", - "- **Batch Size**: 1000-10000 for bulk loading\n", - "- **Cache Size**: 1000-5000 for query caching\n", - "- **Timeout**: 30-60 seconds for queries\n", - "- **Retries**: 3-5 for bulk operations" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "### What You've Learned\n", - "\n", - "In this notebook, you've learned how to:\n", - "\n", - "- Register and manage triplet stores\n", - "- Perform CRUD operations on RDF triples\n", - "- Execute and optimize SPARQL queries\n", - "- Use bulk loading for large datasets\n", - "- Work with multiple store backends\n", - "- Validate triples before operations\n", - "- Choose the right backend for your use case\n", - "\n", - "### Key Takeaways\n", - "\n", - "1. **Multi-Backend Support**: Choose the right backend for your needs\n", - "2. **SPARQL Power**: Full SPARQL 1.1 support with optimization\n", - "3. **Bulk Loading**: Efficient loading with progress tracking\n", - "4. **Query Optimization**: Automatic query optimization and caching\n", - "5. **Validation**: Pre-load validation prevents errors\n", - "6. **Multi-Store**: Manage multiple stores simultaneously\n", - "\n", - "### Next Steps\n", - "\n", - "**Further Reading**:\n", - "- [Triplet Store API Reference](https://semantica.readthedocs.io/reference/triplet_store/)\n", - "- [SPARQL 1.1 Specification](https://www.w3.org/TR/sparql11-query/)\n", - "- [Knowledge Graph Building](../use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n", - "\n", - "---\n", - "\n", - "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/20_Triple_Store.ipynb)\n", + "\n", + "# Triplet Store - Comprehensive Guide\n", + "\n", + "## Overview\n", + "\n", + "This notebook provides a **comprehensive walkthrough** of Semantica's triplet_store module, demonstrating RDF triplet storage, SPARQL querying, and multi-backend support for knowledge graph persistence.\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/triplet_store/)\n", + "\n", + "### Learning Objectives\n", + "\n", + "By the end of this notebook, you will be able to:\n", + "\n", + "- Register and manage triplet stores (Blazegraph, Jena, RDF4J, Virtuoso)\n", + "- Perform CRUD operations on RDF triplets\n", + "- Execute SPARQL queries with optimization\n", + "- Use bulk loading for large datasets\n", + "- Work with multiple store backends\n", + "- Validate and track triplet operations\n", + "- Choose the right backend for your use case\n", + "\n", + "### What You'll Learn\n", + "\n", + "| Component | Purpose | When to Use |\n", + "|-----------|---------|-------------|\n", + "| `TripletManager` | Store coordination | All triplet operations |\n", + "| `QueryEngine` | SPARQL execution | Query optimization |\n", + "| `BulkLoader` | High-volume loading | Large datasets |\n", + "| `BlazegraphAdapter` | Blazegraph backend | High performance |\n", + "| `JenaAdapter` | Jena backend | Java integration |\n", + "| `RDF4JAdapter` | RDF4J backend | Transaction support |\n", + "| `VirtuosoAdapter` | Virtuoso backend | Enterprise scale |\n", + "\n", + "---\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica from PyPI:\n", + "\n", + "```bash\n", + "pip install semantica\n", + "# Or with all optional dependencies:\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Basic Triplet Store Operations\n", + "\n", + "Let's start with the `TripletManager` for basic triplet store operations.\n", + "\n", + "### What is TripletManager?\n", + "\n", + "`TripletManager` is the main coordinator for triplet store operations:\n", + "- **Store Registration**: Register multiple backends\n", + "- **CRUD Operations**: Add, get, update, delete triples\n", + "- **Multi-Store**: Manage multiple stores simultaneously" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import TripletManager\n", + "from semantica.semantic_extract.triple_extractor import Triple\n", + "\n", + "# Create triple manager\n", + "manager = TripletManager()\n", + "\n", + "# Register a Blazegraph store (in-memory for demo)\n", + "store = manager.register_store(\n", + " store_id=\"demo\",\n", + " store_type=\"blazegraph\",\n", + " endpoint=\"http://localhost:9999/blazegraph/sparql\"\n", + ")\n", + "\n", + "print(f\"Registered store: {store.store_id}\")\n", + "print(f\"Store type: {store.store_type}\")\n", + "print(f\"Endpoint: {store.endpoint}\")\n", + "\n", + "# Create a triple\n", + "triple = Triple(\n", + " subject=\"http://example.org/Alice\",\n", + " predicate=\"http://example.org/knows\",\n", + " object=\"http://example.org/Bob\",\n", + " confidence=0.95\n", + ")\n", + "\n", + "# Add triple to store\n", + "result = manager.add_triple(triple, store_id=\"demo\")\n", + "print(f\"\\nTriple added: {result['success']}\")\n", + "print(f\"Triple: {triple.subject} -> {triple.predicate} -> {triple.object}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Store Registration and Management\n", + "\n", + "Register multiple stores and manage them.\n", + "\n", + "### Supported Backends\n", + "\n", + "| Backend | Best For | Performance | Features |\n", + "|---------|----------|-------------|----------|\n", + "| **Blazegraph** | Large datasets | Excellent | GPU acceleration, full-text |\n", + "| **Jena** | Java apps | Good | SHACL, inference |\n", + "| **RDF4J** | Transactions | Good | ACID, federation |\n", + "| **Virtuoso** | Enterprise | Excellent | SQL integration, clustering |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import register_store\n", + "\n", + "# Register multiple stores using convenience function\n", + "blazegraph_store = register_store(\n", + " \"blazegraph_main\",\n", + " \"blazegraph\",\n", + " \"http://localhost:9999/blazegraph/sparql\"\n", + ")\n", + "\n", + "jena_store = register_store(\n", + " \"jena_backup\",\n", + " \"jena\",\n", + " \"http://localhost:3030/ds\"\n", + ")\n", + "\n", + "# List all registered stores\n", + "stores = manager.list_stores()\n", + "print(f\"Registered stores: {stores}\")\n", + "\n", + "# Get specific store\n", + "store = manager.get_store(\"blazegraph_main\")\n", + "print(f\"\\nStore details:\")\n", + "print(f\" ID: {store.store_id}\")\n", + "print(f\" Type: {store.store_type}\")\n", + "print(f\" Endpoint: {store.endpoint}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: CRUD Operations\n", + "\n", + "Perform Create, Read, Update, Delete operations on triples.\n", + "\n", + "### Operations Overview\n", + "\n", + "- **Create**: `add_triple()`, `add_triples()`\n", + "- **Read**: `get_triple()`\n", + "- **Update**: `update_triple()`\n", + "- **Delete**: `delete_triple()`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import add_triple, add_triples, get_triples, update_triple, delete_triple\n", + "\n", + "# Create - Add single triple\n", + "triple1 = Triple(\n", + " subject=\"http://example.org/Alice\",\n", + " predicate=\"http://example.org/hasAge\",\n", + " object=\"30\"\n", + ")\n", + "result = add_triple(triple1, store_id=\"demo\")\n", + "print(f\"Added single triple: {result['success']}\")\n", + "\n", + "# Create - Add multiple triples\n", + "triples = [\n", + " Triple(\"http://example.org/Alice\", \"http://example.org/hasCity\", \"New York\"),\n", + " Triple(\"http://example.org/Bob\", \"http://example.org/hasAge\", \"25\"),\n", + " Triple(\"http://example.org/Bob\", \"http://example.org/hasCity\", \"Boston\")\n", + "]\n", + "result = add_triples(triples, store_id=\"demo\")\n", + "print(f\"\\nAdded {result['total_triples']} triples in {result['batches']} batches\")\n", + "\n", + "# Read - Get triples for a subject\n", + "alice_triples = get_triples(\n", + " subject=\"http://example.org/Alice\",\n", + " store_id=\"demo\"\n", + ")\n", + "print(f\"\\nFound {len(alice_triples)} triples for Alice\")\n", + "\n", + "# Update - Change Alice's age\n", + "old_triple = Triple(\"http://example.org/Alice\", \"http://example.org/hasAge\", \"30\")\n", + "new_triple = Triple(\"http://example.org/Alice\", \"http://example.org/hasAge\", \"31\")\n", + "result = update_triple(old_triple, new_triple, store_id=\"demo\")\n", + "print(f\"\\nUpdated triple: {result['success']}\")\n", + "\n", + "# Delete - Remove a triple\n", + "triple_to_delete = Triple(\"http://example.org/Bob\", \"http://example.org/hasCity\", \"Boston\")\n", + "result = delete_triple(triple_to_delete, store_id=\"demo\")\n", + "print(f\"Deleted triple: {result['success']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: SPARQL Query Execution\n", + "\n", + "Execute SPARQL queries with the QueryEngine.\n", + "\n", + "### Query Types\n", + "\n", + "- **SELECT**: Retrieve variable bindings\n", + "- **ASK**: Boolean queries\n", + "- **CONSTRUCT**: Build RDF graphs\n", + "- **DESCRIBE**: Describe resources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import QueryEngine, BlazegraphAdapter\n", + "\n", + "# Create query engine with caching\n", + "engine = QueryEngine(enable_caching=True, enable_optimization=True)\n", + "\n", + "# Create adapter\n", + "adapter = BlazegraphAdapter(endpoint=\"http://localhost:9999/blazegraph/sparql\")\n", + "\n", + "# SELECT query\n", + "select_query = \"\"\"\n", + "PREFIX ex: \n", + "\n", + "SELECT ?person ?age ?city\n", + "WHERE {\n", + " ?person ex:hasAge ?age .\n", + " ?person ex:hasCity ?city .\n", + "}\n", + "ORDER BY DESC(?age)\n", + "LIMIT 10\n", + "\"\"\"\n", + "\n", + "result = engine.execute_query(select_query, adapter)\n", + "\n", + "print(f\"Query Results:\")\n", + "print(f\" Variables: {result.variables}\")\n", + "print(f\" Results: {len(result.bindings)}\")\n", + "print(f\" Execution time: {result.execution_time:.2f}s\")\n", + "print(f\" Cached: {result.metadata.get('cached', False)}\")\n", + "\n", + "print(\"\\nResults:\")\n", + "for binding in result.bindings:\n", + " person = binding.get('person', {}).get('value', '')\n", + " age = binding.get('age', {}).get('value', '')\n", + " city = binding.get('city', {}).get('value', '')\n", + " print(f\" {person}: Age {age}, City {city}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Query Optimization\n", + "\n", + "Optimize SPARQL queries for better performance.\n", + "\n", + "### Optimization Features\n", + "\n", + "- **Query Planning**: Analyze execution steps\n", + "- **Cost Estimation**: Estimate query cost\n", + "- **Query Rewriting**: Optimize query structure\n", + "- **Caching**: Cache query results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import optimize_query, plan_query\n", + "\n", + "# Original query\n", + "query = \"\"\"\n", + "SELECT ?s ?p ?o\n", + "WHERE {\n", + " ?s ?p ?o .\n", + "}\n", + "\"\"\"\n", + "\n", + "# Optimize query (adds LIMIT if missing)\n", + "optimized = optimize_query(query, add_limit=True, default_limit=1000)\n", + "print(\"Optimized Query:\")\n", + "print(optimized)\n", + "\n", + "# Create query plan\n", + "plan = plan_query(query)\n", + "print(f\"\\nQuery Plan:\")\n", + "print(f\" Original length: {len(plan.query)}\")\n", + "print(f\" Optimized length: {len(plan.optimized_query)}\")\n", + "print(f\" Estimated cost: {plan.estimated_cost}\")\n", + "print(f\" Execution steps:\")\n", + "for i, step in enumerate(plan.execution_steps, 1):\n", + " print(f\" {i}. {step}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Bulk Loading\n", + "\n", + "Load large datasets efficiently with progress tracking.\n", + "\n", + "### Bulk Loading Features\n", + "\n", + "- **Batch Processing**: Process in configurable batches\n", + "- **Progress Tracking**: Monitor loading progress\n", + "- **Retry Mechanism**: Handle failures gracefully\n", + "- **Validation**: Validate before loading" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import BulkLoader, LoadProgress\n", + "\n", + "# Create bulk loader\n", + "loader = BulkLoader(\n", + " batch_size=1000,\n", + " max_retries=3\n", + ")\n", + "\n", + "# Generate sample triples\n", + "large_dataset = [\n", + " Triple(\n", + " f\"http://example.org/entity{i}\",\n", + " \"http://example.org/hasName\",\n", + " f\"Entity {i}\"\n", + " )\n", + " for i in range(5000)\n", + "]\n", + "\n", + "# Progress callback\n", + "def progress_callback(progress: LoadProgress):\n", + " print(f\"Progress: {progress.progress_percentage:.1f}% \"\n", + " f\"({progress.loaded_triples}/{progress.total_triples}) \"\n", + " f\"Batch {progress.current_batch}/{progress.total_batches}\")\n", + "\n", + "# Load triples with progress tracking\n", + "adapter = BlazegraphAdapter(endpoint=\"http://localhost:9999/blazegraph/sparql\")\n", + "progress = loader.load_triples(\n", + " large_dataset,\n", + " adapter,\n", + " progress_callback=progress_callback\n", + ")\n", + "\n", + "print(f\"\\nLoading Complete:\")\n", + "print(f\" Loaded: {progress.loaded_triples}/{progress.total_triples}\")\n", + "print(f\" Failed: {progress.failed_triples}\")\n", + "print(f\" Elapsed time: {progress.elapsed_time:.2f}s\")\n", + "print(f\" Throughput: {progress.metadata.get('throughput', 0):.0f} triples/sec\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Store Adapters\n", + "\n", + "Work with different triplet store backends.\n", + "\n", + "### Blazegraph Adapter\n", + "\n", + "High-performance triplet store with GPU acceleration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import BlazegraphAdapter\n", + "\n", + "# Create Blazegraph adapter\n", + "blazegraph = BlazegraphAdapter(\n", + " endpoint=\"http://localhost:9999/blazegraph/sparql\",\n", + " namespace=\"kb\",\n", + " timeout=30\n", + ")\n", + "\n", + "# Add triples\n", + "triples = [\n", + " Triple(\"http://example.org/Alice\", \"http://example.org/hasSkill\", \"Python\")\n", + "]\n", + "result = blazegraph.add_triples(triples)\n", + "print(f\"Blazegraph - Added: {result['success']}\")\n", + "\n", + "# Execute SPARQL query\n", + "query = \"SELECT ?s ?p ?o WHERE { ?s ?p ?o } LIMIT 5\"\n", + "result = blazegraph.execute_sparql(query)\n", + "print(f\"Blazegraph - Found {len(result['bindings'])} results\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Jena Adapter\n", + "\n", + "Full-featured RDF framework with inference support." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import JenaAdapter\n", + "\n", + "# Create Jena adapter (in-memory)\n", + "jena = JenaAdapter()\n", + "\n", + "# Or connect to Fuseki endpoint\n", + "# jena = JenaAdapter(\n", + "# endpoint=\"http://localhost:3030/ds\",\n", + "# dataset=\"default\",\n", + "# enable_inference=True\n", + "# )\n", + "\n", + "# Add triples with inference\n", + "triples = [\n", + " Triple(\n", + " \"http://example.org/Dog\",\n", + " \"http://www.w3.org/2000/01/rdf-schema#subClassOf\",\n", + " \"http://example.org/Animal\"\n", + " ),\n", + " Triple(\n", + " \"http://example.org/Fido\",\n", + " \"http://www.w3.org/1999/02/22-rdf-syntax-ns#type\",\n", + " \"http://example.org/Dog\"\n", + " )\n", + "]\n", + "result = jena.add_triples(triples)\n", + "print(f\"Jena - Added: {result['success']}\")\n", + "\n", + "# Query with inference (Fido is inferred to be an Animal)\n", + "query = \"\"\"\n", + "PREFIX rdf: \n", + "PREFIX ex: \n", + "SELECT ?animal WHERE {\n", + " ?animal rdf:type ex:Animal .\n", + "}\n", + "\"\"\"\n", + "result = jena.query(query)\n", + "print(f\"Jena - Found {len(result)} animals (with inference)\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### RDF4J Adapter\n", + "\n", + "Java-based RDF framework with transaction support." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import RDF4JAdapter\n", + "\n", + "# Create RDF4J adapter\n", + "rdf4j = RDF4JAdapter(\n", + " server_url=\"http://localhost:8080/rdf4j-server\",\n", + " repository_id=\"test\"\n", + ")\n", + "\n", + "# Add triples with transaction\n", + "rdf4j.begin_transaction()\n", + "try:\n", + " triple = Triple(\n", + " \"http://example.org/Alice\",\n", + " \"http://example.org/hasEmail\",\n", + " \"alice@example.org\"\n", + " )\n", + " rdf4j.add_triple(\n", + " subject=triple.subject,\n", + " predicate=triple.predicate,\n", + " object_literal=triple.object\n", + " )\n", + " rdf4j.commit_transaction()\n", + " print(\"RDF4J - Transaction committed\")\n", + "except Exception as e:\n", + " rdf4j.rollback_transaction()\n", + " print(f\"RDF4J - Transaction rolled back: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Virtuoso Adapter\n", + "\n", + "Enterprise-grade RDF store with SQL integration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import VirtuosoAdapter\n", + "\n", + "# Create Virtuoso adapter\n", + "virtuoso = VirtuosoAdapter(\n", + " host=\"localhost\",\n", + " port=1111,\n", + " user=\"dba\",\n", + " password=\"dba\"\n", + ")\n", + "\n", + "# Create named graph\n", + "graph_uri = \"http://example.org/graph1\"\n", + "virtuoso.create_graph(graph_uri)\n", + "\n", + "# Add triples to named graph\n", + "triple = Triple(\n", + " \"http://example.org/Alice\",\n", + " \"http://example.org/worksAt\",\n", + " \"http://example.org/Company1\"\n", + ")\n", + "virtuoso.add_triple(\n", + " subject=triple.subject,\n", + " predicate=triple.predicate,\n", + " object=triple.object,\n", + " graph=graph_uri\n", + ")\n", + "\n", + "print(f\"Virtuoso - Added triple to graph: {graph_uri}\")\n", + "\n", + "# Query specific graph\n", + "query = f\"\"\"\n", + "PREFIX ex: \n", + "SELECT ?person ?company\n", + "FROM <{graph_uri}>\n", + "WHERE {{\n", + " ?person ex:worksAt ?company .\n", + "}}\n", + "\"\"\"\n", + "result = virtuoso.query(query)\n", + "print(f\"Virtuoso - Found {len(result)} results in graph\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Triple Validation\n", + "\n", + "Validate triples before adding them to the store.\n", + "\n", + "### Validation Checks\n", + "\n", + "- **Required Fields**: Subject, predicate, object\n", + "- **Confidence Range**: 0-1 if provided\n", + "- **URI Format**: Valid URIs\n", + "- **Empty Components**: No empty values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.triplet_store import validate_triples\n", + "\n", + "# Create triples (some invalid)\n", + "triples_to_validate = [\n", + " Triple(\"http://example.org/Alice\", \"http://example.org/knows\", \"http://example.org/Bob\"), # Valid\n", + " Triple(\"\", \"http://example.org/knows\", \"http://example.org/Charlie\"), # Invalid (empty subject)\n", + " Triple(\"http://example.org/Dave\", \"\", \"http://example.org/Eve\"), # Invalid (empty predicate)\n", + " Triple(\"http://example.org/Frank\", \"http://example.org/knows\", \"http://example.org/Grace\", confidence=1.5), # Invalid (confidence > 1)\n", + "]\n", + "\n", + "# Validate triples\n", + "validation = validate_triples(triples_to_validate)\n", + "\n", + "print(\"Validation Results:\")\n", + "print(f\" Valid: {validation['valid']}\")\n", + "print(f\" Valid triples: {validation['valid_triples']}/{validation['total_triples']}\")\n", + "print(f\"\\nErrors: {len(validation['errors'])}\")\n", + "for error in validation['errors']:\n", + " print(f\" - {error}\")\n", + "print(f\"\\nWarnings: {len(validation['warnings'])}\")\n", + "for warning in validation['warnings']:\n", + " print(f\" - {warning}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Multi-Store Operations\n", + "\n", + "Work with multiple stores simultaneously.\n", + "\n", + "### Use Cases\n", + "\n", + "- **Primary/Backup**: Replicate to backup store\n", + "- **Read/Write Split**: Separate read and write stores\n", + "- **Multi-Tenant**: Different stores for different tenants" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Register multiple stores\n", + "manager = TripletManager()\n", + "\n", + "primary = manager.register_store(\n", + " \"primary\",\n", + " \"blazegraph\",\n", + " \"http://localhost:9999/blazegraph/sparql\"\n", + ")\n", + "\n", + "backup = manager.register_store(\n", + " \"backup\",\n", + " \"jena\",\n", + " \"http://localhost:3030/ds\"\n", + ")\n", + "\n", + "# Add to primary store\n", + "triple = Triple(\n", + " \"http://example.org/Document1\",\n", + " \"http://example.org/hasAuthor\",\n", + " \"http://example.org/Alice\"\n", + ")\n", + "manager.add_triple(triple, store_id=\"primary\")\n", + "print(\"Added to primary store\")\n", + "\n", + "# Replicate to backup store\n", + "manager.add_triple(triple, store_id=\"backup\")\n", + "print(\"Replicated to backup store\")\n", + "\n", + "# List all stores\n", + "stores = manager.list_stores()\n", + "print(f\"\\nActive stores: {stores}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: Best Practices\n", + "\n", + "### Choosing the Right Backend\n", + "\n", + "1. **Blazegraph**: High-performance, large datasets, GPU acceleration\n", + "2. **Jena**: Java integration, SHACL validation, inference\n", + "3. **RDF4J**: Transaction support, ACID guarantees, federation\n", + "4. **Virtuoso**: Enterprise scale, SQL integration, clustering\n", + "\n", + "### Performance Tips\n", + "\n", + "- **Batch Operations**: Use `add_triples()` for multiple triples\n", + "- **Query Optimization**: Enable optimization and caching\n", + "- **Bulk Loading**: Use `BulkLoader` for large datasets\n", + "- **Validation**: Validate before loading to avoid errors\n", + "\n", + "### Configuration\n", + "\n", + "- **Batch Size**: 1000-10000 for bulk loading\n", + "- **Cache Size**: 1000-5000 for query caching\n", + "- **Timeout**: 30-60 seconds for queries\n", + "- **Retries**: 3-5 for bulk operations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### What You've Learned\n", + "\n", + "In this notebook, you've learned how to:\n", + "\n", + "- Register and manage triplet stores\n", + "- Perform CRUD operations on RDF triples\n", + "- Execute and optimize SPARQL queries\n", + "- Use bulk loading for large datasets\n", + "- Work with multiple store backends\n", + "- Validate triples before operations\n", + "- Choose the right backend for your use case\n", + "\n", + "### Key Takeaways\n", + "\n", + "1. **Multi-Backend Support**: Choose the right backend for your needs\n", + "2. **SPARQL Power**: Full SPARQL 1.1 support with optimization\n", + "3. **Bulk Loading**: Efficient loading with progress tracking\n", + "4. **Query Optimization**: Automatic query optimization and caching\n", + "5. **Validation**: Pre-load validation prevents errors\n", + "6. **Multi-Store**: Manage multiple stores simultaneously\n", + "\n", + "### Next Steps\n", + "\n", + "**Further Reading**:\n", + "- [Triplet Store API Reference](https://semantica.readthedocs.io/reference/triplet_store/)\n", + "- [SPARQL 1.1 Specification](https://www.w3.org/TR/sparql11-query/)\n", + "- [Knowledge Graph Building](../use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n", + "\n", + "---\n", + "\n", + "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb b/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb index 94f06a51..623367bb 100644 --- a/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb +++ b/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb @@ -20,7 +20,7 @@ "- **Multi-hop Reasoning**: Follows relationships across the graph for deeper context\n", "- **20+ Semantica Modules**: Demonstrates comprehensive use of the framework\n", "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/concepts/) • [GraphRAG Guide](https://semantica.readthedocs.io/concepts/)\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/concepts/) \u2022 [GraphRAG Guide](https://semantica.readthedocs.io/concepts/)\n", "\n", "### What You'll Learn\n", "\n", @@ -33,7 +33,7 @@ "\n", "### Pipeline Overview\n", "\n", - "**Real-World Data Sources (MCP/Web/Feeds) → Parse → Extract Entities & Relationships → Build Knowledge Graph → Generate Embeddings → Vector Store → Hybrid Search → Context Retrieval → GraphRAG Query System → LLM Integration → Answer Generation**\n", + "**Real-World Data Sources (MCP/Web/Feeds) \u2192 Parse \u2192 Extract Entities & Relationships \u2192 Build Knowledge Graph \u2192 Generate Embeddings \u2192 Vector Store \u2192 Hybrid Search \u2192 Context Retrieval \u2192 GraphRAG Query System \u2192 LLM Integration \u2192 Answer Generation**\n", "\n", "---\n", "\n", @@ -56,6 +56,15 @@ "```\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install semantica\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -66,17 +75,6 @@ "\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" - ] - }, { "cell_type": "code", "execution_count": null, @@ -522,8 +520,8 @@ "resolved_result = resolve_entities(deduplicated_entities, method=\"fuzzy\")\n", "resolved_entities = resolved_result.get('entities', deduplicated_entities)\n", "\n", - "print(f\"Deduplicated: {len(flat_entities)} → {len(deduplicated_entities)} entities\")\n", - "print(f\"Resolved: {len(deduplicated_entities)} → {len(resolved_entities)} entities\")\n", + "print(f\"Deduplicated: {len(flat_entities)} \u2192 {len(deduplicated_entities)} entities\")\n", + "print(f\"Resolved: {len(deduplicated_entities)} \u2192 {len(resolved_entities)} entities\")\n", "\n", "print(\"Building knowledge graph...\")\n", "\n", @@ -1468,9 +1466,9 @@ " \n", " print(f\"\\nComplete workflow executed successfully!\")\n", " print(f\"Final Results:\")\n", - " print(f\" Query processed: ✓\")\n", + " print(f\" Query processed: \u2713\")\n", " print(f\" Context retrieved: {workflow_result['metrics']['context_items']} items\")\n", - " print(f\" Answer generated: ✓\")\n", + " print(f\" Answer generated: \u2713\")\n", "else:\n", " print(\"Configure data sources above to run complete workflow with real data\")\n" ] @@ -1510,11 +1508,11 @@ " print(f\"{feature:<20} {trad:<25} {graph:<25}\")\n", "\n", "print(\"\\nGraphRAG Advantages:\")\n", - "print(f\" • Better handling of complex queries requiring relationship understanding\")\n", - "print(f\" • Multi-hop reasoning across entities\")\n", - "print(f\" • More accurate answers through structured knowledge\")\n", - "print(f\" • Better explainability with graph paths\")\n", - "print(f\" • Reduced hallucinations through graph validation\")\n" + "print(f\" \u2022 Better handling of complex queries requiring relationship understanding\")\n", + "print(f\" \u2022 Multi-hop reasoning across entities\")\n", + "print(f\" \u2022 More accurate answers through structured knowledge\")\n", + "print(f\" \u2022 Better explainability with graph paths\")\n", + "print(f\" \u2022 Reduced hallucinations through graph validation\")\n" ] }, { @@ -1574,9 +1572,9 @@ "\n", "# Save vector store (if supported)\n", "print(\"\\nVector Store:\")\n", - "print(f\" Vectors stored: ✓\")\n", - "print(f\" Metadata stored: ✓\")\n", - "print(f\" Ready for reuse: ✓\")\n" + "print(f\" Vectors stored: \u2713\")\n", + "print(f\" Metadata stored: \u2713\")\n", + "print(f\" Ready for reuse: \u2713\")\n" ] }, { @@ -1644,4 +1642,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/biomedical/01_Drug_Discovery_Pipeline.ipynb b/cookbook/use_cases/biomedical/01_Drug_Discovery_Pipeline.ipynb index fe430419..fee625dc 100644 --- a/cookbook/use_cases/biomedical/01_Drug_Discovery_Pipeline.ipynb +++ b/cookbook/use_cases/biomedical/01_Drug_Discovery_Pipeline.ipynb @@ -41,7 +41,7 @@ "\n", "### Pipeline\n", "\n", - "**Drug/Protein Data Sources (APIs, DB, Feeds, MCP) → Parse → Extract Entities (compounds, targets, interactions) → Build Drug-Target KG → Generate Embeddings → Similarity Search → Predict Interactions → Target Identification → Generate Reports → Visualize**\n", + "**Drug/Protein Data Sources (APIs, DB, Feeds, MCP) \u2192 Parse \u2192 Extract Entities (compounds, targets, interactions) \u2192 Build Drug-Target KG \u2192 Generate Embeddings \u2192 Similarity Search \u2192 Predict Interactions \u2192 Target Identification \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -56,9 +56,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -252,7 +250,7 @@ "mcp_ingestor.disconnect(\"biomedical_mcp_server\")\n", "print(f\" Disconnected from MCP server\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Local drug-target interactions: {len(drug_data)}\")\n", "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", "print(f\" Drug APIs: {len(drug_apis)}\")\n", @@ -784,4 +782,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/biomedical/02_Genomic_Variant_Analysis.ipynb b/cookbook/use_cases/biomedical/02_Genomic_Variant_Analysis.ipynb index a9d5c9fb..85aea7a1 100644 --- a/cookbook/use_cases/biomedical/02_Genomic_Variant_Analysis.ipynb +++ b/cookbook/use_cases/biomedical/02_Genomic_Variant_Analysis.ipynb @@ -30,7 +30,7 @@ "\n", "### Pipeline\n", "\n", - "**Genomic Data Sources → Parse → Extract Entities (variants, genes, diseases, pathways) → Build Genomic KG → Analyze Associations → Predict Impact → Pathway Analysis → Generate Reports → Visualize**\n", + "**Genomic Data Sources \u2192 Parse \u2192 Extract Entities (variants, genes, diseases, pathways) \u2192 Build Genomic KG \u2192 Analyze Associations \u2192 Predict Impact \u2192 Pathway Analysis \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -73,7 +71,6 @@ "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.conflicts import ConflictDetector\n", "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", @@ -220,7 +217,7 @@ ")\n", "print(f\" Query pattern: {db_query}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Local variants: {len(genomic_data)}\")\n", "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", "print(f\" Feeds ingested: {len(feed_data_list)}\")\n", @@ -788,4 +785,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb b/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb index 68e09633..47dc84c1 100644 --- a/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb +++ b/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**DeFi Data Sources → Parse → Extract Entities (protocols, pools, tokens, strategies) → Build DeFi KG → Analyze Relationships → Risk Assessment → Yield Optimization → Generate Reports → Visualize**\n", + "**DeFi Data Sources \u2192 Parse \u2192 Extract Entities (protocols, pools, tokens, strategies) \u2192 Build DeFi KG \u2192 Analyze Relationships \u2192 Risk Assessment \u2192 Yield Optimization \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -195,7 +193,7 @@ ")\n", "print(f\" Query pattern: {db_query}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Local protocols: {len(defi_data)}\")\n", "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", "print(f\" Feeds ingested: {len(feed_data_list)}\")\n", @@ -581,7 +579,7 @@ "for opt in sorted(yield_optimization, key=lambda x: x[\"yield_score\"], reverse=True)[:5]:\n", " print(f\" - {opt['protocol']}: Yield Score {opt['yield_score']:.2f}, APY {opt['apy']:.1f}%\")\n", " for suggestion in opt['suggestions']:\n", - " print(f\" → {suggestion}\")\n" + " print(f\" \u2192 {suggestion}\")\n" ] }, { @@ -716,4 +714,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/blockchain/02_Transaction_Network_Analysis.ipynb b/cookbook/use_cases/blockchain/02_Transaction_Network_Analysis.ipynb index 9c44d10e..3c5e5fb3 100644 --- a/cookbook/use_cases/blockchain/02_Transaction_Network_Analysis.ipynb +++ b/cookbook/use_cases/blockchain/02_Transaction_Network_Analysis.ipynb @@ -30,7 +30,7 @@ "\n", "### Pipeline\n", "\n", - "**Real-time Transaction Streams → Parse → Extract Entities (wallets, transactions, addresses) → Build Temporal Transaction KG → Store in Graph DB → Detect Patterns (tumbling, mixing, clustering) → AML Analysis → Generate Alerts → Visualize**\n", + "**Real-time Transaction Streams \u2192 Parse \u2192 Extract Entities (wallets, transactions, addresses) \u2192 Build Temporal Transaction KG \u2192 Store in Graph DB \u2192 Detect Patterns (tumbling, mixing, clustering) \u2192 AML Analysis \u2192 Generate Alerts \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -72,7 +70,6 @@ "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.conflicts import ConflictDetector\n", "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", @@ -191,7 +188,7 @@ "for stream_source in stream_sources:\n", " print(f\" - {stream_source['type']}: {stream_source.get('topic') or stream_source.get('queue')}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Local transactions: {len(transaction_data)}\")\n", "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", "print(f\" Streaming sources: {len(stream_sources)}\")\n", @@ -714,4 +711,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/cybersecurity/01_Anomaly_Detection_Real_Time.ipynb b/cookbook/use_cases/cybersecurity/01_Anomaly_Detection_Real_Time.ipynb index a23335e2..f2001e44 100644 --- a/cookbook/use_cases/cybersecurity/01_Anomaly_Detection_Real_Time.ipynb +++ b/cookbook/use_cases/cybersecurity/01_Anomaly_Detection_Real_Time.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Stream Security Logs → Real-Time Parsing → Extract Entities → Build Temporal KG → Pattern Detection → Anomaly Detection → Generate Alerts → Monitor → Visualize**\n", + "**Stream Security Logs \u2192 Real-Time Parsing \u2192 Extract Entities \u2192 Build Temporal KG \u2192 Pattern Detection \u2192 Anomaly Detection \u2192 Generate Alerts \u2192 Monitor \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -71,7 +69,6 @@ "from semantica.kg import GraphBuilder, TemporalPatternDetector, TemporalGraphQuery, GraphAnalyzer\n", "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", "import tempfile\n", @@ -483,7 +480,7 @@ "print(f\"Monitoring {len(temporal_kg.get('entities', []))} entities in real-time\")\n", "print(f\"Active alerts: {len(alerts)}\")\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Stream Logs → Real-Time Parse → Extract → Temporal KG → Pattern Detection → Anomaly Detection → Alerts → Monitor → Visualize\")\n" + "print(f\"Pipeline complete: Stream Logs \u2192 Real-Time Parse \u2192 Extract \u2192 Temporal KG \u2192 Pattern Detection \u2192 Anomaly Detection \u2192 Alerts \u2192 Monitor \u2192 Visualize\")\n" ] } ], @@ -494,4 +491,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/cybersecurity/02_Incident_Analysis.ipynb b/cookbook/use_cases/cybersecurity/02_Incident_Analysis.ipynb index d50053e7..b5919511 100644 --- a/cookbook/use_cases/cybersecurity/02_Incident_Analysis.ipynb +++ b/cookbook/use_cases/cybersecurity/02_Incident_Analysis.ipynb @@ -28,7 +28,7 @@ "\n", "### Pipeline\n", "\n", - "**Multiple Security Sources → Parse Logs → Extract Security Entities → Build Incident KG → Analyze Relationships → Detect Anomalies → Generate Reports → Visualize**\n", + "**Multiple Security Sources \u2192 Parse Logs \u2192 Extract Security Entities \u2192 Build Incident KG \u2192 Analyze Relationships \u2192 Detect Anomalies \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -53,9 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -69,7 +67,6 @@ "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer, CentralityCalculator\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.conflicts import ConflictDetector\n", "from semantica.kg import ProvenanceTracker\n", "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", @@ -463,7 +460,7 @@ "temporal_viz = temporal_visualizer.visualize_timeline(incident_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Multiple Security Sources → Parse Logs → Extract Entities → Build KG → Analyze → Detect Anomalies → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Multiple Security Sources \u2192 Parse Logs \u2192 Extract Entities \u2192 Build KG \u2192 Analyze \u2192 Detect Anomalies \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -474,4 +471,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/cybersecurity/03_Threat_Correlation.ipynb b/cookbook/use_cases/cybersecurity/03_Threat_Correlation.ipynb index 22e78063..4fc8aaa3 100644 --- a/cookbook/use_cases/cybersecurity/03_Threat_Correlation.ipynb +++ b/cookbook/use_cases/cybersecurity/03_Threat_Correlation.ipynb @@ -28,7 +28,7 @@ "\n", "### Pipeline\n", "\n", - "**Multiple Threat Feeds → Parse → Extract IOCs → Build Temporal KG → Correlate Threats → Detect Campaigns → Generate Reports → Visualize**\n", + "**Multiple Threat Feeds \u2192 Parse \u2192 Extract IOCs \u2192 Build Temporal KG \u2192 Correlate Threats \u2192 Detect Campaigns \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -53,9 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -69,7 +67,6 @@ "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector\n", "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, ExplanationGenerator\n", - "from semantica.kg import ProvenanceTracker\n", "from semantica.conflicts import ConflictDetector\n", "from semantica.export import RDFExporter, ReportGenerator\n", @@ -153,7 +150,7 @@ "\n", "parsed_db = structured_parser.parse_json(json.dumps(db_data)) if db_data else None\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Feeds ingested: {len(feed_data_list)}\")\n", "print(f\" Feed items processed: {len(parsed_feeds)}\")\n", "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", @@ -428,7 +425,7 @@ "temporal_viz = temporal_visualizer.visualize_timeline(threat_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Multi-source ingestion → Extraction → Temporal KG → Correlation → Campaign Detection → Quality → Reports → Visualization\")\n" + "print(f\"Pipeline complete: Multi-source ingestion \u2192 Extraction \u2192 Temporal KG \u2192 Correlation \u2192 Campaign Detection \u2192 Quality \u2192 Reports \u2192 Visualization\")\n" ] } ], @@ -439,4 +436,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/cybersecurity/04_Threat_Intelligence_Hybrid_RAG.ipynb b/cookbook/use_cases/cybersecurity/04_Threat_Intelligence_Hybrid_RAG.ipynb index 96c49606..105e376b 100644 --- a/cookbook/use_cases/cybersecurity/04_Threat_Intelligence_Hybrid_RAG.ipynb +++ b/cookbook/use_cases/cybersecurity/04_Threat_Intelligence_Hybrid_RAG.ipynb @@ -30,7 +30,7 @@ "\n", "### Pipeline\n", "\n", - "**Multi-Source Threat Intel (Files, Web, Feeds, MCP) → Parse → Extract Entities → Build KG → Generate Embeddings → Vector Store → Hybrid RAG Setup → Query Threats → Generate Reports → Visualize**\n", + "**Multi-Source Threat Intel (Files, Web, Feeds, MCP) \u2192 Parse \u2192 Extract Entities \u2192 Build KG \u2192 Generate Embeddings \u2192 Vector Store \u2192 Hybrid RAG Setup \u2192 Query Threats \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -586,7 +584,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(threat_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Multi-Source Threat Intel → Parse → Extract → Build KG → Embeddings → Vector Store → Hybrid RAG → Query → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Multi-Source Threat Intel \u2192 Parse \u2192 Extract \u2192 Build KG \u2192 Embeddings \u2192 Vector Store \u2192 Hybrid RAG \u2192 Query \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -597,4 +595,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/cybersecurity/05_Threat_Intelligence_Integration.ipynb b/cookbook/use_cases/cybersecurity/05_Threat_Intelligence_Integration.ipynb index 10c8919f..605f7fe1 100644 --- a/cookbook/use_cases/cybersecurity/05_Threat_Intelligence_Integration.ipynb +++ b/cookbook/use_cases/cybersecurity/05_Threat_Intelligence_Integration.ipynb @@ -31,7 +31,7 @@ "\n", "### Pipeline\n", "\n", - "**Connect to Threat Intel MCP Server → Ingest Threat Data via MCP → Parse MCP Responses → Extract Threat Entities → Build Threat KG → Generate Embeddings → Hybrid RAG → Analyze Threats → Generate Reports → Visualize**\n", + "**Connect to Threat Intel MCP Server \u2192 Ingest Threat Data via MCP \u2192 Parse MCP Responses \u2192 Extract Threat Entities \u2192 Build Threat KG \u2192 Generate Embeddings \u2192 Hybrid RAG \u2192 Analyze Threats \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -56,9 +56,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -99,7 +97,7 @@ "\n", "# List available resources (threat feeds, vulnerability databases)\n", "resources = mcp_ingestor.list_available_resources(\"threat_server\")\n", - "print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + "print(f\"\\n\ud83d\udcca Available Resources ({len(resources)}):\")\n", "for resource in resources[:5]: # Show first 5\n", " print(f\" - {resource.uri}: {resource.name}\")\n", " if resource.description:\n", @@ -107,7 +105,7 @@ "\n", "# List available tools (threat queries, IOC checks)\n", "tools = mcp_ingestor.list_available_tools(\"threat_server\")\n", - "print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + "print(f\"\\n\ud83d\udd27 Available Tools ({len(tools)}):\")\n", "for tool in tools[:5]: # Show first 5\n", " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n" ] @@ -225,7 +223,7 @@ " print(f\" Loaded {len(sample_data['threat_indicators'])} threat indicators\")\n", " print(f\" Loaded {len(sample_data['vulnerabilities'])} vulnerabilities\")\n", "\n", - "print(f\"\\n📊 Total threat intelligence data items ingested: {len(threat_data)}\")\n" + "print(f\"\\n\ud83d\udcca Total threat intelligence data items ingested: {len(threat_data)}\")\n" ] }, { @@ -548,7 +546,7 @@ "mcp_ingestor.disconnect(\"threat_server\")\n", "print(\" Disconnected from MCP server\")\n", "\n", - "print(f\"📊 Total modules used: 20+\")\n" + "print(f\"\ud83d\udcca Total modules used: 20+\")\n" ] } ], @@ -559,4 +557,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/cybersecurity/06_Vulnerability_Tracking.ipynb b/cookbook/use_cases/cybersecurity/06_Vulnerability_Tracking.ipynb index c4116ac0..64e78a31 100644 --- a/cookbook/use_cases/cybersecurity/06_Vulnerability_Tracking.ipynb +++ b/cookbook/use_cases/cybersecurity/06_Vulnerability_Tracking.ipynb @@ -29,7 +29,7 @@ "\n", "### Pipeline\n", "\n", - "**Real CVE Sources → Parse → Extract Vulnerabilities → Build Temporal KG → Correlate → Predict Impact → Generate Reports → Visualize**\n", + "**Real CVE Sources \u2192 Parse \u2192 Extract Vulnerabilities \u2192 Build Temporal KG \u2192 Correlate \u2192 Predict Impact \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -71,7 +69,6 @@ "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.conflicts import ConflictDetector\n", "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", @@ -141,7 +138,7 @@ ")\n", "print(f\" Query pattern: {db_query}\")\n", "\n", - "print(f\"\\n📊 CVE Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca CVE Ingestion Summary:\")\n", "print(f\" Vulnerability feeds: {len(cve_feed_list)}\")\n", "print(f\" CVE API sources: {len(cve_api_data)}\")\n", "print(f\" Database sources: 1\")\n" @@ -155,4 +152,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/finance/01_Financial_Data_Integration.ipynb b/cookbook/use_cases/finance/01_Financial_Data_Integration.ipynb index 2cff55d1..6fe3a72b 100644 --- a/cookbook/use_cases/finance/01_Financial_Data_Integration.ipynb +++ b/cookbook/use_cases/finance/01_Financial_Data_Integration.ipynb @@ -6,7 +6,7 @@ "source": [ "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/01_Financial_Data_Integration.ipynb)\n", "\n", - "# 📈 Financial Data Integration Pipeline\n", + "# \ud83d\udcc8 Financial Data Integration Pipeline\n", "\n", "## Overview\n", "\n", @@ -28,7 +28,7 @@ "pip install semantica[all]\n", "```\n", "\n", - "### 🧩 Modules Used (20+)\n", + "### \ud83e\udde9 Modules Used (20+)\n", "\n", "- **Ingestion**: FileIngestor, WebIngestor, FeedIngestor, StreamIngestor, DBIngestor, EmailIngestor, RepoIngestor, MCPIngestor\n", "- **Parsing**: `MCPParser`, `JSONParser`, `StructuredDataParser`\n", @@ -39,13 +39,13 @@ "- **Export**: `JSONExporter`, `CSVExporter`, `RDFExporter`, `ReportGenerator`\n", "- **Visualization**: `KGVisualizer`, `TemporalVisualizer`, `AnalyticsVisualizer`\n", "\n", - "### 🔄 Pipeline\n", + "### \ud83d\udd04 Pipeline\n", "\n", - "**Connect to Financial MCP Server → Ingest Market Data via MCP → Parse MCP Responses → Extract Financial Entities → Build Financial KG → Analyze Trends → Generate Reports → Visualize**\n", + "**Connect to Financial MCP Server \u2192 Ingest Market Data via MCP \u2192 Parse MCP Responses \u2192 Extract Financial Entities \u2192 Build Financial KG \u2192 Analyze Trends \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", - "## 🔌 Step 1: Connect to Financial Data MCP Server\n", + "## \ud83d\udd0c Step 1: Connect to Financial Data MCP Server\n", "\n", "Connect to a Python/FastMCP MCP server that provides financial data via URL. The MCP server can expose resources (datasets, market data) and tools (queries, calculations).\n" ] @@ -56,9 +56,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -95,7 +93,7 @@ "\n", "# List available resources (datasets, market data feeds)\n", "resources = mcp_ingestor.list_available_resources(\"financial_server\")\n", - "print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + "print(f\"\\n\ud83d\udcca Available Resources ({len(resources)}):\")\n", "for resource in resources[:5]: # Show first 5\n", " print(f\" - {resource.uri}: {resource.name}\")\n", " if resource.description:\n", @@ -103,7 +101,7 @@ "\n", "# List available tools (queries, calculations)\n", "tools = mcp_ingestor.list_available_tools(\"financial_server\")\n", - "print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + "print(f\"\\n\ud83d\udd27 Available Tools ({len(tools)}):\")\n", "for tool in tools[:5]: # Show first 5\n", " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n" ] @@ -112,7 +110,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 📥 Step 2: Ingest Financial Data from MCP Server\n", + "## \ud83d\udce5 Step 2: Ingest Financial Data from MCP Server\n", "\n", "Ingest financial data using both resource-based and tool-based methods from the MCP server.\n" ] @@ -223,14 +221,14 @@ " financial_data.append(sample_data)\n", " print(f\" Loaded {len(sample_data['stock_prices'])} stock prices\")\n", "\n", - "print(f\"\\n📊 Total financial data items ingested: {len(financial_data)}\")\n" + "print(f\"\\n\ud83d\udcca Total financial data items ingested: {len(financial_data)}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 📄 Step 3: Parse MCP Data\n", + "## \ud83d\udcc4 Step 3: Parse MCP Data\n", "\n", "Parse the data received from MCP server responses (JSON, structured data).\n" ] @@ -270,7 +268,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## ⛏️ Step 4: Extract Financial Entities and Relationships\n", + "## \u26cf\ufe0f Step 4: Extract Financial Entities and Relationships\n", "\n", "Extract financial entities (companies, stocks, sectors) and relationships from MCP data.\n" ] @@ -365,7 +363,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 🕸️ Step 5: Build Financial Knowledge Graph\n", + "## \ud83d\udd78\ufe0f Step 5: Build Financial Knowledge Graph\n", "\n", "Build a knowledge graph from the extracted financial entities and relationships.\n" ] @@ -406,7 +404,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 📊 Step 6: Analyze Financial Trends\n", + "## \ud83d\udcca Step 6: Analyze Financial Trends\n", "\n", "Analyze financial trends using temporal queries and pattern detection.\n" ] @@ -463,7 +461,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 📤 Step 7: Export and Visualize\n", + "## \ud83d\udce4 Step 7: Export and Visualize\n", "\n", "Export the financial knowledge graph and generate visualizations.\n" ] @@ -518,7 +516,7 @@ "mcp_ingestor.disconnect(\"financial_server\")\n", "print(\" Disconnected from MCP server\")\n", "\n", - "print(f\"📊 Total modules used: 20+\")\n" + "print(f\"\ud83d\udcca Total modules used: 20+\")\n" ] } ], @@ -529,4 +527,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb b/cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb index e95d61ba..11773db5 100644 --- a/cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb +++ b/cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Financial Documents → Parse → Extract Entities → Build KG → Analyze Relationships → Generate Insights → Export → Visualize**\n", + "**Financial Documents \u2192 Parse \u2192 Extract Entities \u2192 Build KG \u2192 Analyze Relationships \u2192 Generate Insights \u2192 Export \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -143,7 +141,7 @@ "if web_content:\n", " print(f\" Ingested SEC EDGAR content\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Financial reports ingested: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Financial feeds: {len(financial_feed_list)}\")\n", "print(f\" Database sources: 1\")\n" @@ -424,7 +422,7 @@ "temporal_viz = temporal_visualizer.visualize_timeline(financial_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Financial Documents → Parse → Extract → Build KG → Analyze Relationships → Generate Insights → Export → Visualize\")\n" + "print(f\"Pipeline complete: Financial Documents \u2192 Parse \u2192 Extract \u2192 Build KG \u2192 Analyze Relationships \u2192 Generate Insights \u2192 Export \u2192 Visualize\")\n" ] } ], @@ -435,4 +433,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/finance/03_Fraud_Detection.ipynb b/cookbook/use_cases/finance/03_Fraud_Detection.ipynb index 37cee036..28481351 100644 --- a/cookbook/use_cases/finance/03_Fraud_Detection.ipynb +++ b/cookbook/use_cases/finance/03_Fraud_Detection.ipynb @@ -1,5 +1,5 @@ { - "cells": [ + "cells": [ { "cell_type": "markdown", "metadata": {}, @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Transaction Stream → Parse → Extract → Build Temporal KG → Store in Graph DB → Detect Patterns → Anomaly Detection → Generate Alerts → Visualize**\n", + "**Transaction Stream \u2192 Parse \u2192 Extract \u2192 Build Temporal KG \u2192 Store in Graph DB \u2192 Detect Patterns \u2192 Anomaly Detection \u2192 Generate Alerts \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -70,7 +68,6 @@ "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector\n", "from semantica.kg import GraphBuilder, TemporalPatternDetector, GraphAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", "import tempfile\n", @@ -421,7 +418,7 @@ "\n", "print(f\"Graph quality score: {quality_score.get('overall_score', 0):.3f}\")\n", "print(f\"Total modules used: 20+ (including GraphStore)\")\n", - "print(f\"Pipeline complete: Transaction Stream → Parse → Extract → Temporal KG → Store in Graph DB → Pattern Detection → Anomaly Detection → Reports → Visualization\")\n" + "print(f\"Pipeline complete: Transaction Stream \u2192 Parse \u2192 Extract \u2192 Temporal KG \u2192 Store in Graph DB \u2192 Pattern Detection \u2192 Anomaly Detection \u2192 Reports \u2192 Visualization\")\n" ] } ], @@ -432,4 +429,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/finance/04_Investment_Analysis_Hybrid_RAG.ipynb b/cookbook/use_cases/finance/04_Investment_Analysis_Hybrid_RAG.ipynb index 1504261f..03016693 100644 --- a/cookbook/use_cases/finance/04_Investment_Analysis_Hybrid_RAG.ipynb +++ b/cookbook/use_cases/finance/04_Investment_Analysis_Hybrid_RAG.ipynb @@ -40,7 +40,7 @@ "\n", "### Pipeline\n", "\n", - "**Multi-Source Investment Data → Parse → Extract Entities → Build KG → Generate Embeddings → Vector Store → Hybrid RAG Setup → Query Insights → Generate Reports → Visualize**\n", + "**Multi-Source Investment Data \u2192 Parse \u2192 Extract Entities \u2192 Build KG \u2192 Generate Embeddings \u2192 Vector Store \u2192 Hybrid RAG Setup \u2192 Query Insights \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -158,7 +156,7 @@ " api_content_list.append(api_content)\n", " print(f\" Ingested API: {api_url}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Investment data files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Financial feeds: {len(financial_feed_list)}\")\n", "print(f\" Investment APIs: {len(api_content_list)}\")\n", @@ -403,7 +401,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(investment_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Multi-Source Investment Data → Parse → Extract → Build KG → Embeddings → Vector Store → Hybrid RAG → Query → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Multi-Source Investment Data \u2192 Parse \u2192 Extract \u2192 Build KG \u2192 Embeddings \u2192 Vector Store \u2192 Hybrid RAG \u2192 Query \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -414,4 +412,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/finance/05_Regulatory_Compliance.ipynb b/cookbook/use_cases/finance/05_Regulatory_Compliance.ipynb index 1a132139..abca5f66 100644 --- a/cookbook/use_cases/finance/05_Regulatory_Compliance.ipynb +++ b/cookbook/use_cases/finance/05_Regulatory_Compliance.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Regulatory Documents → Parse → Extract Compliance Rules → Build Compliance Ontology → Validate Compliance → Generate Reports → Visualize**\n", + "**Regulatory Documents \u2192 Parse \u2192 Extract Compliance Rules \u2192 Build Compliance Ontology \u2192 Validate Compliance \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -71,7 +69,6 @@ "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n", "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.conflicts import ConflictDetector\n", "from semantica.export import JSONExporter, CSVExporter, RDFExporter, OWLExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", @@ -141,7 +138,7 @@ " regulatory_web_list.append(web_content)\n", " print(f\" Ingested regulatory source: {source_url}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Regulatory documents: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Regulatory web sources: {len(regulatory_web_list)}\")\n", "print(f\" Database sources: 1\")\n" @@ -241,14 +238,14 @@ "property_generator = PropertyGenerator()\n", "ontology_validator = OntologyValidator()\n", "\n", - "compliance_kg = builder.build(compliance_entities, compliance_relationships)\n", - "\n", - "compliance_ontology = ontology_generator.generate_ontology({\"entities\": compliance_entities, \"relationships\": compliance_relationships}, entities=compliance_entities, relationships=compliance_relationships)\n", - "\n", - "classes = class_inferrer.infer_classes(compliance_entities)\n", - "properties = property_generator.infer_properties(compliance_entities, compliance_relationships, classes)\n", - "\n", - "validation_result = ontology_validator.validate_ontology(compliance_ontology)\n", + "compliance_kg = builder.build(compliance_entities, compliance_relationships)\n", + "\n", + "compliance_ontology = ontology_generator.generate_ontology({\"entities\": compliance_entities, \"relationships\": compliance_relationships}, entities=compliance_entities, relationships=compliance_relationships)\n", + "\n", + "classes = class_inferrer.infer_classes(compliance_entities)\n", + "properties = property_generator.infer_properties(compliance_entities, compliance_relationships, classes)\n", + "\n", + "validation_result = ontology_validator.validate_ontology(compliance_ontology)\n", "\n", "print(f\"Built compliance knowledge graph\")\n", "print(f\" Entities: {len(compliance_kg.get('entities', []))}\")\n", @@ -377,7 +374,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(compliance_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Regulatory Documents → Parse → Extract Rules → Build Ontology → Validate Compliance → Generate Reports → Visualize\")\n" + "print(f\"Pipeline complete: Regulatory Documents \u2192 Parse \u2192 Extract Rules \u2192 Build Ontology \u2192 Validate Compliance \u2192 Generate Reports \u2192 Visualize\")\n" ] } ], @@ -388,4 +385,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/healthcare/01_Clinical_Reports_Processing.ipynb b/cookbook/use_cases/healthcare/01_Clinical_Reports_Processing.ipynb index d9c5efcd..def82281 100644 --- a/cookbook/use_cases/healthcare/01_Clinical_Reports_Processing.ipynb +++ b/cookbook/use_cases/healthcare/01_Clinical_Reports_Processing.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Clinical Documents (Files, APIs, DB, MCP) → Parse → Extract Medical Entities → Build Medical KG → Store in Triple Store → Query Patient Data → Generate Reports → Visualize**\n", + "**Clinical Documents (Files, APIs, DB, MCP) \u2192 Parse \u2192 Extract Medical Entities \u2192 Build Medical KG \u2192 Store in Triple Store \u2192 Query Patient Data \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -189,7 +187,7 @@ "mcp_ingestor.disconnect(\"clinical_mcp_server\")\n", "print(f\" Disconnected from MCP server\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Clinical reports: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" FHIR API sources: {len(fhir_content_list)}\")\n", "print(f\" Database sources: 1\")\n", @@ -429,7 +427,7 @@ "temporal_viz = temporal_visualizer.visualize_timeline(medical_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Clinical Documents → Parse → Extract → Build KG → Triple Store → Query → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Clinical Documents \u2192 Parse \u2192 Extract \u2192 Build KG \u2192 Triple Store \u2192 Query \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -440,4 +438,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/healthcare/02_Disease_Network_Analysis.ipynb b/cookbook/use_cases/healthcare/02_Disease_Network_Analysis.ipynb index 1d72979a..4625e728 100644 --- a/cookbook/use_cases/healthcare/02_Disease_Network_Analysis.ipynb +++ b/cookbook/use_cases/healthcare/02_Disease_Network_Analysis.ipynb @@ -40,7 +40,7 @@ "\n", "### Pipeline\n", "\n", - "**Disease Data Sources → Parse → Extract Disease Relationships → Build Disease Ontology → Analyze Networks → Predict Outcomes → Generate Reports → Visualize**\n", + "**Disease Data Sources \u2192 Parse \u2192 Extract Disease Relationships \u2192 Build Disease Ontology \u2192 Analyze Networks \u2192 Predict Outcomes \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -146,7 +144,7 @@ " disease_api_list.append(api_content)\n", " print(f\" Ingested disease API: {api_url}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Disease data files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Disease API sources: {len(disease_api_list)}\")\n", "print(f\" Database sources: 1\")\n" @@ -416,7 +414,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(disease_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Disease Data → Parse → Extract → Build Ontology → Analyze Networks → Predict Outcomes → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Disease Data \u2192 Parse \u2192 Extract \u2192 Build Ontology \u2192 Analyze Networks \u2192 Predict Outcomes \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -427,4 +425,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/healthcare/03_Drug_Interactions_Analysis.ipynb b/cookbook/use_cases/healthcare/03_Drug_Interactions_Analysis.ipynb index 23048aa6..6981aa2c 100644 --- a/cookbook/use_cases/healthcare/03_Drug_Interactions_Analysis.ipynb +++ b/cookbook/use_cases/healthcare/03_Drug_Interactions_Analysis.ipynb @@ -30,7 +30,7 @@ "\n", "### Pipeline\n", "\n", - "**Drug Data Sources → Parse → Extract Drug Info → Build Drug KG → Detect Interactions → Generate Ontology → Generate Reports → Visualize**\n", + "**Drug Data Sources \u2192 Parse \u2192 Extract Drug Info \u2192 Build Drug KG \u2192 Detect Interactions \u2192 Generate Ontology \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -73,7 +71,6 @@ "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery\n", "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.conflicts import ConflictDetector\n", "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", @@ -152,7 +149,7 @@ " drug_api_list.append(api_content)\n", " print(f\" Ingested drug API: {api_url}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Drug data files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Drug API sources: {len(drug_api_list)}\")\n", "print(f\" Database sources: 1\")\n" @@ -427,7 +424,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(drug_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Drug Data → Parse → Extract → Build KG → Detect Interactions → Generate Ontology → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Drug Data \u2192 Parse \u2192 Extract \u2192 Build KG \u2192 Detect Interactions \u2192 Generate Ontology \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -438,4 +435,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/healthcare/04_Healthcare_GraphRAG_Hybrid.ipynb b/cookbook/use_cases/healthcare/04_Healthcare_GraphRAG_Hybrid.ipynb index 60fc73ca..5c68cb32 100644 --- a/cookbook/use_cases/healthcare/04_Healthcare_GraphRAG_Hybrid.ipynb +++ b/cookbook/use_cases/healthcare/04_Healthcare_GraphRAG_Hybrid.ipynb @@ -64,7 +64,7 @@ "\n", "### Pipeline Overview\n", "\n", - "**Medical Ontologies + Clinical Documents + EHRs (Virtual) → Parse → Extract Medical Entities → Build Materialized KG → Generate Embeddings → Vector Store → GraphRAG Setup → Query Orchestration → Generate Answers → Visualize & Export**\n", + "**Medical Ontologies + Clinical Documents + EHRs (Virtual) \u2192 Parse \u2192 Extract Medical Entities \u2192 Build Materialized KG \u2192 Generate Embeddings \u2192 Vector Store \u2192 GraphRAG Setup \u2192 Query Orchestration \u2192 Generate Answers \u2192 Visualize & Export**\n", "\n", "---\n", "\n", @@ -77,9 +77,16 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -U \"semantica[all]\"\n" ] }, { @@ -223,7 +230,7 @@ "if isinstance(clinical_data, dict) and 'content' in clinical_data:\n", " normalized_text = text_normalizer.normalize_text(clinical_data['content'])\n", " clinical_data['normalized_content'] = normalized_text\n" - ] + ] }, { "cell_type": "markdown", @@ -360,7 +367,7 @@ "\n", "print(f\" - Classes: {len(ontology_result.get('classes', []))}\")\n", "print(f\" - Properties: {len(ontology_result.get('properties', []))}\")\n", - "print(f\" - OWL generated: {len(owl_ontology) if owl_ontology else 0} characters\")\n", + "print(f\" - OWL generated: {len(owl_ontology) if owl_ontology else 0} characters\")\n" ] }, { @@ -780,8 +787,8 @@ "2. **Materialized Knowledge Graphs**: Semantica's KG modules enable building persistent knowledge graphs from medical ontologies and documents\n", "3. **Virtual Data Integration**: Semantica's DBIngestor allows virtual integration with EHRs without data replication\n", "4. **Hybrid Search**: Semantica's HybridSearch combines vector similarity with knowledge graph queries\n", - "5. **Query Orchestration**: Semantica's Reasoning and Triplet Store modules enable dynamic query orchestration\n", - "6. **Explainability**: Semantica's ExplanationGenerator provides traceable, explainable answers\n", + "5. **Query Orchestration**: Semantica's Reasoning and Triplet Store modules enable dynamic query orchestration\n", + "6. **Explainability**: Semantica's ExplanationGenerator provides traceable, explainable answers\n", "\n", "### Semantica-Specific Performance Considerations\n", "\n", @@ -824,4 +831,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/healthcare/05_Medical_Database_Integration.ipynb b/cookbook/use_cases/healthcare/05_Medical_Database_Integration.ipynb index 3df7f29a..fcb7bc02 100644 --- a/cookbook/use_cases/healthcare/05_Medical_Database_Integration.ipynb +++ b/cookbook/use_cases/healthcare/05_Medical_Database_Integration.ipynb @@ -31,7 +31,7 @@ "\n", "### Pipeline\n", "\n", - "**Connect to Medical MCP Server → Ingest Patient/Drug Data via MCP → Parse MCP Responses → Extract Medical Entities → Build Healthcare KG → Query & Analyze → Generate Reports → Visualize**\n", + "**Connect to Medical MCP Server \u2192 Ingest Patient/Drug Data via MCP \u2192 Parse MCP Responses \u2192 Extract Medical Entities \u2192 Build Healthcare KG \u2192 Query & Analyze \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -56,9 +56,16 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -U \"semantica[all]\"\n" ] }, { @@ -88,7 +95,7 @@ "\n", "# Connect to MCP server with authentication (if required)\n", "mcp_ingestor.connect(\n", - " \"medical_server\",\n, + " \"medical_server\",\n", " url=medical_mcp_url,\n", " headers={\n", " \"Authorization\": \"Bearer your_token\",\n", @@ -98,7 +105,7 @@ "\n", "# List available resources (patient records, drug databases)\n", "resources = mcp_ingestor.list_available_resources(\"medical_server\")\n", - "print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + "print(f\"\\n\ud83d\udcca Available Resources ({len(resources)}):\")\n", "for resource in resources[:5]: # Show first 5\n", " print(f\" - {resource.uri}: {resource.name}\")\n", " if resource.description:\n", @@ -106,7 +113,7 @@ "\n", "# List available tools (queries, drug interaction checks)\n", "tools = mcp_ingestor.list_available_tools(\"medical_server\")\n", - "print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + "print(f\"\\n\ud83d\udd27 Available Tools ({len(tools)}):\")\n", "for tool in tools[:5]: # Show first 5\n", " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n" ] @@ -219,7 +226,7 @@ " print(f\" Loaded {len(sample_data['patient_records'])} patient records\")\n", " print(f\" Loaded {len(sample_data['drug_interactions'])} drug interactions\")\n", "\n", - "print(f\"\\n📊 Total medical data items ingested: {len(medical_data)}\")\n" + "print(f\"\\n\ud83d\udcca Total medical data items ingested: {len(medical_data)}\")\n" ] }, { @@ -579,7 +586,7 @@ "mcp_ingestor.disconnect(\"medical_server\")\n", "print(\" Disconnected from MCP server\")\n", "\n", - "print(f\"📊 Total modules used: 20+\")\n" + "print(f\"\ud83d\udcca Total modules used: 20+\")\n" ] } ], @@ -590,4 +597,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/healthcare/06_Patient_Records_Temporal.ipynb b/cookbook/use_cases/healthcare/06_Patient_Records_Temporal.ipynb index f292759b..dfb057b0 100644 --- a/cookbook/use_cases/healthcare/06_Patient_Records_Temporal.ipynb +++ b/cookbook/use_cases/healthcare/06_Patient_Records_Temporal.ipynb @@ -28,7 +28,7 @@ "\n", "### Pipeline\n", "\n", - "**Patient Records → Parse → Extract Medical Entities → Build Temporal KG → Generate Ontology → Store in Triplet Store → Query History → Export → Visualize**\n", + "**Patient Records \u2192 Parse \u2192 Extract Medical Entities \u2192 Build Temporal KG \u2192 Generate Ontology \u2192 Store in Triplet Store \u2192 Query History \u2192 Export \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -53,9 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -337,7 +335,7 @@ "temporal_viz = temporal_visualizer.visualize_timeline(patient_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Patient Records → Parse → Extract → Temporal KG → Ontology → Triplet Store → Query → Export → Visualize\")\n" + "print(f\"Pipeline complete: Patient Records \u2192 Parse \u2192 Extract \u2192 Temporal KG \u2192 Ontology \u2192 Triplet Store \u2192 Query \u2192 Export \u2192 Visualize\")\n" ] } ], @@ -348,4 +346,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/intelligence/01_Criminal_Network_Analysis.ipynb b/cookbook/use_cases/intelligence/01_Criminal_Network_Analysis.ipynb index 08e2e641..914df873 100644 --- a/cookbook/use_cases/intelligence/01_Criminal_Network_Analysis.ipynb +++ b/cookbook/use_cases/intelligence/01_Criminal_Network_Analysis.ipynb @@ -69,7 +69,7 @@ "\n", "### Pipeline Overview\n", "\n", - "**Data Sources (Police Reports, Court Records, Surveillance) → MCP Integration → Agent-Based Data Gathering → Parse → Extract Entities/Relationships → Build Criminal Network KG → Graph Analytics → GraphRAG → Agent Analysis → Pattern Detection → Generate Intelligence Report → Visualize**\n", + "**Data Sources (Police Reports, Court Records, Surveillance) \u2192 MCP Integration \u2192 Agent-Based Data Gathering \u2192 Parse \u2192 Extract Entities/Relationships \u2192 Build Criminal Network KG \u2192 Graph Analytics \u2192 GraphRAG \u2192 Agent Analysis \u2192 Pattern Detection \u2192 Generate Intelligence Report \u2192 Visualize**\n", "\n", "---\n" ] @@ -80,9 +80,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -932,4 +930,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/intelligence/02_Intelligence_Analysis.ipynb b/cookbook/use_cases/intelligence/02_Intelligence_Analysis.ipynb index cd5dd628..5e09d638 100644 --- a/cookbook/use_cases/intelligence/02_Intelligence_Analysis.ipynb +++ b/cookbook/use_cases/intelligence/02_Intelligence_Analysis.ipynb @@ -43,7 +43,7 @@ "### Key Features\n", "\n", "- **Orchestrator-Worker Pattern**: 7 specialized workers coordinated by ExecutionEngine\n", - "- **Complete Ontology Pipeline**: 6-stage ontology generation (semantic network parsing → YAML-to-definition → definition-to-types → hierarchy generation → TTL generation → symbolic validation)\n", + "- **Complete Ontology Pipeline**: 6-stage ontology generation (semantic network parsing \u2192 YAML-to-definition \u2192 definition-to-types \u2192 hierarchy generation \u2192 TTL generation \u2192 symbolic validation)\n", "- **All Graph Analytics**: PageRank, Betweenness, Closeness, Eigenvector centrality, Louvain community detection, connectivity analysis, path finding\n", "- **Hybrid RAG**: Vector store + knowledge graph queries + hybrid search + query orchestration\n", "- **Multi-Source Intelligence**: OSINT, threat intelligence, social media, news, public records, geospatial data\n", @@ -71,7 +71,7 @@ "\n", "### Pipeline Overview\n", "\n", - "**OSINT Sources → MCP Integration → Orchestrator Setup → Parallel Workers (Data Ingestion → Ontology Building → Graph Construction → Graph Analytics → Hybrid RAG → Intelligence Analysis → Report Generation) → Visualization → Reporting**\n", + "**OSINT Sources \u2192 MCP Integration \u2192 Orchestrator Setup \u2192 Parallel Workers (Data Ingestion \u2192 Ontology Building \u2192 Graph Construction \u2192 Graph Analytics \u2192 Hybrid RAG \u2192 Intelligence Analysis \u2192 Report Generation) \u2192 Visualization \u2192 Reporting**\n", "\n", "## Installation\n", "\n", @@ -92,9 +92,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -772,4 +770,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/intelligence/03_Law_Enforcement_Forensics.ipynb b/cookbook/use_cases/intelligence/03_Law_Enforcement_Forensics.ipynb index a9f39d5e..f455a878 100644 --- a/cookbook/use_cases/intelligence/03_Law_Enforcement_Forensics.ipynb +++ b/cookbook/use_cases/intelligence/03_Law_Enforcement_Forensics.ipynb @@ -57,7 +57,7 @@ "\n", "### Pipeline Overview\n", "\n", - "**Case Files → Parse → Extract Evidence Entities/Relationships → Build Temporal Case KG → Graph Analytics → GraphRAG → Agent Analysis → Cross-Case Correlation → Generate Forensic Report → Visualize**\n", + "**Case Files \u2192 Parse \u2192 Extract Evidence Entities/Relationships \u2192 Build Temporal Case KG \u2192 Graph Analytics \u2192 GraphRAG \u2192 Agent Analysis \u2192 Cross-Case Correlation \u2192 Generate Forensic Report \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -78,9 +78,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -538,4 +536,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/renewable_energy/01_Energy_Market_Analysis.ipynb b/cookbook/use_cases/renewable_energy/01_Energy_Market_Analysis.ipynb index 39b20192..955691c3 100644 --- a/cookbook/use_cases/renewable_energy/01_Energy_Market_Analysis.ipynb +++ b/cookbook/use_cases/renewable_energy/01_Energy_Market_Analysis.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Energy Market Sources → Parse → Extract Entities → Build Temporal Market KG → Analyze Pricing → Predict Trends → Generate Reports → Visualize**\n", + "**Energy Market Sources \u2192 Parse \u2192 Extract Entities \u2192 Build Temporal Market KG \u2192 Analyze Pricing \u2192 Predict Trends \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -71,7 +69,6 @@ "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", "import tempfile\n", @@ -165,7 +162,7 @@ " energy_feed_list.append(feed_data)\n", " print(f\" Ingested feed: {feed_url}\")\n", "\n", - "print(f\"\\n📊 Energy Market Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Energy Market Ingestion Summary:\")\n", "print(f\" Energy market files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Energy APIs: {len(energy_api_list)}\")\n", "print(f\" Energy feeds: {len(energy_feed_list)}\")\n", @@ -439,7 +436,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(energy_market_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Energy Market Sources → Parse → Extract → Build Temporal KG → Analyze Pricing → Predict Trends → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Energy Market Sources \u2192 Parse \u2192 Extract \u2192 Build Temporal KG \u2192 Analyze Pricing \u2192 Predict Trends \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -450,4 +447,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/renewable_energy/02_Environmental_Impact.ipynb b/cookbook/use_cases/renewable_energy/02_Environmental_Impact.ipynb index 3cf3c178..2e0b1485 100644 --- a/cookbook/use_cases/renewable_energy/02_Environmental_Impact.ipynb +++ b/cookbook/use_cases/renewable_energy/02_Environmental_Impact.ipynb @@ -30,7 +30,7 @@ "\n", "### Pipeline\n", "\n", - "**Environmental Data Sources → Parse → Extract Entities → Build Impact KG → Analyze Relationships → Assess Impact → Generate Ontology → Reports → Visualize**\n", + "**Environmental Data Sources \u2192 Parse \u2192 Extract Entities \u2192 Build Impact KG \u2192 Analyze Relationships \u2192 Assess Impact \u2192 Generate Ontology \u2192 Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -167,7 +165,7 @@ " environmental_feed_list.append(feed_data)\n", " print(f\" Ingested feed: {feed_url}\")\n", "\n", - "print(f\"\\n📊 Environmental Data Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Environmental Data Ingestion Summary:\")\n", "print(f\" Environmental data files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Environmental APIs: {len(environmental_api_list)}\")\n", "print(f\" Environmental feeds: {len(environmental_feed_list)}\")\n", @@ -480,7 +478,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(impact_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Environmental Data → Parse → Extract → Build Impact KG → Analyze Relationships → Assess Impact → Generate Ontology → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Environmental Data \u2192 Parse \u2192 Extract \u2192 Build Impact KG \u2192 Analyze Relationships \u2192 Assess Impact \u2192 Generate Ontology \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -491,4 +489,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/renewable_energy/03_Grid_Management.ipynb b/cookbook/use_cases/renewable_energy/03_Grid_Management.ipynb index f5dade1c..990be409 100644 --- a/cookbook/use_cases/renewable_energy/03_Grid_Management.ipynb +++ b/cookbook/use_cases/renewable_energy/03_Grid_Management.ipynb @@ -29,7 +29,7 @@ "\n", "### Pipeline\n", "\n", - "**Stream Grid Data → Parse → Extract Entities → Build Temporal Grid KG → Monitor Grid Health → Detect Anomalies → Predict Failures → Generate Alerts → Visualize**\n", + "**Stream Grid Data \u2192 Parse \u2192 Extract Entities \u2192 Build Temporal Grid KG \u2192 Monitor Grid Health \u2192 Detect Anomalies \u2192 Predict Failures \u2192 Generate Alerts \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -71,7 +69,6 @@ "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", "import tempfile\n", @@ -168,7 +165,7 @@ "file_objects = file_ingestor.ingest_file(grid_stream_file, read_content=True)\n", "parsed_data = structured_parser.parse_json(grid_stream_file)\n", "\n", - "print(f\"\\n📊 Grid Data Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Grid Data Ingestion Summary:\")\n", "print(f\" Grid stream files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Streaming sources: {len(stream_sources)}\")\n", "print(f\" Database sources: 1\")\n" @@ -467,7 +464,7 @@ "\n", "print(f\"Real-time grid monitoring active\")\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Stream Grid Data → Parse → Extract → Build Temporal Grid KG → Monitor Health → Detect Anomalies → Predict Failures → Alerts → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Stream Grid Data \u2192 Parse \u2192 Extract \u2192 Build Temporal Grid KG \u2192 Monitor Health \u2192 Detect Anomalies \u2192 Predict Failures \u2192 Alerts \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -478,4 +475,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/renewable_energy/04_Resource_Optimization.ipynb b/cookbook/use_cases/renewable_energy/04_Resource_Optimization.ipynb index f39b4804..dc7df6df 100644 --- a/cookbook/use_cases/renewable_energy/04_Resource_Optimization.ipynb +++ b/cookbook/use_cases/renewable_energy/04_Resource_Optimization.ipynb @@ -29,7 +29,7 @@ "\n", "### Pipeline\n", "\n", - "**Resource Data Sources → Parse → Extract Entities → Build Resource KG → Analyze Efficiency → Optimize Allocation → Generate Reports → Visualize**\n", + "**Resource Data Sources \u2192 Parse \u2192 Extract Entities \u2192 Build Resource KG \u2192 Analyze Efficiency \u2192 Optimize Allocation \u2192 Generate Reports \u2192 Visualize**\n", "\n", "## Installation\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -71,7 +69,6 @@ "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.conflicts import ConflictDetector\n", "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", @@ -152,7 +149,7 @@ " resource_api_list.append(api_content)\n", " print(f\" Ingested resource API: {api_url}\")\n", "\n", - "print(f\"\\n📊 Resource Data Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Resource Data Ingestion Summary:\")\n", "print(f\" Resource data files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Resource APIs: {len(resource_api_list)}\")\n", "print(f\" Database sources: 1\")\n" @@ -439,7 +436,7 @@ "temporal_viz = temporal_visualizer.visualize_timeline(resource_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Resource Data → Parse → Extract → Build Resource KG → Analyze Efficiency → Optimize Allocation → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Resource Data \u2192 Parse \u2192 Extract \u2192 Build Resource KG \u2192 Analyze Efficiency \u2192 Optimize Allocation \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -450,4 +447,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/supply_chain/01_Supply_Chain_Data_Integration.ipynb b/cookbook/use_cases/supply_chain/01_Supply_Chain_Data_Integration.ipynb index c20a393d..2dda89d8 100644 --- a/cookbook/use_cases/supply_chain/01_Supply_Chain_Data_Integration.ipynb +++ b/cookbook/use_cases/supply_chain/01_Supply_Chain_Data_Integration.ipynb @@ -40,7 +40,7 @@ "\n", "### Pipeline\n", "\n", - "**Connect to Supply Chain MCP Server → Ingest Logistics Data via MCP → Parse MCP Responses → Extract Supply Chain Entities → Build Supply Chain KG → Analyze Supply Chain → Generate Reports → Visualize**\n", + "**Connect to Supply Chain MCP Server \u2192 Ingest Logistics Data via MCP \u2192 Parse MCP Responses \u2192 Extract Supply Chain Entities \u2192 Build Supply Chain KG \u2192 Analyze Supply Chain \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -97,7 +95,7 @@ "\n", "# List available resources (inventory databases, shipment records)\n", "resources = mcp_ingestor.list_available_resources(\"supply_chain_server\")\n", - "print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + "print(f\"\\n\ud83d\udcca Available Resources ({len(resources)}):\")\n", "for resource in resources[:5]: # Show first 5\n", " print(f\" - {resource.uri}: {resource.name}\")\n", " if resource.description:\n", @@ -105,7 +103,7 @@ "\n", "# List available tools (logistics queries, inventory checks)\n", "tools = mcp_ingestor.list_available_tools(\"supply_chain_server\")\n", - "print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + "print(f\"\\n\ud83d\udd27 Available Tools ({len(tools)}):\")\n", "for tool in tools[:5]: # Show first 5\n", " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n" ] @@ -230,7 +228,7 @@ " print(f\" Loaded {len(sample_data['inventory'])} inventory records\")\n", " print(f\" Loaded {len(sample_data['shipments'])} shipment records\")\n", "\n", - "print(f\"\\n📊 Total supply chain data items ingested: {len(supply_chain_data)}\")\n" + "print(f\"\\n\ud83d\udcca Total supply chain data items ingested: {len(supply_chain_data)}\")\n" ] }, { @@ -566,7 +564,7 @@ "mcp_ingestor.disconnect(\"supply_chain_server\")\n", "print(\" Disconnected from MCP server\")\n", "\n", - "print(f\"📊 Total modules used: 20+\")\n" + "print(f\"\ud83d\udcca Total modules used: 20+\")\n" ] } ], @@ -577,4 +575,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/supply_chain/02_Supply_Chain_Risk_Management.ipynb b/cookbook/use_cases/supply_chain/02_Supply_Chain_Risk_Management.ipynb index eb144b20..251b6ccd 100644 --- a/cookbook/use_cases/supply_chain/02_Supply_Chain_Risk_Management.ipynb +++ b/cookbook/use_cases/supply_chain/02_Supply_Chain_Risk_Management.ipynb @@ -77,9 +77,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -745,4 +743,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb b/cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb index 9a247dfa..4b790deb 100644 --- a/cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb +++ b/cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Stream Market Data → Parse → Extract Entities → Build Temporal Market KG → Analyze Patterns → Predict Trends → Generate Reports → Visualize**\n", + "**Stream Market Data \u2192 Parse \u2192 Extract Entities \u2192 Build Temporal Market KG \u2192 Analyze Patterns \u2192 Predict Trends \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -71,7 +69,6 @@ "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", "import tempfile\n", @@ -160,7 +157,7 @@ " financial_feed_list.append(feed_data)\n", " print(f\" Ingested feed: {feed_url}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Market data files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Financial feeds: {len(financial_feed_list)}\")\n", "print(f\" Streaming sources: {len(stream_sources)}\")\n", @@ -364,7 +361,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(market_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Stream Market Data → Parse → Extract → Build Temporal KG → Analyze Patterns → Predict Trends → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Stream Market Data \u2192 Parse \u2192 Extract \u2192 Build Temporal KG \u2192 Analyze Patterns \u2192 Predict Trends \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -375,4 +372,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/trading/02_News_Sentiment_Analysis.ipynb b/cookbook/use_cases/trading/02_News_Sentiment_Analysis.ipynb index 67b34c79..59ace499 100644 --- a/cookbook/use_cases/trading/02_News_Sentiment_Analysis.ipynb +++ b/cookbook/use_cases/trading/02_News_Sentiment_Analysis.ipynb @@ -40,7 +40,7 @@ "\n", "### Pipeline\n", "\n", - "**Ingest News → Parse → Extract Entities → Build News KG → Generate Embeddings → Analyze Sentiment → Generate Trading Signals → Export → Visualize**\n", + "**Ingest News \u2192 Parse \u2192 Extract Entities \u2192 Build News KG \u2192 Generate Embeddings \u2192 Analyze Sentiment \u2192 Generate Trading Signals \u2192 Export \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -55,9 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -73,7 +71,6 @@ "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", "import tempfile\n", @@ -148,7 +145,7 @@ " print(f\" Ingested feed: {feed_url}\")\n", " print(f\" Items: {len(feed_data.items) if hasattr(feed_data, 'items') else 0}\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" News files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Financial feeds: {len(financial_feed_list)}\")\n", "print(f\" Database sources: 1\")\n" @@ -376,7 +373,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(news_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Ingest News → Parse → Extract → Build KG → Embeddings → Sentiment Analysis → Trading Signals → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Ingest News \u2192 Parse \u2192 Extract \u2192 Build KG \u2192 Embeddings \u2192 Sentiment Analysis \u2192 Trading Signals \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -387,4 +384,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb b/cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb index 986d2fae..37bef3d9 100644 --- a/cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb +++ b/cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Real-Time Trading Streams → Parse → Extract Entities → Build Temporal KG → Monitor Positions → Detect Anomalies → Generate Alerts → Visualize**\n", + "**Real-Time Trading Streams \u2192 Parse \u2192 Extract Entities \u2192 Build Temporal KG \u2192 Monitor Positions \u2192 Detect Anomalies \u2192 Generate Alerts \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -71,7 +69,6 @@ "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", "import tempfile\n", @@ -151,7 +148,7 @@ "file_objects = file_ingestor.ingest_file(trading_stream_file, read_content=True)\n", "parsed_data = structured_parser.parse_data(trading_stream_file, data_format=\"json\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Trading stream files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Streaming sources: {len(stream_sources)}\")\n", "print(f\" Database sources: 1\")\n" @@ -406,7 +403,7 @@ "\n", "print(f\"Real-time monitoring active\")\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Real-Time Streams → Parse → Extract → Build Temporal KG → Monitor Positions → Detect Anomalies → Alerts → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Real-Time Streams \u2192 Parse \u2192 Extract \u2192 Build Temporal KG \u2192 Monitor Positions \u2192 Detect Anomalies \u2192 Alerts \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -417,4 +414,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/trading/04_Risk_Assessment.ipynb b/cookbook/use_cases/trading/04_Risk_Assessment.ipynb index 55459dc2..491e7bb7 100644 --- a/cookbook/use_cases/trading/04_Risk_Assessment.ipynb +++ b/cookbook/use_cases/trading/04_Risk_Assessment.ipynb @@ -1,5 +1,5 @@ { - "cells": [ + "cells": [ { "cell_type": "markdown", "metadata": {}, @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Risk Data Sources → Parse → Extract Risk Entities → Build Risk KG → Analyze Risk Relationships → Assess Portfolio Risk → Generate Reports → Visualize**\n", + "**Risk Data Sources \u2192 Parse \u2192 Extract Risk Entities \u2192 Build Risk KG \u2192 Analyze Risk Relationships \u2192 Assess Portfolio Risk \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -146,7 +144,7 @@ "file_objects = file_ingestor.ingest_file(risk_data_file, read_content=True)\n", "parsed_data = structured_parser.parse_data(risk_data_file, data_format=\"json\")\n", "\n", - "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Ingestion Summary:\")\n", "print(f\" Risk data files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Database sources: 1\")\n" ] @@ -411,7 +409,7 @@ "temporal_viz = temporal_visualizer.visualize_timeline(risk_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Risk Data → Parse → Extract → Build Risk KG → Analyze Relationships → Assess Portfolio Risk → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Risk Data \u2192 Parse \u2192 Extract \u2192 Build Risk KG \u2192 Analyze Relationships \u2192 Assess Portfolio Risk \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -422,4 +420,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb b/cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb index fdd9a6be..0c2ceaa3 100644 --- a/cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb +++ b/cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb @@ -39,7 +39,7 @@ "\n", "### Pipeline\n", "\n", - "**Historical Data → Parse → Extract Entities → Build Temporal KG → Test Strategies → Analyze Performance → Generate Reports → Visualize**\n", + "**Historical Data \u2192 Parse \u2192 Extract Entities \u2192 Build Temporal KG \u2192 Test Strategies \u2192 Analyze Performance \u2192 Generate Reports \u2192 Visualize**\n", "\n", "---\n", "\n", @@ -54,9 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -U \"semantica[all]\"\n", - "import semantica\n", - "print(semantica.__version__)\n" + "!pip install semantica\n" ] }, { @@ -138,7 +136,7 @@ " historical_feed_list.append(feed_data)\n", " print(f\" Ingested feed: {feed_url}\")\n", "\n", - "print(f\"\\n📊 Historical Data Ingestion Summary:\")\n", + "print(f\"\\n\ud83d\udcca Historical Data Ingestion Summary:\")\n", "print(f\" Historical data files: {len([file_objects]) if file_objects else 0}\")\n", "print(f\" Historical market APIs: {len(historical_api_list)}\")\n", "print(f\" Historical feeds: {len(historical_feed_list)}\")\n", @@ -417,7 +415,7 @@ "analytics_viz = analytics_visualizer.visualize_analytics(historical_kg, output=\"interactive\")\n", "\n", "print(f\"Total modules used: 20+\")\n", - "print(f\"Pipeline complete: Historical Data → Parse → Extract → Build Temporal KG → Test Strategies → Analyze Performance → Reports → Visualize\")\n" + "print(f\"Pipeline complete: Historical Data \u2192 Parse \u2192 Extract \u2192 Build Temporal KG \u2192 Test Strategies \u2192 Analyze Performance \u2192 Reports \u2192 Visualize\")\n" ] } ], @@ -428,4 +426,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file