diff --git a/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb b/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb index 2cb86c9a..20e95b25 100644 --- a/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb +++ b/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb @@ -24,62 +24,105 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "aa672d39", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n" + ] + } + ], "source": [ - "# Install dependencies\n", - "!pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq sentence-transformers" - ] - }, - { - "cell_type": "markdown", - "id": "b7a767b5", - "metadata": {}, - "source": [ - "## 🛠️ Phase 0: Environment & Foundation\n", - "We configure **Groq** as our primary LLM provider and use **Sentence-Transformers** for local embeddings to avoid API dependencies." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6840539f", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import json\n", - "import pandas as pd\n", - "from semantica.core import Semantica, ConfigManager\n", - "from semantica.seed import SeedDataManager\n", - "from semantica.vector_store import VectorStore\n", - "\n", - "# 1. Groq Configuration\n", + "# Install dependencies\n", + "!pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq sentence-transformers" + ] + }, + { + "cell_type": "markdown", + "id": "b7a767b5", + "metadata": {}, + "source": [ + "## 🛠️ Phase 0: Environment & Foundation\n", + "We configure **Groq** as our primary LLM provider and use **Sentence-Transformers** for local embeddings to avoid API dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6840539f", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n" + ] + }, + { + "data": { + "text/html": [ + "

🧠 Semantica - 📊 Current Progress

StatusActionModuleSubmoduleFileTime
✅Semantica is building🧠 kgGraphBuilder-20.76s
🔄Semantica is building🧠 kgEntityResolver-146.04s
✅Semantica is deduplicating🔄 deduplicationDuplicateDetector-1.99s
✅Semantica is deduplicating🔄 deduplicationSimilarityCalculator-0.05s
✅Semantica is deduplicating🔄 deduplicationEntityMerger-0.33s
✅Semantica is deduplicating🔄 deduplicationMergeStrategyManager-0.09s
✅Semantica is resolving⚠️ conflictsConflictDetector-0.01s
✅Semantica is indexing📊 vector_storeVectorStore-0.02s
✅Semantica is processing🔗 contextContextRetriever-0.02s
✅Semantica is visualizing📈 visualizationKGVisualizer-0.64s
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Phase 0 Complete. Seeded 3 primary nodes.\n" + ] + } + ], + "source": [ + "import os\n", + "import json\n", + "import pandas as pd\n", + "from semantica.core import Semantica, ConfigManager\n", + "from semantica.seed import SeedDataManager\n", + "from semantica.vector_store import VectorStore\n", + "\n", + "# 1. Groq Configuration\n", "import getpass\n", "if \"GROQ_API_KEY\" not in os.environ:\n", " os.environ[\"GROQ_API_KEY\"] = getpass.getpass(\"Enter your Groq API Key: \")\n", - "\n", - "config_dict = {\n", - " \"project_name\": \"Skincare_Intelligence\",\n", - " \"embedding\": {\"provider\": \"sentence_transformers\", \"model\": \"all-MiniLM-L6-v2\"}, \n", - " \"extraction\": {\n", - " \"provider\": \"groq\", \n", - " \"model\": \"llama-3.1-8b-instant\", \n", - " \"temperature\": 0.0\n", - " },\n", - " \"inference\": {\n", - " \"provider\": \"groq\",\n", - " \"model\": \"llama-3.1-70b-versatile\"\n", - " },\n", - " \"vector_store\": {\"provider\": \"faiss\", \"dimension\": 384},\n", - " \"knowledge_graph\": {\"backend\": \"networkx\", \"merge_entities\": True}\n", - "}\n", - "\n", - "config = ConfigManager().load_from_dict(config_dict)\n", - "core = Semantica(config=config)\n", - "vs = VectorStore(backend=\"faiss\", dimension=384)\n", + "\n", + "config_dict = {\n", + " \"project_name\": \"Skincare_Intelligence\",\n", + " \"embedding\": {\"provider\": \"sentence_transformers\", \"model\": \"all-MiniLM-L6-v2\"}, \n", + " \"extraction\": {\n", + " \"provider\": \"groq\", \n", + " \"model\": \"llama-3.1-8b-instant\", \n", + " \"temperature\": 0.0\n", + " },\n", + " \"inference\": {\n", + " \"provider\": \"groq\",\n", + " \"model\": \"llama-3.1-70b-versatile\"\n", + " },\n", + " \"vector_store\": {\"provider\": \"faiss\", \"dimension\": 384},\n", + " \"knowledge_graph\": {\"backend\": \"networkx\", \"merge_entities\": True}\n", + "}\n", + "\n", + "config = ConfigManager().load_from_dict(config_dict)\n", + "core = Semantica(config=config)\n", + "vs = VectorStore(backend=\"faiss\", dimension=384)\n", "\n", "# 2. Seeding Ground Truth\n", "foundation_data = {\n", @@ -112,10 +155,85 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "879f9fd9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ingesting from: https://makeupandbeautyblog.com/feed\n", + "Ingesting from: https://www.westlakedermatology.com/feed\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to fetch feed https://www.westlakedermatology.com/feed: 403 Client Error: Forbidden for url: https://www.westlakedermatology.com/feed\n", + "Failed to ingest feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.westlakedermatology.com/feed\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Failed to ingest https://www.westlakedermatology.com/feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.westlakedermatology.com/feed\n", + "Ingesting from: https://www.contourderm.com/feed\n", + "Ingesting from: https://www.michelegreenmd.com/feed\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to fetch feed https://www.michelegreenmd.com/feed: 403 Client Error: Forbidden for url: https://www.michelegreenmd.com/feed\n", + "Failed to ingest feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.michelegreenmd.com/feed\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Failed to ingest https://www.michelegreenmd.com/feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.michelegreenmd.com/feed\n", + "Ingesting from: https://www.skincarephysicians.net/blog/feed/\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to fetch feed https://www.skincarephysicians.net/blog/feed/: 403 Client Error: Forbidden for url: https://www.skincarephysicians.net/blog/feed/\n", + "Failed to ingest feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.skincarephysicians.net/blog/feed/\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Failed to ingest https://www.skincarephysicians.net/blog/feed/: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.skincarephysicians.net/blog/feed/\n", + "Ingesting from: https://www.beautifulwithbrains.com/blog/feed/\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to fetch feed https://www.beautifulwithbrains.com/blog/feed/: 403 Client Error: Forbidden for url: https://www.beautifulwithbrains.com/blog/feed/\n", + "Failed to ingest feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.beautifulwithbrains.com/blog/feed/\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Failed to ingest https://www.beautifulwithbrains.com/blog/feed/: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.beautifulwithbrains.com/blog/feed/\n", + "Ingesting from: https://www.drbaileyskincare.com/blogs/blog.atom\n", + "✅ Phase 1 Complete. Generated 77 semantic chunks.\n" + ] + } + ], "source": [ "from semantica.ingest import ingest_feed, ingest_file\n", "from semantica.split import EntityAwareChunker\n", @@ -175,10 +293,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "54fa8fad", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting intelligence from chunks...\n", + "✅ Phase 2 Complete. Extracted 30 entities.\n" + ] + } + ], "source": [ "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", "\n", @@ -209,10 +336,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "97a63461", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Phase 3 Complete. Graph contains 9 resolved entities.\n" + ] + } + ], "source": [ "from semantica.kg import GraphBuilder, EntityResolver\n", "\n", @@ -244,10 +386,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "4edf258c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "🔍 Processing Multi-Hop Query: What ingredients synergize with Retinol to prevent irritation?\n", + "\n", + "❌ No relevant context found in the knowledge graph.\n" + ] + } + ], "source": [ "from semantica.context import ContextRetriever\n", "from semantica.reasoning import InferenceEngine\n", @@ -299,10 +452,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "4b52490d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\umap\\__init__.py:9: ImportWarning: Tensorflow not installed; ParametricUMAP will be unavailable\n", + " warn(\n" + ] + } + ], "source": [ "from semantica.visualization import KGVisualizer\n", "import matplotlib.pyplot as plt\n",