mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Update GraphRAG cookbook notebook
This commit is contained in:
KaifAhmad1
@@ -24,62 +24,105 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 1,
|
||||
"id": "aa672d39",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Install dependencies\n",
|
||||
"!pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq sentence-transformers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b7a767b5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🛠️ Phase 0: Environment & Foundation\n",
|
||||
"We configure **Groq** as our primary LLM provider and use **Sentence-Transformers** for local embeddings to avoid API dependencies."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6840539f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import json\n",
|
||||
"import pandas as pd\n",
|
||||
"from semantica.core import Semantica, ConfigManager\n",
|
||||
"from semantica.seed import SeedDataManager\n",
|
||||
"from semantica.vector_store import VectorStore\n",
|
||||
"\n",
|
||||
"# 1. Groq Configuration\n",
|
||||
"# Install dependencies\n",
|
||||
"!pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq sentence-transformers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b7a767b5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🛠️ Phase 0: Environment & Foundation\n",
|
||||
"We configure **Groq** as our primary LLM provider and use **Sentence-Transformers** for local embeddings to avoid API dependencies."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "6840539f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>File</th><th>Time</th></tr><tr><td>✅</td><td>Semantica is building</td><td>🧠 kg</td><td>GraphBuilder</td><td>-</td><td>20.76s</td></tr><tr><td>🔄</td><td>Semantica is building</td><td>🧠 kg</td><td>EntityResolver</td><td>-</td><td>146.04s</td></tr><tr><td>✅</td><td>Semantica is deduplicating</td><td>🔄 deduplication</td><td>DuplicateDetector</td><td>-</td><td>1.99s</td></tr><tr><td>✅</td><td>Semantica is deduplicating</td><td>🔄 deduplication</td><td>SimilarityCalculator</td><td>-</td><td>0.05s</td></tr><tr><td>✅</td><td>Semantica is deduplicating</td><td>🔄 deduplication</td><td>EntityMerger</td><td>-</td><td>0.33s</td></tr><tr><td>✅</td><td>Semantica is deduplicating</td><td>🔄 deduplication</td><td>MergeStrategyManager</td><td>-</td><td>0.09s</td></tr><tr><td>✅</td><td>Semantica is resolving</td><td>⚠️ conflicts</td><td>ConflictDetector</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is indexing</td><td>📊 vector_store</td><td>VectorStore</td><td>-</td><td>0.02s</td></tr><tr><td>✅</td><td>Semantica is processing</td><td>🔗 context</td><td>ContextRetriever</td><td>-</td><td>0.02s</td></tr><tr><td>✅</td><td>Semantica is visualizing</td><td>📈 visualization</td><td>KGVisualizer</td><td>-</td><td>0.64s</td></tr></table></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Phase 0 Complete. Seeded 3 primary nodes.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import json\n",
|
||||
"import pandas as pd\n",
|
||||
"from semantica.core import Semantica, ConfigManager\n",
|
||||
"from semantica.seed import SeedDataManager\n",
|
||||
"from semantica.vector_store import VectorStore\n",
|
||||
"\n",
|
||||
"# 1. Groq Configuration\n",
|
||||
"import getpass\n",
|
||||
"if \"GROQ_API_KEY\" not in os.environ:\n",
|
||||
" os.environ[\"GROQ_API_KEY\"] = getpass.getpass(\"Enter your Groq API Key: \")\n",
|
||||
"\n",
|
||||
"config_dict = {\n",
|
||||
" \"project_name\": \"Skincare_Intelligence\",\n",
|
||||
" \"embedding\": {\"provider\": \"sentence_transformers\", \"model\": \"all-MiniLM-L6-v2\"}, \n",
|
||||
" \"extraction\": {\n",
|
||||
" \"provider\": \"groq\", \n",
|
||||
" \"model\": \"llama-3.1-8b-instant\", \n",
|
||||
" \"temperature\": 0.0\n",
|
||||
" },\n",
|
||||
" \"inference\": {\n",
|
||||
" \"provider\": \"groq\",\n",
|
||||
" \"model\": \"llama-3.1-70b-versatile\"\n",
|
||||
" },\n",
|
||||
" \"vector_store\": {\"provider\": \"faiss\", \"dimension\": 384},\n",
|
||||
" \"knowledge_graph\": {\"backend\": \"networkx\", \"merge_entities\": True}\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"config = ConfigManager().load_from_dict(config_dict)\n",
|
||||
"core = Semantica(config=config)\n",
|
||||
"vs = VectorStore(backend=\"faiss\", dimension=384)\n",
|
||||
"\n",
|
||||
"config_dict = {\n",
|
||||
" \"project_name\": \"Skincare_Intelligence\",\n",
|
||||
" \"embedding\": {\"provider\": \"sentence_transformers\", \"model\": \"all-MiniLM-L6-v2\"}, \n",
|
||||
" \"extraction\": {\n",
|
||||
" \"provider\": \"groq\", \n",
|
||||
" \"model\": \"llama-3.1-8b-instant\", \n",
|
||||
" \"temperature\": 0.0\n",
|
||||
" },\n",
|
||||
" \"inference\": {\n",
|
||||
" \"provider\": \"groq\",\n",
|
||||
" \"model\": \"llama-3.1-70b-versatile\"\n",
|
||||
" },\n",
|
||||
" \"vector_store\": {\"provider\": \"faiss\", \"dimension\": 384},\n",
|
||||
" \"knowledge_graph\": {\"backend\": \"networkx\", \"merge_entities\": True}\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"config = ConfigManager().load_from_dict(config_dict)\n",
|
||||
"core = Semantica(config=config)\n",
|
||||
"vs = VectorStore(backend=\"faiss\", dimension=384)\n",
|
||||
"\n",
|
||||
"# 2. Seeding Ground Truth\n",
|
||||
"foundation_data = {\n",
|
||||
@@ -112,10 +155,85 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"id": "879f9fd9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Ingesting from: https://makeupandbeautyblog.com/feed\n",
|
||||
"Ingesting from: https://www.westlakedermatology.com/feed\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Failed to fetch feed https://www.westlakedermatology.com/feed: 403 Client Error: Forbidden for url: https://www.westlakedermatology.com/feed\n",
|
||||
"Failed to ingest feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.westlakedermatology.com/feed\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Failed to ingest https://www.westlakedermatology.com/feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.westlakedermatology.com/feed\n",
|
||||
"Ingesting from: https://www.contourderm.com/feed\n",
|
||||
"Ingesting from: https://www.michelegreenmd.com/feed\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Failed to fetch feed https://www.michelegreenmd.com/feed: 403 Client Error: Forbidden for url: https://www.michelegreenmd.com/feed\n",
|
||||
"Failed to ingest feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.michelegreenmd.com/feed\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Failed to ingest https://www.michelegreenmd.com/feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.michelegreenmd.com/feed\n",
|
||||
"Ingesting from: https://www.skincarephysicians.net/blog/feed/\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Failed to fetch feed https://www.skincarephysicians.net/blog/feed/: 403 Client Error: Forbidden for url: https://www.skincarephysicians.net/blog/feed/\n",
|
||||
"Failed to ingest feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.skincarephysicians.net/blog/feed/\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Failed to ingest https://www.skincarephysicians.net/blog/feed/: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.skincarephysicians.net/blog/feed/\n",
|
||||
"Ingesting from: https://www.beautifulwithbrains.com/blog/feed/\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Failed to fetch feed https://www.beautifulwithbrains.com/blog/feed/: 403 Client Error: Forbidden for url: https://www.beautifulwithbrains.com/blog/feed/\n",
|
||||
"Failed to ingest feed: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.beautifulwithbrains.com/blog/feed/\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Failed to ingest https://www.beautifulwithbrains.com/blog/feed/: Failed to fetch feed: 403 Client Error: Forbidden for url: https://www.beautifulwithbrains.com/blog/feed/\n",
|
||||
"Ingesting from: https://www.drbaileyskincare.com/blogs/blog.atom\n",
|
||||
"✅ Phase 1 Complete. Generated 77 semantic chunks.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.ingest import ingest_feed, ingest_file\n",
|
||||
"from semantica.split import EntityAwareChunker\n",
|
||||
@@ -175,10 +293,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"id": "54fa8fad",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Extracting intelligence from chunks...\n",
|
||||
"✅ Phase 2 Complete. Extracted 30 entities.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"\n",
|
||||
@@ -209,10 +336,25 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 5,
|
||||
"id": "97a63461",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Phase 3 Complete. Graph contains 9 resolved entities.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.kg import GraphBuilder, EntityResolver\n",
|
||||
"\n",
|
||||
@@ -244,10 +386,21 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 6,
|
||||
"id": "4edf258c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"🔍 Processing Multi-Hop Query: What ingredients synergize with Retinol to prevent irritation?\n",
|
||||
"\n",
|
||||
"❌ No relevant context found in the knowledge graph.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.context import ContextRetriever\n",
|
||||
"from semantica.reasoning import InferenceEngine\n",
|
||||
@@ -299,10 +452,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 7,
|
||||
"id": "4b52490d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\umap\\__init__.py:9: ImportWarning: Tensorflow not installed; ParametricUMAP will be unavailable\n",
|
||||
" warn(\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.visualization import KGVisualizer\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
|
||||
Reference in New Issue
Block a user