mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Update GraphRAG notebook with real data sources and fix API usage
This commit is contained in:
KaifAhmad1
@@ -18,7 +18,6 @@
|
||||
"- **Complete Pipeline**: From data ingestion to LLM-powered question answering\n",
|
||||
"- **Hybrid Retrieval**: Combines vector similarity search with knowledge graph traversal\n",
|
||||
"- **Multi-hop Reasoning**: Follows relationships across the graph for deeper context\n",
|
||||
"- **20+ Semantica Modules**: Demonstrates comprehensive use of the framework\n",
|
||||
"\n",
|
||||
"**Documentation**: [API Reference](https://semantica.readthedocs.io/concepts/) \u2022 [GraphRAG Guide](https://semantica.readthedocs.io/concepts/)\n",
|
||||
"\n",
|
||||
@@ -44,25 +43,32 @@
|
||||
"```bash\n",
|
||||
"pip install semantica\n",
|
||||
"\n",
|
||||
"# Or with all optional dependencies:\n",
|
||||
"pip install semantica[all]\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"### Additional Dependencies\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"pip install openai anthropic # For LLM integration\n",
|
||||
"pip install jupyter # For running this notebook\n",
|
||||
"```\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!pip install semantica\n"
|
||||
"!pip install -qU semantica\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -124,8 +130,23 @@
|
||||
"\n",
|
||||
"mcp_ingestor = MCPIngestor()\n",
|
||||
"\n",
|
||||
"# 1. Connect to Real MCP Server (e.g., Brave Search)\n",
|
||||
"# Note: You must run the MCP server locally first. \n",
|
||||
"# Command: npx @modelcontextprotocol/server-brave-search\n",
|
||||
"# Then use the appropriate connection method. Since we cannot easily start it here,\n",
|
||||
"# we will wrap it in a try-except block to gracefully handle its absence.\n",
|
||||
"try:\n",
|
||||
" # Note: Replace with your actual MCP server URL if running\n",
|
||||
" # Standard python MCP servers or FastMCP often expose an SSE endpoint\n",
|
||||
" mcp_ingestor.connect(\"brave_search_mcp\", url=\"http://localhost:8000/sse\") \n",
|
||||
" print(\"Successfully connected to Brave Search MCP\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"Note: Could not connect to local MCP server: {e}\")\n",
|
||||
" print(\"Instructions: To use MCP, run a compatible server locally (e.g., npx @modelcontextprotocol/server-brave-search)\")\n",
|
||||
" print(\"Continuing with Web and RSS sources only...\")\n",
|
||||
"\n",
|
||||
"connected_servers = mcp_ingestor.get_connected_servers()\n",
|
||||
"print(f\"Connected MCP Servers: {len(connected_servers)}\")\n",
|
||||
"print(f\"\\nConnected MCP Servers: {len(connected_servers)}\")\n",
|
||||
"for server_name in connected_servers:\n",
|
||||
" print(f\" - {server_name}\")\n",
|
||||
"\n",
|
||||
@@ -137,10 +158,10 @@
|
||||
" print(f\"\\n{server_name}:\")\n",
|
||||
" print(f\" Resources: {len(resources)}\")\n",
|
||||
" print(f\" Tools: {len(tools)}\")\n",
|
||||
" if tools:\n",
|
||||
" print(f\" Available Tools: {[t.name for t in tools]}\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Error connecting to {server_name}: {e}\")\n",
|
||||
"\n",
|
||||
"print(\"\\nNote: Configure your MCP server URLs above to ingest real data\")\n"
|
||||
" print(f\"Error inspecting {server_name}: {e}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -163,8 +184,25 @@
|
||||
"\n",
|
||||
"all_documents = []\n",
|
||||
"\n",
|
||||
"# 1. Ingest via MCP Tools (e.g. Web Search)\n",
|
||||
"if \"brave_search_mcp\" in connected_servers:\n",
|
||||
" try:\n",
|
||||
" print(f\"\\nPerforming Web Search via Brave MCP...\")\n",
|
||||
" # Search for recent relevant topics\n",
|
||||
" search_results = mcp_ingestor.ingest_tool_output(\n",
|
||||
" \"brave_search_mcp\", \n",
|
||||
" tool_name=\"brave_web_search\",\n",
|
||||
" arguments={\"query\": \"latest advancements in GraphRAG and AI Agents\", \"count\": 2}\n",
|
||||
" )\n",
|
||||
" all_documents.append(search_results)\n",
|
||||
" print(f\" Ingested search results via MCP\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\" Error using MCP tool: {e}\")\n",
|
||||
"\n",
|
||||
"# 2. Ingest other MCP resources\n",
|
||||
"if connected_servers:\n",
|
||||
" for server_name in connected_servers:\n",
|
||||
" if server_name == \"brave_search_mcp\": continue # Already handled\n",
|
||||
" try:\n",
|
||||
" print(f\"\\nIngesting from {server_name}...\")\n",
|
||||
" mcp_data = mcp_ingestor.ingest_all_resources(server_name)\n",
|
||||
@@ -200,13 +238,18 @@
|
||||
"source": [
|
||||
"web_ingestor = WebIngestor()\n",
|
||||
"\n",
|
||||
"web_sources = []\n",
|
||||
"# Real documentation and article sources\n",
|
||||
"web_sources = [\n",
|
||||
" \"https://docs.anthropic.com/en/docs/build-with-claude/tool-use\", # Relevant to agents\n",
|
||||
" \"https://python.langchain.com/docs/concepts/#retrieval\" # Relevant to RAG\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"web_documents = []\n",
|
||||
"for url in web_sources:\n",
|
||||
" try:\n",
|
||||
" print(f\"Scraping {url}...\")\n",
|
||||
" docs = web_ingestor.ingest(url)\n",
|
||||
" # Use ingest_url instead of ingest\n",
|
||||
" docs = web_ingestor.ingest_url(url)\n",
|
||||
" if isinstance(docs, list):\n",
|
||||
" web_documents.extend(docs)\n",
|
||||
" else:\n",
|
||||
@@ -238,18 +281,25 @@
|
||||
"source": [
|
||||
"feed_ingestor = FeedIngestor()\n",
|
||||
"\n",
|
||||
"feed_urls = []\n",
|
||||
"# Real Tech News Feeds\n",
|
||||
"feed_urls = [\n",
|
||||
" \"https://techcrunch.com/category/artificial-intelligence/feed/\",\n",
|
||||
" \"http://feeds.bbci.co.uk/news/technology/rss.xml\"\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"feed_documents = []\n",
|
||||
"for feed_url in feed_urls:\n",
|
||||
" try:\n",
|
||||
" print(f\"Fetching feed {feed_url}...\")\n",
|
||||
" feeds = feed_ingestor.ingest(feed_url)\n",
|
||||
" # Use ingest_feed instead of ingest\n",
|
||||
" feeds = feed_ingestor.ingest_feed(feed_url)\n",
|
||||
" if isinstance(feeds, list):\n",
|
||||
" feed_documents.extend(feeds)\n",
|
||||
" else:\n",
|
||||
" feed_documents.append(feeds)\n",
|
||||
" print(f\" Fetched {len(feeds) if isinstance(feeds, list) else 1} feed item(s)\")\n",
|
||||
" # Check for feed items\n",
|
||||
" count = len(feeds.items) if hasattr(feeds, 'items') else (len(feeds) if isinstance(feeds, list) else 1)\n",
|
||||
" print(f\" Fetched {count} feed item(s)\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\" Error fetching feed {feed_url}: {e}\")\n",
|
||||
"\n",
|
||||
@@ -1623,8 +1673,22 @@
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
Reference in New Issue
Block a user