mengzhx/semantica
1
0
Fork 0
mirror of https://github.com/semantica-agi/semantica.git synced 2026-09-10 04:00:35 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
110f6deb1e490a77f67cbd6fa7f47320736bf00a
semantica/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb
T

1381 lines
65 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Real-World Multi-Source Integration: Python Ecosystem Analysis\n",
"\n",
"This cookbook demonstrates how to build a **Knowledge Graph of the Python Ecosystem** using **real-world data** from live sources.\n",
"\n",
"We will ingest data from:\n",
"1. **Official Website (Web)**: `https://www.python.org/` (using `WebIngestor`)\n",
"2. **Package Registry (API)**: PyPI JSON API for `pandas` (using `RESTIngestor` + `FileIngestor`)\n",
"3. **Source Code (Repo)**: CPython GitHub Repository (using Raw Content)\n",
"4. **Database (DB)**: Local SQLite metrics (using `DBIngestor`)\n",
"5. **Live Search (MCP)**: Real-time search via Model Context Protocol (using `MCPIngestor`)\n",
"\n",
"**Goal**: Construct a unified graph linking Python, key libraries, source code, and live context."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Installation & Setup\n",
"!pip install -q semantica requests beautifulsoup4 fastmcp networkx fastembed"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import requests\n",
"import tempfile\n",
"import logging\n",
"from datetime import datetime\n",
"\n",
"# Semantica Imports\n",
"from semantica.ingest import WebIngestor, FileIngestor, RESTIngestor, MCPIngestor, DBIngestor\n",
"from semantica.kg import GraphBuilder\n",
"from semantica.visualization import KGVisualizer\n",
"\n",
"# Setup Workspace\n",
"WORKSPACE_DIR = tempfile.mkdtemp()\n",
"print(f\"Workspace created at: {WORKSPACE_DIR}\")\n",
"\n",
"# Configure Logging to show ingestion progress\n",
"logging.basicConfig(level=logging.INFO, format='%(name)s - %(levelname)s - %(message)s')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Source 1: Official Website (Web Ingestion)\n",
"\n",
"We use `WebIngestor` to crawl the official Python homepage. This demonstrates handling unstructured HTML content."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"--- 1. Ingesting Web Sources: Python.org + Docs + PEPs ---\")\n",
"\n",
"try:\n",
" web_ingestor = WebIngestor(delay=0.5)\n",
"\n",
" web_targets = {\n",
" \"python_home\": \"https://www.python.org/\",\n",
" \"psf\": \"https://www.python.org/psf/\",\n",
" \"python_docs\": \"https://docs.python.org/3/\",\n",
" \"whatsnew_313\": \"https://docs.python.org/3/whatsnew/3.13.html\",\n",
" \"asyncio_docs\": \"https://docs.python.org/3/library/asyncio.html\",\n",
" \"typing_docs\": \"https://docs.python.org/3/library/typing.html\",\n",
" \"pep_703\": \"https://peps.python.org/pep-0703/\",\n",
" \"pep_8\": \"https://peps.python.org/pep-0008/\",\n",
" \"pep_484\": \"https://peps.python.org/pep-0484/\",\n",
" \"packaging_guide\": \"https://packaging.python.org/en/latest/\",\n",
" }\n",
"\n",
" web_pages = {}\n",
" for key, url in web_targets.items():\n",
" page = web_ingestor.ingest_url(url)\n",
" web_pages[key] = page\n",
" title = getattr(page, \"title\", \"\")\n",
" text = getattr(page, \"text\", \"\")\n",
" print(f\"Ingested {key}: {url}\")\n",
" print(f\" Title: {title}\")\n",
" print(f\" Content Length: {len(text)} characters\")\n",
"\n",
" entities = [\n",
" {\n",
" \"id\": \"Python\",\n",
" \"name\": \"Python\",\n",
" \"type\": \"ProgrammingLanguage\",\n",
" \"properties\": {\n",
" \"website\": web_targets[\"python_home\"],\n",
" \"docs\": web_targets[\"python_docs\"],\n",
" },\n",
" \"source\": \"python_web\",\n",
" },\n",
" {\n",
" \"id\": \"CPython\",\n",
" \"name\": \"CPython\",\n",
" \"type\": \"Interpreter\",\n",
" \"properties\": {\"repo_url\": \"https://github.com/python/cpython\"},\n",
" \"source\": \"python_web\",\n",
" },\n",
" {\n",
" \"id\": \"Python Software Foundation\",\n",
" \"name\": \"Python Software Foundation\",\n",
" \"type\": \"Organization\",\n",
" \"properties\": {\"url\": web_targets[\"psf\"]},\n",
" \"source\": \"python_web\",\n",
" },\n",
" {\n",
" \"id\": \"Python Documentation\",\n",
" \"name\": \"Python Documentation\",\n",
" \"type\": \"Documentation\",\n",
" \"properties\": {\"url\": web_targets[\"python_docs\"]},\n",
" \"source\": \"python_docs\",\n",
" },\n",
" {\n",
" \"id\": \"Python Packaging User Guide\",\n",
" \"name\": \"Python Packaging User Guide\",\n",
" \"type\": \"Documentation\",\n",
" \"properties\": {\"url\": web_targets[\"packaging_guide\"]},\n",
" \"source\": \"python_docs\",\n",
" },\n",
" {\n",
" \"id\": \"Python 3.13\",\n",
" \"name\": \"Python 3.13\",\n",
" \"type\": \"SoftwareVersion\",\n",
" \"properties\": {\"release_notes\": web_targets[\"whatsnew_313\"]},\n",
" \"source\": \"python_docs\",\n",
" },\n",
" {\n",
" \"id\": \"asyncio\",\n",
" \"name\": \"asyncio\",\n",
" \"type\": \"StdlibModule\",\n",
" \"properties\": {\"docs_url\": web_targets[\"asyncio_docs\"]},\n",
" \"source\": \"python_docs\",\n",
" },\n",
" {\n",
" \"id\": \"typing\",\n",
" \"name\": \"typing\",\n",
" \"type\": \"StdlibModule\",\n",
" \"properties\": {\"docs_url\": web_targets[\"typing_docs\"]},\n",
" \"source\": \"python_docs\",\n",
" },\n",
" {\n",
" \"id\": \"PEP 703\",\n",
" \"name\": \"PEP 703\",\n",
" \"type\": \"PEP\",\n",
" \"properties\": {\n",
" \"url\": web_targets[\"pep_703\"],\n",
" \"title\": getattr(web_pages.get(\"pep_703\"), \"title\", \"\"),\n",
" },\n",
" \"source\": \"pep_site\",\n",
" },\n",
" {\n",
" \"id\": \"PEP 8\",\n",
" \"name\": \"PEP 8\",\n",
" \"type\": \"PEP\",\n",
" \"properties\": {\"url\": web_targets[\"pep_8\"], \"title\": getattr(web_pages.get(\"pep_8\"), \"title\", \"\")},\n",
" \"source\": \"pep_site\",\n",
" },\n",
" {\n",
" \"id\": \"PEP 484\",\n",
" \"name\": \"PEP 484\",\n",
" \"type\": \"PEP\",\n",
" \"properties\": {\"url\": web_targets[\"pep_484\"], \"title\": getattr(web_pages.get(\"pep_484\"), \"title\", \"\")},\n",
" \"source\": \"pep_site\",\n",
" },\n",
" {\n",
" \"id\": \"Global Interpreter Lock\",\n",
" \"name\": \"Global Interpreter Lock\",\n",
" \"type\": \"Concept\",\n",
" \"properties\": {\"abbrev\": \"GIL\"},\n",
" \"source\": \"pep_site\",\n",
" },\n",
" {\n",
" \"id\": \"Type Hints\",\n",
" \"name\": \"Type Hints\",\n",
" \"type\": \"Concept\",\n",
" \"properties\": {},\n",
" \"source\": \"pep_site\",\n",
" },\n",
" {\n",
" \"id\": \"No-GIL Build\",\n",
" \"name\": \"No-GIL Build\",\n",
" \"type\": \"Feature\",\n",
" \"properties\": {\"description\": \"CPython build configuration without the GIL\"},\n",
" \"source\": \"pep_site\",\n",
" },\n",
" {\n",
" \"id\": \"Free-Threaded Python\",\n",
" \"name\": \"Free-Threaded Python\",\n",
" \"type\": \"Feature\",\n",
" \"properties\": {\"description\": \"Python builds that allow threads without a global lock\"},\n",
" \"source\": \"pep_site\",\n",
" },\n",
" {\n",
" \"id\": \"Packaging\",\n",
" \"name\": \"Packaging\",\n",
" \"type\": \"Concept\",\n",
" \"properties\": {},\n",
" \"source\": \"python_docs\",\n",
" },\n",
" ]\n",
"\n",
" relationships = [\n",
" {\"source\": \"Python Software Foundation\", \"target\": \"Python\", \"type\": \"governs\"},\n",
" {\"source\": \"Python Software Foundation\", \"target\": \"CPython\", \"type\": \"supports\"},\n",
" {\"source\": \"CPython\", \"target\": \"Python\", \"type\": \"implements\"},\n",
" {\"source\": \"Python Documentation\", \"target\": \"Python\", \"type\": \"documents\"},\n",
" {\"source\": \"Python Documentation\", \"target\": \"Python 3.13\", \"type\": \"documents\"},\n",
" {\"source\": \"Python Documentation\", \"target\": \"asyncio\", \"type\": \"documents\"},\n",
" {\"source\": \"Python Documentation\", \"target\": \"typing\", \"type\": \"documents\"},\n",
" {\"source\": \"Python 3.13\", \"target\": \"Python\", \"type\": \"version_of\"},\n",
" {\"source\": \"asyncio\", \"target\": \"Python\", \"type\": \"stdlib_of\"},\n",
" {\"source\": \"typing\", \"target\": \"Python\", \"type\": \"stdlib_of\"},\n",
" {\"source\": \"Python Packaging User Guide\", \"target\": \"Packaging\", \"type\": \"documents\"},\n",
" {\"source\": \"Python Packaging User Guide\", \"target\": \"PyPI\", \"type\": \"mentions\"},\n",
" {\"source\": \"PEP 703\", \"target\": \"No-GIL Build\", \"type\": \"proposes\"},\n",
" {\"source\": \"PEP 703\", \"target\": \"Free-Threaded Python\", \"type\": \"proposes\"},\n",
" {\"source\": \"PEP 703\", \"target\": \"Global Interpreter Lock\", \"type\": \"discusses\"},\n",
" {\"source\": \"No-GIL Build\", \"target\": \"Python 3.13\", \"type\": \"planned_for\"},\n",
" {\"source\": \"PEP 703\", \"target\": \"CPython\", \"type\": \"targets\"},\n",
" {\"source\": \"PEP 8\", \"target\": \"Python\", \"type\": \"style_guide_for\"},\n",
" {\"source\": \"PEP 484\", \"target\": \"Type Hints\", \"type\": \"introduces\"},\n",
" {\"source\": \"Type Hints\", \"target\": \"typing\", \"type\": \"implemented_by\"},\n",
" ]\n",
"\n",
" source_web = {\n",
" \"name\": \"Python Web + Docs + PEPs + Packaging\",\n",
" \"type\": \"unstructured_web\",\n",
" \"entities\": entities,\n",
" \"relationships\": relationships,\n",
" }\n",
"\n",
"except Exception as e:\n",
" print(f\"Web Ingestion Failed: {e}\")\n",
" source_web = {\n",
" \"name\": \"Python Web + Docs + PEPs + Packaging (Offline)\",\n",
" \"type\": \"unstructured_web\",\n",
" \"entities\": [\n",
" {\"id\": \"Python\", \"name\": \"Python\", \"type\": \"ProgrammingLanguage\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"Python Documentation\", \"name\": \"Python Documentation\", \"type\": \"Documentation\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"CPython\", \"name\": \"CPython\", \"type\": \"Interpreter\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"Python Software Foundation\", \"name\": \"Python Software Foundation\", \"type\": \"Organization\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"PEP 703\", \"name\": \"PEP 703\", \"type\": \"PEP\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"Global Interpreter Lock\", \"name\": \"Global Interpreter Lock\", \"type\": \"Concept\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"Type Hints\", \"name\": \"Type Hints\", \"type\": \"Concept\", \"properties\": {}, \"source\": \"offline\"},\n",
" ],\n",
" \"relationships\": [\n",
" {\"source\": \"Python Software Foundation\", \"target\": \"Python\", \"type\": \"governs\"},\n",
" {\"source\": \"CPython\", \"target\": \"Python\", \"type\": \"implements\"},\n",
" {\"source\": \"Python Documentation\", \"target\": \"Python\", \"type\": \"documents\"},\n",
" {\"source\": \"PEP 703\", \"target\": \"Global Interpreter Lock\", \"type\": \"discusses\"},\n",
" {\"source\": \"Type Hints\", \"target\": \"Python\", \"type\": \"feature_of\"}\n",
" ],\n",
" }\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Source 2: Package Registry (API -> File Ingestion)\n",
"\n",
"We fetch live metadata for the `pandas` library from PyPI's JSON API. We save this as a JSON file and then ingest it using `FileIngestor` to demonstrate structured file handling."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"\\n--- 2. Ingesting API Sources: PyPI (pandas, numpy, scipy, matplotlib, scikit-learn, requests, fastapi) ---\")\n",
"\n",
"import re\n",
"from urllib.parse import urlparse\n",
"\n",
"packages = [\"pandas\", \"numpy\", \"scipy\", \"matplotlib\", \"scikit-learn\", \"requests\", \"fastapi\"]\n",
"file_ingestor = FileIngestor()\n",
"api_ingestor = RESTIngestor(timeout=30)\n",
"\n",
"entities = [\n",
" {\"id\": \"PyPI\", \"name\": \"PyPI\", \"type\": \"PackageRegistry\", \"properties\": {\"url\": \"https://pypi.org/\"}, \"source\": \"pypi_api\"},\n",
" {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"pypi_api\"},\n",
"]\n",
"relationships = []\n",
"\n",
"seen_entities = {e[\"id\"] for e in entities}\n",
"\n",
"def _dep_name(req: str) -> str:\n",
" if not req:\n",
" return \"\"\n",
" req = req.split(\";\")[0].strip()\n",
" m = re.match(r\"^([A-Za-z0-9_.-]+)\", req)\n",
" return (m.group(1) if m else \"\").strip()\n",
"\n",
"def _url_entity_id(url: str) -> str:\n",
" return f\"URL::{url.strip()}\"\n",
"\n",
"def _add_url_entity(url: str, label: str, source: str) -> str:\n",
" url = (url or \"\").strip()\n",
" if not url:\n",
" return \"\"\n",
" uid = _url_entity_id(url)\n",
" if uid not in seen_entities:\n",
" entities.append({\"id\": uid, \"name\": label or url, \"type\": \"WebResource\", \"properties\": {\"url\": url}, \"source\": source})\n",
" seen_entities.add(uid)\n",
" return uid\n",
"\n",
"def _is_github(url: str) -> bool:\n",
" try:\n",
" return urlparse(url).netloc.lower().endswith(\"github.com\")\n",
" except Exception:\n",
" return False\n",
"\n",
"try:\n",
" for pkg in packages:\n",
" pypi_url = f\"https://pypi.org/pypi/{pkg}/json\"\n",
" local_json_path = os.path.join(WORKSPACE_DIR, f\"{pkg}_pypi.json\")\n",
"\n",
" api_data = api_ingestor.ingest_endpoint(pypi_url)\n",
" data = api_data.data if isinstance(api_data.data, dict) else {}\n",
"\n",
" with open(local_json_path, \"w\", encoding=\"utf-8\") as f:\n",
" json.dump(data, f)\n",
"\n",
" ingested_file = file_ingestor.ingest_file(local_json_path)\n",
" print(f\"Ingested File: {ingested_file.name} ({ingested_file.size} bytes)\")\n",
"\n",
" info = data.get(\"info\", {})\n",
" project_urls = info.get(\"project_urls\") or {}\n",
"\n",
" if pkg not in seen_entities:\n",
" entities.append(\n",
" {\n",
" \"id\": pkg,\n",
" \"name\": pkg,\n",
" \"type\": \"Library\",\n",
" \"properties\": {\n",
" \"version\": info.get(\"version\"),\n",
" \"summary\": info.get(\"summary\"),\n",
" \"license\": info.get(\"license\"),\n",
" \"requires_python\": info.get(\"requires_python\"),\n",
" },\n",
" \"source\": \"pypi_api\",\n",
" }\n",
" )\n",
" seen_entities.add(pkg)\n",
"\n",
" relationships.append({\"source\": pkg, \"target\": \"PyPI\", \"type\": \"published_on\"})\n",
" relationships.append({\"source\": pkg, \"target\": \"Python\", \"type\": \"written_in\"})\n",
"\n",
" home_page = (info.get(\"home_page\") or \"\").strip()\n",
" if home_page:\n",
" hp_id = _add_url_entity(home_page, f\"{pkg} homepage\", \"pypi_api\")\n",
" if hp_id:\n",
" relationships.append({\"source\": pkg, \"target\": hp_id, \"type\": \"has_homepage\"})\n",
"\n",
" normalized_project_urls = {}\n",
" for k, v in project_urls.items():\n",
" if not k or not v:\n",
" continue\n",
" normalized_project_urls[str(k).strip().lower()] = str(v).strip()\n",
"\n",
" for key, url in normalized_project_urls.items():\n",
" if not url:\n",
" continue\n",
" label = f\"{pkg} {key}\"\n",
" url_id = _add_url_entity(url, label, \"pypi_api\")\n",
" if not url_id:\n",
" continue\n",
" if \"doc\" in key:\n",
" relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"has_documentation\"})\n",
" elif \"bug\" in key or \"issue\" in key:\n",
" relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"issues_at\"})\n",
" elif \"source\" in key or \"github\" in key:\n",
" relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"has_source\"})\n",
" else:\n",
" relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"related_resource\"})\n",
" if _is_github(url):\n",
" relationships.append({\"source\": url_id, \"target\": \"GitHub\", \"type\": \"hosted_on\"})\n",
"\n",
" requires_dist = info.get(\"requires_dist\") or []\n",
" dep_names = []\n",
" for req in requires_dist:\n",
" name = _dep_name(req)\n",
" if name:\n",
" dep_names.append(name)\n",
"\n",
" unique_deps = sorted(set(dep_names))[:12]\n",
" for dep in unique_deps:\n",
" if dep not in seen_entities:\n",
" entities.append({\"id\": dep, \"name\": dep, \"type\": \"Library\", \"properties\": {}, \"source\": \"pypi_requires_dist\"})\n",
" seen_entities.add(dep)\n",
" relationships.append({\"source\": pkg, \"target\": dep, \"type\": \"depends_on\"})\n",
"\n",
" print(f\"Extracted Entity: {pkg} (v{info.get('version')}) with {len(unique_deps)} dependencies (capped)\")\n",
"\n",
" for rel in [\n",
" {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
" {\"source\": \"scipy\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
" {\"source\": \"matplotlib\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
" {\"source\": \"scikit-learn\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
" {\"source\": \"scikit-learn\", \"target\": \"scipy\", \"type\": \"built_on\"},\n",
" ]:\n",
" relationships.append(rel)\n",
"\n",
" source_api = {\n",
" \"name\": \"PyPI Registry\",\n",
" \"type\": \"structured_api\",\n",
" \"entities\": entities,\n",
" \"relationships\": relationships,\n",
" }\n",
"\n",
"except Exception as e:\n",
" print(f\"API Ingestion Failed: {e}\")\n",
" source_api = {\n",
" \"name\": \"PyPI Registry (Offline)\",\n",
" \"type\": \"structured_api\",\n",
" \"entities\": [\n",
" {\"id\": \"PyPI\", \"name\": \"PyPI\", \"type\": \"PackageRegistry\", \"properties\": {\"url\": \"https://pypi.org/\"}, \"source\": \"offline\"},\n",
" {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"offline\"},\n",
" {\"id\": \"pandas\", \"name\": \"pandas\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"numpy\", \"name\": \"numpy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"scipy\", \"name\": \"scipy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"matplotlib\", \"name\": \"matplotlib\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"scikit-learn\", \"name\": \"scikit-learn\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"requests\", \"name\": \"requests\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
" {\"id\": \"fastapi\", \"name\": \"fastapi\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
" ],\n",
" \"relationships\": [\n",
" {\"source\": \"pandas\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
" {\"source\": \"numpy\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
" {\"source\": \"scipy\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
" {\"source\": \"matplotlib\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
" {\"source\": \"scikit-learn\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
" {\"source\": \"requests\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
" {\"source\": \"fastapi\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
" {\"source\": \"pandas\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
" {\"source\": \"numpy\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
" {\"source\": \"scipy\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
" {\"source\": \"matplotlib\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
" {\"source\": \"scikit-learn\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
" {\"source\": \"requests\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
" {\"source\": \"fastapi\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
" {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
" {\"source\": \"scipy\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
" {\"source\": \"scikit-learn\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
" {\"source\": \"scikit-learn\", \"target\": \"scipy\", \"type\": \"built_on\"}\n",
" ],\n",
" }\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Source 3: Source Code Repository (Raw Content)\n",
"\n",
"We fetch the raw `README.rst` from the official CPython GitHub repository. This represents unstructured technical documentation."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"\\n--- 3. Ingesting Repo Sources: CPython + pandas + NumPy + SciPy (GitHub) ---\")\n",
"\n",
"try:\n",
" repo_ingestor = WebIngestor(delay=0.5)\n",
"\n",
" headers = {\"Accept\": \"application/vnd.github+json\", \"User-Agent\": \"semantica-cookbook\"}\n",
"\n",
" repo_targets = {\n",
" \"python/cpython\": {\"readme_raw\": \"https://raw.githubusercontent.com/python/cpython/main/README.rst\", \"library\": \"Python\"},\n",
" \"pandas-dev/pandas\": {\"readme_raw\": \"https://raw.githubusercontent.com/pandas-dev/pandas/main/README.md\", \"library\": \"pandas\"},\n",
" \"numpy/numpy\": {\"readme_raw\": \"https://raw.githubusercontent.com/numpy/numpy/main/README.md\", \"library\": \"numpy\"},\n",
" \"scipy/scipy\": {\"readme_raw\": \"https://raw.githubusercontent.com/scipy/scipy/main/README.rst\", \"library\": \"scipy\"},\n",
" }\n",
"\n",
" entities = [\n",
" {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"github_api\"}\n",
" ]\n",
" relationships = []\n",
"\n",
" for repo_full, cfg in repo_targets.items():\n",
" readme_url = cfg[\"readme_raw\"]\n",
" readme = repo_ingestor.ingest_url(readme_url)\n",
" print(f\"Fetched {repo_full} README: {len(readme.text)} chars\")\n",
"\n",
" repo_url = f\"https://github.com/{repo_full}\"\n",
" entities.append(\n",
" {\n",
" \"id\": repo_full,\n",
" \"name\": repo_full,\n",
" \"type\": \"Repository\",\n",
" \"properties\": {\"repo_url\": repo_url, \"readme_url\": readme_url, \"readme_title\": getattr(readme, \"title\", \"\")},\n",
" \"source\": \"github_raw\",\n",
" }\n",
" )\n",
" relationships.append({\"source\": repo_full, \"target\": \"GitHub\", \"type\": \"hosted_on\"})\n",
"\n",
" owner = repo_full.split(\"/\")[0]\n",
" org_id = f\"GitHubOrg::{owner}\"\n",
" entities.append({\"id\": org_id, \"name\": owner, \"type\": \"Organization\", \"properties\": {\"url\": f\"https://github.com/{owner}\"}, \"source\": \"github_api\"})\n",
" relationships.append({\"source\": repo_full, \"target\": org_id, \"type\": \"owned_by\"})\n",
"\n",
" lib = cfg.get(\"library\")\n",
" if lib:\n",
" relationships.append({\"source\": lib, \"target\": repo_full, \"type\": \"source_code_in\"})\n",
" relationships.append({\"source\": repo_full, \"target\": lib, \"type\": \"source_code_for\"})\n",
"\n",
" repo_api_url = f\"https://api.github.com/repos/{repo_full}\"\n",
" repo_resp = requests.get(repo_api_url, headers=headers, timeout=30)\n",
" if repo_resp.status_code == 200:\n",
" meta = repo_resp.json() or {}\n",
" repo_props = {\n",
" \"stars\": meta.get(\"stargazers_count\"),\n",
" \"forks\": meta.get(\"forks_count\"),\n",
" \"open_issues\": meta.get(\"open_issues_count\"),\n",
" \"language\": meta.get(\"language\"),\n",
" \"updated_at\": meta.get(\"updated_at\"),\n",
" }\n",
" for ent in entities:\n",
" if isinstance(ent, dict) and ent.get(\"id\") == repo_full:\n",
" ent.setdefault(\"properties\", {}).update({k: v for k, v in repo_props.items() if v is not None})\n",
" break\n",
" else:\n",
" print(f\"Repo metadata unavailable for {repo_full} (status {repo_resp.status_code}).\")\n",
"\n",
" relationships.append({\"source\": \"python/cpython\", \"target\": \"CPython\", \"type\": \"repository_for\"})\n",
" relationships.append({\"source\": \"python/cpython\", \"target\": \"Python\", \"type\": \"implements\"})\n",
"\n",
" gh_releases_url = \"https://api.github.com/repos/python/cpython/releases?per_page=8\"\n",
" release_resp = requests.get(gh_releases_url, headers=headers, timeout=30)\n",
" if release_resp.status_code == 200:\n",
" releases = release_resp.json() or []\n",
" for r in releases:\n",
" tag = r.get(\"tag_name\")\n",
" if not tag:\n",
" continue\n",
" release_id = f\"Release::python/cpython::{tag}\"\n",
" entities.append(\n",
" {\n",
" \"id\": release_id,\n",
" \"name\": f\"CPython {tag}\",\n",
" \"type\": \"Release\",\n",
" \"properties\": {\"tag\": tag, \"published_at\": r.get(\"published_at\"), \"url\": r.get(\"html_url\")},\n",
" \"source\": \"github_api\",\n",
" }\n",
" )\n",
" relationships.append({\"source\": release_id, \"target\": \"python/cpython\", \"type\": \"release_of\"})\n",
" if tag.startswith(\"v\") and len(tag) >= 4:\n",
" major_minor = \".\".join(tag.lstrip(\"v\").split(\".\")[:2])\n",
" relationships.append({\"source\": release_id, \"target\": f\"Python {major_minor}\", \"type\": \"implements\"})\n",
" else:\n",
" print(f\"GitHub releases unavailable (status {release_resp.status_code}).\")\n",
"\n",
" source_repo = {\n",
" \"name\": \"GitHub Repos + Metadata + Releases\",\n",
" \"type\": \"unstructured_repo\",\n",
" \"entities\": entities,\n",
" \"relationships\": relationships,\n",
" }\n",
"\n",
"except Exception as e:\n",
" print(f\"Repo Ingestion Failed: {e}\")\n",
" source_repo = {\n",
" \"name\": \"GitHub Repos + Metadata + Releases (Offline)\",\n",
" \"type\": \"unstructured_repo\",\n",
" \"entities\": [\n",
" {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"offline\"},\n",
" {\"id\": \"python/cpython\", \"name\": \"python/cpython\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/python/cpython\"}, \"source\": \"offline\"},\n",
" {\"id\": \"pandas-dev/pandas\", \"name\": \"pandas-dev/pandas\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/pandas-dev/pandas\"}, \"source\": \"offline\"},\n",
" {\"id\": \"numpy/numpy\", \"name\": \"numpy/numpy\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/numpy/numpy\"}, \"source\": \"offline\"},\n",
" {\"id\": \"scipy/scipy\", \"name\": \"scipy/scipy\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/scipy/scipy\"}, \"source\": \"offline\"},\n",
" ],\n",
" \"relationships\": [\n",
" {\"source\": \"python/cpython\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n",
" {\"source\": \"pandas-dev/pandas\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n",
" {\"source\": \"numpy/numpy\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n",
" {\"source\": \"scipy/scipy\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n",
" {\"source\": \"python/cpython\", \"target\": \"Python\", \"type\": \"implements\"},\n",
" {\"source\": \"pandas\", \"target\": \"pandas-dev/pandas\", \"type\": \"source_code_in\"},\n",
" {\"source\": \"numpy\", \"target\": \"numpy/numpy\", \"type\": \"source_code_in\"},\n",
" {\"source\": \"scipy\", \"target\": \"scipy/scipy\", \"type\": \"source_code_in\"}\n",
" ],\n",
" }\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Source 4: Database (SQLite via DBIngestor)\n",
"\n",
"We ingest structured data from a local SQLite database using `DBIngestor`. This demonstrates database connectivity and SQL query extraction.\n",
"\n",
"**Note:** This example creates a small SQLite database inside the temporary workspace so it works offline.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"\\n--- 4. Ingesting Database Source: SQLite (local) ---\")\n",
"\n",
"import sqlite3\n",
"\n",
"db_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem_metrics.sqlite\")\n",
"\n",
"try:\n",
" conn = sqlite3.connect(db_path)\n",
" cur = conn.cursor()\n",
" cur.execute(\n",
" \"\"\"\n",
" CREATE TABLE IF NOT EXISTS library_metrics (\n",
" library TEXT PRIMARY KEY,\n",
" downloads INTEGER,\n",
" stars INTEGER,\n",
" last_updated TEXT\n",
" )\n",
" \"\"\"\n",
" )\n",
"\n",
" cur.execute(\n",
" \"\"\"\n",
" CREATE TABLE IF NOT EXISTS library_categories (\n",
" library TEXT,\n",
" category TEXT,\n",
" PRIMARY KEY (library, category)\n",
" )\n",
" \"\"\"\n",
" )\n",
"\n",
" cur.execute(\n",
" \"\"\"\n",
" CREATE TABLE IF NOT EXISTS library_dependencies (\n",
" library TEXT,\n",
" dependency TEXT,\n",
" relation TEXT,\n",
" PRIMARY KEY (library, dependency, relation)\n",
" )\n",
" \"\"\"\n",
" )\n",
"\n",
" sample_rows = [\n",
" (\"numpy\", 80000000, 28000, datetime.utcnow().isoformat()),\n",
" (\"pandas\", 50000000, 45000, datetime.utcnow().isoformat()),\n",
" (\"scipy\", 20000000, 12000, datetime.utcnow().isoformat()),\n",
" (\"matplotlib\", 25000000, 21000, datetime.utcnow().isoformat()),\n",
" (\"scikit-learn\", 18000000, 60000, datetime.utcnow().isoformat()),\n",
" (\"requests\", 90000000, 52000, datetime.utcnow().isoformat()),\n",
" (\"fastapi\", 22000000, 80000, datetime.utcnow().isoformat()),\n",
" ]\n",
" cur.executemany(\n",
" \"INSERT OR REPLACE INTO library_metrics (library, downloads, stars, last_updated) VALUES (?, ?, ?, ?)\",\n",
" sample_rows,\n",
" )\n",
"\n",
" category_rows = [\n",
" (\"numpy\", \"Numerical\"),\n",
" (\"pandas\", \"Data Analysis\"),\n",
" (\"scipy\", \"Scientific Computing\"),\n",
" (\"matplotlib\", \"Visualization\"),\n",
" (\"scikit-learn\", \"Machine Learning\"),\n",
" (\"requests\", \"Networking\"),\n",
" (\"fastapi\", \"Web\"),\n",
" ]\n",
" cur.executemany(\n",
" \"INSERT OR REPLACE INTO library_categories (library, category) VALUES (?, ?)\",\n",
" category_rows,\n",
" )\n",
"\n",
" dependency_rows = [\n",
" (\"pandas\", \"numpy\", \"depends_on\"),\n",
" (\"scipy\", \"numpy\", \"depends_on\"),\n",
" (\"matplotlib\", \"numpy\", \"depends_on\"),\n",
" (\"scikit-learn\", \"numpy\", \"depends_on\"),\n",
" (\"scikit-learn\", \"scipy\", \"depends_on\"),\n",
" (\"fastapi\", \"pydantic\", \"depends_on\"),\n",
" (\"fastapi\", \"starlette\", \"built_on\"),\n",
" (\"requests\", \"urllib3\", \"depends_on\"),\n",
" (\"requests\", \"certifi\", \"depends_on\"),\n",
" ]\n",
" cur.executemany(\n",
" \"INSERT OR REPLACE INTO library_dependencies (library, dependency, relation) VALUES (?, ?, ?)\",\n",
" dependency_rows,\n",
" )\n",
" conn.commit()\n",
" conn.close()\n",
"\n",
" db_ingestor = DBIngestor()\n",
" sqlite_conn_str = f\"sqlite:///{db_path}\"\n",
"\n",
" rows = db_ingestor.execute_query(\n",
" sqlite_conn_str,\n",
" \"SELECT library, downloads, stars, last_updated FROM library_metrics WHERE downloads >= :min_downloads\",\n",
" min_downloads=1000000,\n",
" )\n",
"\n",
" cat_rows = db_ingestor.execute_query(\n",
" sqlite_conn_str,\n",
" \"SELECT library, category FROM library_categories\",\n",
" )\n",
"\n",
" dep_rows = db_ingestor.execute_query(\n",
" sqlite_conn_str,\n",
" \"SELECT library, dependency, relation FROM library_dependencies\",\n",
" )\n",
"\n",
" if not rows and not cat_rows and not dep_rows:\n",
" raise RuntimeError(\"No data found in SQLite database\")\n",
"\n",
" entities = []\n",
" relationships = []\n",
"\n",
" entities.append({\"id\": \"SQLite\", \"name\": \"SQLite\", \"type\": \"Database\", \"properties\": {\"path\": db_path}, \"source\": \"sqlite_db\"})\n",
" for row in rows:\n",
" lib = row.get(\"library\")\n",
" if not lib:\n",
" continue\n",
" entities.append({\"id\": lib, \"name\": lib, \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db\"})\n",
" metric_id = f\"{lib}::metrics\"\n",
" entities.append(\n",
" {\n",
" \"id\": metric_id,\n",
" \"name\": f\"{lib} metrics\",\n",
" \"type\": \"LibraryMetrics\",\n",
" \"properties\": {\n",
" \"downloads\": row.get(\"downloads\"),\n",
" \"stars\": row.get(\"stars\"),\n",
" \"last_updated\": row.get(\"last_updated\"),\n",
" },\n",
" \"source\": \"sqlite_db\",\n",
" }\n",
" )\n",
" relationships.append({\"source\": metric_id, \"target\": lib, \"type\": \"metrics_for\"})\n",
" relationships.append({\"source\": lib, \"target\": \"Python\", \"type\": \"ecosystem_of\"})\n",
" relationships.append({\"source\": metric_id, \"target\": \"SQLite\", \"type\": \"stored_in\"})\n",
"\n",
" for row in cat_rows:\n",
" lib = row.get(\"library\")\n",
" cat = row.get(\"category\")\n",
" if not lib or not cat:\n",
" continue\n",
" cat_id = f\"Category::{cat}\"\n",
" entities.append({\"id\": cat_id, \"name\": cat, \"type\": \"Category\", \"properties\": {}, \"source\": \"sqlite_db\"})\n",
" relationships.append({\"source\": lib, \"target\": cat_id, \"type\": \"categorized_as\"})\n",
"\n",
" for row in dep_rows:\n",
" lib = row.get(\"library\")\n",
" dep = row.get(\"dependency\")\n",
" rel = row.get(\"relation\") or \"depends_on\"\n",
" if not lib or not dep:\n",
" continue\n",
" entities.append({\"id\": dep, \"name\": dep, \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db\"})\n",
" relationships.append({\"source\": lib, \"target\": dep, \"type\": rel})\n",
"\n",
" print(f\"Ingested {len(entities)} rows from SQLite metrics table\")\n",
" source_db = {\n",
" \"name\": \"SQLite Metrics\",\n",
" \"type\": \"database\",\n",
" \"entities\": entities,\n",
" \"relationships\": relationships,\n",
" }\n",
"\n",
"except Exception as e:\n",
" print(f\"Database ingestion via DBIngestor skipped: {e}\")\n",
" entities = [\n",
" {\"id\": \"SQLite\", \"name\": \"SQLite\", \"type\": \"Database\", \"properties\": {\"path\": db_path}, \"source\": \"sqlite_db_offline\"},\n",
" {\"id\": \"pandas\", \"name\": \"pandas\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n",
" {\"id\": \"pandas::metrics\", \"name\": \"pandas metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 50000000, \"stars\": 45000}, \"source\": \"sqlite_db_offline\"},\n",
" {\"id\": \"numpy\", \"name\": \"numpy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n",
" {\"id\": \"numpy::metrics\", \"name\": \"numpy metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 80000000, \"stars\": 28000}, \"source\": \"sqlite_db_offline\"},\n",
" {\"id\": \"scipy\", \"name\": \"scipy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n",
" {\"id\": \"scipy::metrics\", \"name\": \"scipy metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 20000000, \"stars\": 12000}, \"source\": \"sqlite_db_offline\"},\n",
" {\"id\": \"fastapi\", \"name\": \"fastapi\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n",
" {\"id\": \"fastapi::metrics\", \"name\": \"fastapi metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 22000000, \"stars\": 80000}, \"source\": \"sqlite_db_offline\"},\n",
" ]\n",
" relationships = [\n",
" {\"source\": \"pandas::metrics\", \"target\": \"pandas\", \"type\": \"metrics_for\"},\n",
" {\"source\": \"numpy::metrics\", \"target\": \"numpy\", \"type\": \"metrics_for\"},\n",
" {\"source\": \"scipy::metrics\", \"target\": \"scipy\", \"type\": \"metrics_for\"},\n",
" {\"source\": \"fastapi::metrics\", \"target\": \"fastapi\", \"type\": \"metrics_for\"},\n",
" {\"source\": \"pandas\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n",
" {\"source\": \"numpy\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n",
" {\"source\": \"scipy\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n",
" {\"source\": \"fastapi\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n",
" {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"depends_on\"},\n",
" {\"source\": \"fastapi\", \"target\": \"pydantic\", \"type\": \"depends_on\"}\n",
" ]\n",
" source_db = {\n",
" \"name\": \"SQLite Metrics (Offline)\",\n",
" \"type\": \"database\",\n",
" \"entities\": entities,\n",
" \"relationships\": relationships,\n",
" }\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Source 5: Model Context Protocol (MCP)\n",
"\n",
"We attempt to connect to a local MCP server (e.g., a Web Search tool) to get live context. \n",
"\n",
"**Note:** If no MCP server is running at `localhost:8000`, this section will gracefully fallback to simulated data, but the code provided is production-ready for MCP integration.\n",
"\n",
"Useful MCP server directories / references:\n",
"- https://github.com/modelcontextprotocol/servers\n",
"- https://glama.ai/mcp/servers\n",
"- https://github.com/punkpeye/awesome-mcp-servers\n",
"- https://github.com/wong2/awesome-mcp-servers\n",
"- https://mcp.so\n",
"- Brave Search MCP Server: https://github.com/brave/brave-search-mcp-server"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"\\n--- 5. Ingesting via MCP (Model Context Protocol) ---\")\n",
"\n",
"mcp_server_urls = [\n",
" \"http://localhost:8000/mcp\",\n",
" \"http://localhost:8000/sse\",\n",
" \"http://127.0.0.1:8000/mcp\",\n",
" \"http://127.0.0.1:8000/sse\",\n",
" \"http://localhost:8080/mcp\",\n",
" \"http://localhost:8080/sse\",\n",
" \"http://127.0.0.1:8080/mcp\",\n",
" \"http://127.0.0.1:8080/sse\",\n",
"]\n",
"\n",
"try:\n",
" mcp_client_logger = logging.getLogger(\"semantica.mcp_client\")\n",
" mcp_ingestor_logger = logging.getLogger(\"semantica.mcp_ingestor\")\n",
" prev_client_level = mcp_client_logger.level\n",
" prev_ingestor_level = mcp_ingestor_logger.level\n",
" mcp_client_logger.setLevel(logging.CRITICAL)\n",
" mcp_ingestor_logger.setLevel(logging.CRITICAL)\n",
"\n",
" mcp = MCPIngestor()\n",
" connected_url = None\n",
" for url in mcp_server_urls:\n",
" try:\n",
" mcp.connect(\"web_search\", url=url)\n",
" connected_url = url\n",
" break\n",
" except Exception:\n",
" continue\n",
" if not connected_url:\n",
" raise RuntimeError(\"No MCP server reachable on known local SSE endpoints\")\n",
"\n",
" print(f\"Connected to MCP Server at {connected_url}\")\n",
" \n",
" payload = {\"query\": \"latest python 3.13 features\"}\n",
" mcp_result = None\n",
" for tool_name in [\"search\", \"brave_web_search\", \"brave_local_search\"]:\n",
" try:\n",
" mcp_result = mcp.ingest_tool_output(\"web_search\", tool_name, payload)\n",
" break\n",
" except Exception:\n",
" continue\n",
" if mcp_result is None:\n",
" raise RuntimeError(\"No compatible MCP search tool found\")\n",
" search_results = getattr(mcp_result, \"content\", mcp_result)\n",
" if not isinstance(search_results, dict):\n",
" raise RuntimeError(\"MCP tool output content was not a dict\")\n",
" \n",
" print(\"Received Live Data from MCP.\")\n",
" \n",
" raw_entities = search_results.get(\"entities\", []) or []\n",
" raw_relationships = search_results.get(\"relationships\", []) or []\n",
" normalized_entities = []\n",
" for ent in raw_entities:\n",
" if not isinstance(ent, dict):\n",
" continue\n",
" name = ent.get(\"name\") or ent.get(\"text\") or ent.get(\"id\")\n",
" if not name:\n",
" continue\n",
" normalized_entities.append(\n",
" {\n",
" \"id\": ent.get(\"id\") or name,\n",
" \"name\": name,\n",
" \"type\": ent.get(\"type\") or ent.get(\"label\") or \"Entity\",\n",
" \"properties\": ent.get(\"properties\") or ent.get(\"metadata\") or {},\n",
" \"source\": \"mcp_live\",\n",
" }\n",
" )\n",
"\n",
" normalized_relationships = []\n",
" for rel in raw_relationships:\n",
" if not isinstance(rel, dict):\n",
" continue\n",
" src = rel.get(\"source\") or rel.get(\"subject\")\n",
" tgt = rel.get(\"target\") or rel.get(\"object\")\n",
" rtype = rel.get(\"type\") or rel.get(\"label\") or rel.get(\"predicate\")\n",
" if not src or not tgt or not rtype:\n",
" continue\n",
" normalized_relationships.append({\"source\": src, \"target\": tgt, \"type\": rtype, \"properties\": rel.get(\"properties\") or rel.get(\"metadata\") or {}})\n",
"\n",
" endpoint_ids = {e.get(\"id\") for e in normalized_entities if isinstance(e, dict) and e.get(\"id\")}\n",
" for rel in normalized_relationships:\n",
" src = rel.get(\"source\")\n",
" tgt = rel.get(\"target\")\n",
" for node_id in [src, tgt]:\n",
" if node_id and node_id not in endpoint_ids:\n",
" normalized_entities.append({\"id\": node_id, \"name\": node_id, \"type\": \"Entity\", \"properties\": {}, \"source\": \"mcp_live\"})\n",
" endpoint_ids.add(node_id)\n",
"\n",
" source_mcp = {\n",
" \"name\": \"MCP Search\",\n",
" \"type\": \"agent_tool\",\n",
" \"entities\": normalized_entities,\n",
" \"relationships\": normalized_relationships,\n",
" \"source\": \"mcp_live\"\n",
" }\n",
"\n",
"except Exception as e:\n",
" print(\"MCP Server not detected. Using simulated 'Live Search' data.\")\n",
" print(\"To enable: start a local MCP server (commonly `http://localhost:8000/mcp` or `http://localhost:8000/sse`)\")\n",
" \n",
" source_mcp = {\n",
" \"name\": \"MCP Search (Simulated)\",\n",
" \"type\": \"agent_tool\",\n",
" \"entities\": [\n",
" {\n",
" \"id\": \"Python 3.13\",\n",
" \"name\": \"Python 3.13\",\n",
" \"type\": \"SoftwareVersion\",\n",
" \"properties\": {\"status\": \"In Development\", \"feature\": \"No-GIL Build\"},\n",
" \"source\": \"mcp_simulated\"\n",
" }\n",
" ],\n",
" \"relationships\": [\n",
" {\"source\": \"Python 3.13\", \"target\": \"Python\", \"type\": \"version_of\"}\n",
" ]\n",
" }\n",
"finally:\n",
" if 'mcp_client_logger' in locals() and 'prev_client_level' in locals():\n",
" mcp_client_logger.setLevel(prev_client_level)\n",
" if 'mcp_ingestor_logger' in locals() and 'prev_ingestor_level' in locals():\n",
" mcp_ingestor_logger.setLevel(prev_ingestor_level)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Phase 5: Knowledge Graph Construction\n",
"\n",
"We merge all these real-world data points into a single Knowledge Graph."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"\\n--- Building Knowledge Graph ---\")\n",
"\n",
"all_sources = [source_web, source_api, source_repo, source_db, source_mcp]\n",
"\n",
"builder = GraphBuilder(merge_entities=True, resolve_conflicts=True)\n",
"\n",
"kg = builder.build(sources=all_sources) or {}\n",
"kg.setdefault(\"entities\", [])\n",
"kg.setdefault(\"relationships\", [])\n",
"\n",
"entity_ids = set()\n",
"for node in kg.get(\"entities\", []):\n",
" if isinstance(node, dict):\n",
" node_id = node.get(\"id\") or node.get(\"entity_id\") or node.get(\"name\")\n",
" if node_id:\n",
" entity_ids.add(node_id)\n",
"\n",
"missing_ids = set()\n",
"for rel in kg.get(\"relationships\", []):\n",
" if not isinstance(rel, dict):\n",
" continue\n",
" src = rel.get(\"source\") or rel.get(\"subject\")\n",
" tgt = rel.get(\"target\") or rel.get(\"object\")\n",
" if src and src not in entity_ids:\n",
" missing_ids.add(src)\n",
" if tgt and tgt not in entity_ids:\n",
" missing_ids.add(tgt)\n",
"\n",
"for mid in sorted(missing_ids):\n",
" kg[\"entities\"].append({\"id\": mid, \"name\": mid, \"type\": \"Entity\", \"properties\": {}, \"source\": \"auto\"})\n",
" entity_ids.add(mid)\n",
"\n",
"print(f\"Graph Statistics:\")\n",
"print(f\"Nodes: {len(kg.get('entities', []))}\")\n",
"print(f\"Edges: {len(kg.get('relationships', []))}\")\n",
"\n",
"# List all nodes to verify integration\n",
"print(\"\\nEntities in Graph:\")\n",
"for node in kg.get('entities', []):\n",
" name = node.get('name') or node.get('label') or node.get('id')\n",
" ntype = node.get('type') or node.get('label') or 'Entity'\n",
" print(f\"- {name} ({ntype})\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Phase 6: Advanced Graph Analytics\n",
"\n",
"We can perform network analysis on the constructed graph to find key entities. Here, we calculate **Degree Centrality** to identify the most connected nodes."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.kg import CentralityCalculator, CommunityDetector\n",
"\n",
"print(\"\\n--- Running Graph Analytics (Semantica) ---\")\n",
"\n",
"# 1. Centrality Analysis\n",
"centrality_calc = CentralityCalculator()\n",
"degree_centrality = centrality_calc.calculate_degree_centrality(kg)\n",
"\n",
"print(\"Top 5 Most Central Entities (Degree):\")\n",
"for ranking in degree_centrality.get(\"rankings\", [])[:5]:\n",
" print(f\"- {ranking['node']}: {ranking['score']:.4f}\")\n",
"\n",
"# 2. Community Detection\n",
"try:\n",
" detector = CommunityDetector()\n",
" result = detector.detect_communities(kg, algorithm=\"louvain\") or {}\n",
"\n",
" communities_raw = result.get(\"communities\")\n",
" if communities_raw is None:\n",
" communities_raw = result.get(\"node_assignments\")\n",
"\n",
" communities = []\n",
" if isinstance(communities_raw, list):\n",
" for c in communities_raw:\n",
" if isinstance(c, (list, tuple, set)):\n",
" communities.append(list(c))\n",
" elif isinstance(c, dict):\n",
" communities.append(list(c.keys()))\n",
" else:\n",
" communities.append([str(c)])\n",
" elif isinstance(communities_raw, dict):\n",
" comm_map = {}\n",
" for node_id, comm_id in communities_raw.items():\n",
" comm_map.setdefault(comm_id, []).append(node_id)\n",
" # Sort by community ID safely\n",
" sorted_keys = sorted(comm_map.keys(), key=lambda x: str(x))\n",
" communities = [comm_map[k] for k in sorted_keys]\n",
"\n",
" print(f\"\\nDetected {len(communities)} Communities:\")\n",
" for i, comm in enumerate(communities[:3]):\n",
" # Ensure elements are strings\n",
" sample = [str(x) for x in list(comm)[:5]]\n",
" print(f\"Community {i+1}: {', '.join(sample)}...\")\n",
"except Exception as e:\n",
" print(f\"\\nCommunity detection skipped: {e}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Phase 7: Semantic Querying\n",
"\n",
"We can query the graph to find specific relationships, such as tracing the lineage of Python versions or finding libraries related to Python."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.kg import ConnectivityAnalyzer\n",
"\n",
"print(\"\\n--- Semantic Querying & Path Finding ---\")\n",
"\n",
"kg = globals().get(\"kg\")\n",
"kg = kg if isinstance(kg, dict) else {}\n",
"kg.setdefault(\"entities\", [])\n",
"kg.setdefault(\"relationships\", [])\n",
"\n",
"analyzer = ConnectivityAnalyzer()\n",
"\n",
"# 1. Check Connectivity\n",
"connectivity = analyzer.analyze_connectivity(kg)\n",
"print(f\"Graph Connected: {connectivity.get('is_connected')}\")\n",
"print(f\"Connected Components: {connectivity.get('num_components')}\")\n",
"\n",
"# 2. Find Path between Entities\n",
"source = \"pandas\"\n",
"target = \"Python\"\n",
"\n",
"print(f\"\\nFinding path from '{source}' to '{target}':\")\n",
"try:\n",
" path_result = analyzer.calculate_shortest_paths(kg, source=source, target=target)\n",
" \n",
" if path_result.get(\"exists\"):\n",
" path = path_result[\"path\"]\n",
" print(f\"Path Found: {' -> '.join(path)}\")\n",
" print(f\"Distance: {path_result['distance']}\")\n",
" else:\n",
" print(\"No path found.\")\n",
"except Exception as e:\n",
" print(f\"Path finding error: {e}\")\n",
" # Fallback to simple neighbor check\n",
" print(\"Falling back to direct neighbor check...\")\n",
" found = False\n",
" for rel in kg.get('relationships', []):\n",
" if rel.get('source') == source and rel.get('target') == target:\n",
" print(f\" - [{rel.get('type', 'related_to')}] -> {target}\")\n",
" found = True\n",
" if not found:\n",
" print(\"No direct edge found.\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Phase 8: Export & Persistence\n",
"\n",
"Finally, we save the constructed Knowledge Graph to a JSON file for external use or visualization in other tools."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.export import GraphExporter\n",
"\n",
"print(\"\\n--- Exporting Knowledge Graph ---\")\n",
"\n",
"# Use Semantica's GraphExporter for robust export\n",
"exporter = GraphExporter(format=\"json\", include_attributes=True)\n",
"export_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem_kg.json\")\n",
"\n",
"try:\n",
" exporter.export_knowledge_graph(kg, export_path)\n",
" print(f\"Graph saved to: {export_path}\")\n",
" \n",
" # Optional: Export to GraphML for Gephi\n",
" graphml_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem.graphml\")\n",
" exporter_ml = GraphExporter(format=\"graphml\")\n",
" exporter_ml.export_knowledge_graph(kg, graphml_path)\n",
" print(f\"GraphML saved to: {graphml_path} (Ready for Gephi/Cytoscape)\")\n",
" \n",
"except Exception as e:\n",
" print(f\"Export failed: {e}\")\n",
" # Fallback\n",
" import json\n",
" with open(export_path, \"w\") as f:\n",
" json.dump(kg, f, default=str)\n",
" print(\"Fallback export used.\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Phase 9: Context Engineering for LLM Agents\n",
"\n",
"This is the **critical step** where we turn our Knowledge Graph into a queryable **Context** for AI Agents.\n",
"We use the `AgentContext` module to ingest our graph and enable **Retrieval Augmented Generation (RAG)** capabilities."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"\\n--- Context Engineering ---\")\n",
"\n",
"import json\n",
"from semantica.context import AgentContext, ContextGraph\n",
"from semantica.vector_store import VectorStore\n",
"\n",
"kg = globals().get(\"kg\")\n",
"kg = kg if isinstance(kg, dict) else {}\n",
"kg.setdefault(\"entities\", [])\n",
"kg.setdefault(\"relationships\", [])\n",
"\n",
"# 1. Initialize Vector Store (with FastEmbed support)\n",
"# We try to use the high-performance 'fastembed' model if available\n",
"vs = VectorStore(backend=\"inmemory\", dimension=384)\n",
"try:\n",
" if hasattr(vs, \"embedder\") and vs.embedder:\n",
" print(\"Initializing FastEmbed model (BAAI/bge-small-en-v1.5)...\")\n",
" vs.embedder.set_text_model(method=\"fastembed\", model_name=\"BAAI/bge-small-en-v1.5\")\n",
"except Exception as e:\n",
" print(f\"FastEmbed not available ({e}). Using fallback keyword/random embedding.\")\n",
" print(\"Tip: Run '!pip install fastembed' and restart kernel for better results.\")\n",
"\n",
"# 2. Initialize Context Graph\n",
"cg = ContextGraph()\n",
"\n",
"# 3. Create the Agent Context\n",
"# This binds the Vector Store (Content) and Knowledge Graph (Structure) together\n",
"context = AgentContext(vector_store=vs, knowledge_graph=cg)\n",
"\n",
"# 4. Ingest Graph Structure\n",
"# We map our generic KG data to the specific structure ContextGraph expects\n",
"print(\"Building Context Graph structure...\")\n",
"\n",
"kg_entities = kg.get(\"entities\", []) if isinstance(kg, dict) else []\n",
"kg_relationships = kg.get(\"relationships\", []) if isinstance(kg, dict) else []\n",
"\n",
"context_entities = []\n",
"for node in kg_entities:\n",
" if not isinstance(node, dict):\n",
" continue\n",
" name = node.get(\"name\") or node.get(\"id\")\n",
" if not name:\n",
" continue\n",
" context_entities.append(\n",
" {\n",
" \"id\": node.get(\"id\") or name,\n",
" \"text\": name,\n",
" \"type\": node.get(\"type\") or \"Entity\",\n",
" \"metadata\": node.get(\"properties\") or {},\n",
" }\n",
" )\n",
"\n",
"context_relationships = []\n",
"for rel in kg_relationships:\n",
" if not isinstance(rel, dict):\n",
" continue\n",
" src = rel.get(\"source\")\n",
" tgt = rel.get(\"target\")\n",
" rtype = rel.get(\"type\")\n",
" if not src or not tgt or not rtype:\n",
" continue\n",
" context_relationships.append({\"source_id\": src, \"target_id\": tgt, \"type\": rtype})\n",
"\n",
"cg.build_from_entities_and_relationships(context_entities, context_relationships)\n",
"print(f\"Context Graph: {cg.stats()['node_count']} nodes, {cg.stats()['edge_count']} edges\")\n",
"\n",
"# 5. Index Entities for Vector Retrieval (Batch Store)\n",
"# We transform entities into \"documents\" so the Vector Store can index them.\n",
"# This allows the Agent to \"find\" the graph nodes using semantic search.\n",
"print(\"Indexing entities into Vector Store...\")\n",
"\n",
"entity_documents = []\n",
"for node in kg_entities:\n",
" # Create a rich textual description for the embedding\n",
" if not isinstance(node, dict):\n",
" continue\n",
" name = node.get(\"name\") or node.get(\"id\")\n",
" if not name:\n",
" continue\n",
" description = f\"{name} is a {node.get('type', 'Entity')}.\"\n",
" props = node.get('properties', {})\n",
" if props:\n",
" # Flatten properties into string for better semantic context\n",
" prop_str = \", \".join([f\"{k}: {v}\" for k,v in props.items() if isinstance(v, (str, int, float))])\n",
" description += f\" Properties: {prop_str}\"\n",
" \n",
" # Create document object\n",
" entity_documents.append({\n",
" \"content\": description,\n",
" \"metadata\": {\n",
" \"source\": \"knowledge_graph\",\n",
" \"original_id\": node.get(\"id\") or name,\n",
" \"type\": node.get('type', 'Entity')\n",
" }\n",
" })\n",
"\n",
"# Batch store all entity descriptions\n",
"# extract_entities=False because we are storing the entities themselves\n",
"context.store(entity_documents, extract_entities=False)\n",
"print(f\"Successfully indexed {len(entity_documents)} entities.\")\n",
"\n",
"# 6. Simulate an Agent Query (GraphRAG)\n",
"query = \"pandas library\"\n",
"print(f\"\\nAgent Query: '{query}'\")\n",
"\n",
"# Retrieve context using Hybrid Search (Vector + Graph)\n",
"results = context.retrieve(\n",
" query,\n",
" use_graph=True,\n",
" expand_graph=True, # Follow edges to get related context (e.g. pandas -> Python)\n",
" max_results=3\n",
")\n",
"\n",
"print(\"\\n--- Retrieved Context for LLM ---\")\n",
"if results:\n",
" for res in results:\n",
" # Access dictionary keys instead of attributes\n",
" print(f\"Content: {res['content']}\")\n",
" print(f\"Score: {res['score']:.4f}\")\n",
" \n",
" # Check for related entities in the dictionary\n",
" if 'related_entities' in res and res['related_entities']:\n",
" # related_entities is a list of dicts, we want the 'text' or 'id'\n",
" related = [e.get('text', e.get('id', 'Unknown')) for e in res['related_entities']]\n",
" print(f\"Graph Expansion: {', '.join(related)}\")\n",
" print(\"-\" * 30)\n",
"else:\n",
" print(\"No context retrieved.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"\\n--- Visualizing Graph ---\")\n",
"visualizer = KGVisualizer(layout=\"force\", color_scheme=\"vibrant\")\n",
"fig = visualizer.visualize_network(kg, output=\"interactive\")\n",
"fig.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue View Git Blame Copy Permalink