{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Real-World Multi-Source Integration: Python Ecosystem Analysis\n", "\n", "This cookbook demonstrates how to build a **Knowledge Graph of the Python Ecosystem** using **real-world data** from live sources.\n", "\n", "We will ingest data from:\n", "1. **Official Website (Web)**: `https://www.python.org/` (using `WebIngestor`)\n", "2. **Package Registry (API)**: PyPI JSON API for `pandas` (using `RESTIngestor` + `FileIngestor`)\n", "3. **Source Code (Repo)**: CPython GitHub Repository (using Raw Content)\n", "4. **Database (DB)**: Local SQLite metrics (using `DBIngestor`)\n", "5. **Live Search (MCP)**: Real-time search via Model Context Protocol (using `MCPIngestor`)\n", "\n", "**Goal**: Construct a unified graph linking Python, key libraries, source code, and live context." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Installation & Setup\n", "!pip install -q semantica requests beautifulsoup4 fastmcp networkx fastembed" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import requests\n", "import tempfile\n", "import logging\n", "from datetime import datetime\n", "\n", "# Semantica Imports\n", "from semantica.ingest import WebIngestor, FileIngestor, RESTIngestor, MCPIngestor, DBIngestor\n", "from semantica.kg import GraphBuilder\n", "from semantica.visualization import KGVisualizer\n", "\n", "# Setup Workspace\n", "WORKSPACE_DIR = tempfile.mkdtemp()\n", "print(f\"Workspace created at: {WORKSPACE_DIR}\")\n", "\n", "# Configure Logging to show ingestion progress\n", "logging.basicConfig(level=logging.INFO, format='%(name)s - %(levelname)s - %(message)s')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Source 1: Official Website (Web Ingestion)\n", "\n", "We use `WebIngestor` to crawl the official Python homepage. This demonstrates handling unstructured HTML content." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(\"--- 1. Ingesting Web Sources: Python.org + Docs + PEPs ---\")\n", "\n", "try:\n", " web_ingestor = WebIngestor(delay=0.5)\n", "\n", " web_targets = {\n", " \"python_home\": \"https://www.python.org/\",\n", " \"psf\": \"https://www.python.org/psf/\",\n", " \"python_docs\": \"https://docs.python.org/3/\",\n", " \"whatsnew_313\": \"https://docs.python.org/3/whatsnew/3.13.html\",\n", " \"asyncio_docs\": \"https://docs.python.org/3/library/asyncio.html\",\n", " \"typing_docs\": \"https://docs.python.org/3/library/typing.html\",\n", " \"pep_703\": \"https://peps.python.org/pep-0703/\",\n", " \"pep_8\": \"https://peps.python.org/pep-0008/\",\n", " \"pep_484\": \"https://peps.python.org/pep-0484/\",\n", " \"packaging_guide\": \"https://packaging.python.org/en/latest/\",\n", " }\n", "\n", " web_pages = {}\n", " for key, url in web_targets.items():\n", " page = web_ingestor.ingest_url(url)\n", " web_pages[key] = page\n", " title = getattr(page, \"title\", \"\")\n", " text = getattr(page, \"text\", \"\")\n", " print(f\"Ingested {key}: {url}\")\n", " print(f\" Title: {title}\")\n", " print(f\" Content Length: {len(text)} characters\")\n", "\n", " entities = [\n", " {\n", " \"id\": \"Python\",\n", " \"name\": \"Python\",\n", " \"type\": \"ProgrammingLanguage\",\n", " \"properties\": {\n", " \"website\": web_targets[\"python_home\"],\n", " \"docs\": web_targets[\"python_docs\"],\n", " },\n", " \"source\": \"python_web\",\n", " },\n", " {\n", " \"id\": \"CPython\",\n", " \"name\": \"CPython\",\n", " \"type\": \"Interpreter\",\n", " \"properties\": {\"repo_url\": \"https://github.com/python/cpython\"},\n", " \"source\": \"python_web\",\n", " },\n", " {\n", " \"id\": \"Python Software Foundation\",\n", " \"name\": \"Python Software Foundation\",\n", " \"type\": \"Organization\",\n", " \"properties\": {\"url\": web_targets[\"psf\"]},\n", " \"source\": \"python_web\",\n", " },\n", " {\n", " \"id\": \"Python Documentation\",\n", " \"name\": \"Python Documentation\",\n", " \"type\": \"Documentation\",\n", " \"properties\": {\"url\": web_targets[\"python_docs\"]},\n", " \"source\": \"python_docs\",\n", " },\n", " {\n", " \"id\": \"Python Packaging User Guide\",\n", " \"name\": \"Python Packaging User Guide\",\n", " \"type\": \"Documentation\",\n", " \"properties\": {\"url\": web_targets[\"packaging_guide\"]},\n", " \"source\": \"python_docs\",\n", " },\n", " {\n", " \"id\": \"Python 3.13\",\n", " \"name\": \"Python 3.13\",\n", " \"type\": \"SoftwareVersion\",\n", " \"properties\": {\"release_notes\": web_targets[\"whatsnew_313\"]},\n", " \"source\": \"python_docs\",\n", " },\n", " {\n", " \"id\": \"asyncio\",\n", " \"name\": \"asyncio\",\n", " \"type\": \"StdlibModule\",\n", " \"properties\": {\"docs_url\": web_targets[\"asyncio_docs\"]},\n", " \"source\": \"python_docs\",\n", " },\n", " {\n", " \"id\": \"typing\",\n", " \"name\": \"typing\",\n", " \"type\": \"StdlibModule\",\n", " \"properties\": {\"docs_url\": web_targets[\"typing_docs\"]},\n", " \"source\": \"python_docs\",\n", " },\n", " {\n", " \"id\": \"PEP 703\",\n", " \"name\": \"PEP 703\",\n", " \"type\": \"PEP\",\n", " \"properties\": {\n", " \"url\": web_targets[\"pep_703\"],\n", " \"title\": getattr(web_pages.get(\"pep_703\"), \"title\", \"\"),\n", " },\n", " \"source\": \"pep_site\",\n", " },\n", " {\n", " \"id\": \"PEP 8\",\n", " \"name\": \"PEP 8\",\n", " \"type\": \"PEP\",\n", " \"properties\": {\"url\": web_targets[\"pep_8\"], \"title\": getattr(web_pages.get(\"pep_8\"), \"title\", \"\")},\n", " \"source\": \"pep_site\",\n", " },\n", " {\n", " \"id\": \"PEP 484\",\n", " \"name\": \"PEP 484\",\n", " \"type\": \"PEP\",\n", " \"properties\": {\"url\": web_targets[\"pep_484\"], \"title\": getattr(web_pages.get(\"pep_484\"), \"title\", \"\")},\n", " \"source\": \"pep_site\",\n", " },\n", " {\n", " \"id\": \"Global Interpreter Lock\",\n", " \"name\": \"Global Interpreter Lock\",\n", " \"type\": \"Concept\",\n", " \"properties\": {\"abbrev\": \"GIL\"},\n", " \"source\": \"pep_site\",\n", " },\n", " {\n", " \"id\": \"Type Hints\",\n", " \"name\": \"Type Hints\",\n", " \"type\": \"Concept\",\n", " \"properties\": {},\n", " \"source\": \"pep_site\",\n", " },\n", " {\n", " \"id\": \"No-GIL Build\",\n", " \"name\": \"No-GIL Build\",\n", " \"type\": \"Feature\",\n", " \"properties\": {\"description\": \"CPython build configuration without the GIL\"},\n", " \"source\": \"pep_site\",\n", " },\n", " {\n", " \"id\": \"Free-Threaded Python\",\n", " \"name\": \"Free-Threaded Python\",\n", " \"type\": \"Feature\",\n", " \"properties\": {\"description\": \"Python builds that allow threads without a global lock\"},\n", " \"source\": \"pep_site\",\n", " },\n", " {\n", " \"id\": \"Packaging\",\n", " \"name\": \"Packaging\",\n", " \"type\": \"Concept\",\n", " \"properties\": {},\n", " \"source\": \"python_docs\",\n", " },\n", " ]\n", "\n", " relationships = [\n", " {\"source\": \"Python Software Foundation\", \"target\": \"Python\", \"type\": \"governs\"},\n", " {\"source\": \"Python Software Foundation\", \"target\": \"CPython\", \"type\": \"supports\"},\n", " {\"source\": \"CPython\", \"target\": \"Python\", \"type\": \"implements\"},\n", " {\"source\": \"Python Documentation\", \"target\": \"Python\", \"type\": \"documents\"},\n", " {\"source\": \"Python Documentation\", \"target\": \"Python 3.13\", \"type\": \"documents\"},\n", " {\"source\": \"Python Documentation\", \"target\": \"asyncio\", \"type\": \"documents\"},\n", " {\"source\": \"Python Documentation\", \"target\": \"typing\", \"type\": \"documents\"},\n", " {\"source\": \"Python 3.13\", \"target\": \"Python\", \"type\": \"version_of\"},\n", " {\"source\": \"asyncio\", \"target\": \"Python\", \"type\": \"stdlib_of\"},\n", " {\"source\": \"typing\", \"target\": \"Python\", \"type\": \"stdlib_of\"},\n", " {\"source\": \"Python Packaging User Guide\", \"target\": \"Packaging\", \"type\": \"documents\"},\n", " {\"source\": \"Python Packaging User Guide\", \"target\": \"PyPI\", \"type\": \"mentions\"},\n", " {\"source\": \"PEP 703\", \"target\": \"No-GIL Build\", \"type\": \"proposes\"},\n", " {\"source\": \"PEP 703\", \"target\": \"Free-Threaded Python\", \"type\": \"proposes\"},\n", " {\"source\": \"PEP 703\", \"target\": \"Global Interpreter Lock\", \"type\": \"discusses\"},\n", " {\"source\": \"No-GIL Build\", \"target\": \"Python 3.13\", \"type\": \"planned_for\"},\n", " {\"source\": \"PEP 703\", \"target\": \"CPython\", \"type\": \"targets\"},\n", " {\"source\": \"PEP 8\", \"target\": \"Python\", \"type\": \"style_guide_for\"},\n", " {\"source\": \"PEP 484\", \"target\": \"Type Hints\", \"type\": \"introduces\"},\n", " {\"source\": \"Type Hints\", \"target\": \"typing\", \"type\": \"implemented_by\"},\n", " ]\n", "\n", " source_web = {\n", " \"name\": \"Python Web + Docs + PEPs + Packaging\",\n", " \"type\": \"unstructured_web\",\n", " \"entities\": entities,\n", " \"relationships\": relationships,\n", " }\n", "\n", "except Exception as e:\n", " print(f\"Web Ingestion Failed: {e}\")\n", " source_web = {\n", " \"name\": \"Python Web + Docs + PEPs + Packaging (Offline)\",\n", " \"type\": \"unstructured_web\",\n", " \"entities\": [\n", " {\"id\": \"Python\", \"name\": \"Python\", \"type\": \"ProgrammingLanguage\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"Python Documentation\", \"name\": \"Python Documentation\", \"type\": \"Documentation\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"CPython\", \"name\": \"CPython\", \"type\": \"Interpreter\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"Python Software Foundation\", \"name\": \"Python Software Foundation\", \"type\": \"Organization\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"PEP 703\", \"name\": \"PEP 703\", \"type\": \"PEP\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"Global Interpreter Lock\", \"name\": \"Global Interpreter Lock\", \"type\": \"Concept\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"Type Hints\", \"name\": \"Type Hints\", \"type\": \"Concept\", \"properties\": {}, \"source\": \"offline\"},\n", " ],\n", " \"relationships\": [\n", " {\"source\": \"Python Software Foundation\", \"target\": \"Python\", \"type\": \"governs\"},\n", " {\"source\": \"CPython\", \"target\": \"Python\", \"type\": \"implements\"},\n", " {\"source\": \"Python Documentation\", \"target\": \"Python\", \"type\": \"documents\"},\n", " {\"source\": \"PEP 703\", \"target\": \"Global Interpreter Lock\", \"type\": \"discusses\"},\n", " {\"source\": \"Type Hints\", \"target\": \"Python\", \"type\": \"feature_of\"}\n", " ],\n", " }\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Source 2: Package Registry (API -> File Ingestion)\n", "\n", "We fetch live metadata for the `pandas` library from PyPI's JSON API. We save this as a JSON file and then ingest it using `FileIngestor` to demonstrate structured file handling." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(\"\\n--- 2. Ingesting API Sources: PyPI (pandas, numpy, scipy, matplotlib, scikit-learn, requests, fastapi) ---\")\n", "\n", "import re\n", "from urllib.parse import urlparse\n", "\n", "packages = [\"pandas\", \"numpy\", \"scipy\", \"matplotlib\", \"scikit-learn\", \"requests\", \"fastapi\"]\n", "file_ingestor = FileIngestor()\n", "api_ingestor = RESTIngestor(timeout=30)\n", "\n", "entities = [\n", " {\"id\": \"PyPI\", \"name\": \"PyPI\", \"type\": \"PackageRegistry\", \"properties\": {\"url\": \"https://pypi.org/\"}, \"source\": \"pypi_api\"},\n", " {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"pypi_api\"},\n", "]\n", "relationships = []\n", "\n", "seen_entities = {e[\"id\"] for e in entities}\n", "\n", "def _dep_name(req: str) -> str:\n", " if not req:\n", " return \"\"\n", " req = req.split(\";\")[0].strip()\n", " m = re.match(r\"^([A-Za-z0-9_.-]+)\", req)\n", " return (m.group(1) if m else \"\").strip()\n", "\n", "def _url_entity_id(url: str) -> str:\n", " return f\"URL::{url.strip()}\"\n", "\n", "def _add_url_entity(url: str, label: str, source: str) -> str:\n", " url = (url or \"\").strip()\n", " if not url:\n", " return \"\"\n", " uid = _url_entity_id(url)\n", " if uid not in seen_entities:\n", " entities.append({\"id\": uid, \"name\": label or url, \"type\": \"WebResource\", \"properties\": {\"url\": url}, \"source\": source})\n", " seen_entities.add(uid)\n", " return uid\n", "\n", "def _is_github(url: str) -> bool:\n", " try:\n", " return urlparse(url).netloc.lower().endswith(\"github.com\")\n", " except Exception:\n", " return False\n", "\n", "try:\n", " for pkg in packages:\n", " pypi_url = f\"https://pypi.org/pypi/{pkg}/json\"\n", " local_json_path = os.path.join(WORKSPACE_DIR, f\"{pkg}_pypi.json\")\n", "\n", " api_data = api_ingestor.ingest_endpoint(pypi_url)\n", " data = api_data.data if isinstance(api_data.data, dict) else {}\n", "\n", " with open(local_json_path, \"w\", encoding=\"utf-8\") as f:\n", " json.dump(data, f)\n", "\n", " ingested_file = file_ingestor.ingest_file(local_json_path)\n", " print(f\"Ingested File: {ingested_file.name} ({ingested_file.size} bytes)\")\n", "\n", " info = data.get(\"info\", {})\n", " project_urls = info.get(\"project_urls\") or {}\n", "\n", " if pkg not in seen_entities:\n", " entities.append(\n", " {\n", " \"id\": pkg,\n", " \"name\": pkg,\n", " \"type\": \"Library\",\n", " \"properties\": {\n", " \"version\": info.get(\"version\"),\n", " \"summary\": info.get(\"summary\"),\n", " \"license\": info.get(\"license\"),\n", " \"requires_python\": info.get(\"requires_python\"),\n", " },\n", " \"source\": \"pypi_api\",\n", " }\n", " )\n", " seen_entities.add(pkg)\n", "\n", " relationships.append({\"source\": pkg, \"target\": \"PyPI\", \"type\": \"published_on\"})\n", " relationships.append({\"source\": pkg, \"target\": \"Python\", \"type\": \"written_in\"})\n", "\n", " home_page = (info.get(\"home_page\") or \"\").strip()\n", " if home_page:\n", " hp_id = _add_url_entity(home_page, f\"{pkg} homepage\", \"pypi_api\")\n", " if hp_id:\n", " relationships.append({\"source\": pkg, \"target\": hp_id, \"type\": \"has_homepage\"})\n", "\n", " normalized_project_urls = {}\n", " for k, v in project_urls.items():\n", " if not k or not v:\n", " continue\n", " normalized_project_urls[str(k).strip().lower()] = str(v).strip()\n", "\n", " for key, url in normalized_project_urls.items():\n", " if not url:\n", " continue\n", " label = f\"{pkg} {key}\"\n", " url_id = _add_url_entity(url, label, \"pypi_api\")\n", " if not url_id:\n", " continue\n", " if \"doc\" in key:\n", " relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"has_documentation\"})\n", " elif \"bug\" in key or \"issue\" in key:\n", " relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"issues_at\"})\n", " elif \"source\" in key or \"github\" in key:\n", " relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"has_source\"})\n", " else:\n", " relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"related_resource\"})\n", " if _is_github(url):\n", " relationships.append({\"source\": url_id, \"target\": \"GitHub\", \"type\": \"hosted_on\"})\n", "\n", " requires_dist = info.get(\"requires_dist\") or []\n", " dep_names = []\n", " for req in requires_dist:\n", " name = _dep_name(req)\n", " if name:\n", " dep_names.append(name)\n", "\n", " unique_deps = sorted(set(dep_names))[:12]\n", " for dep in unique_deps:\n", " if dep not in seen_entities:\n", " entities.append({\"id\": dep, \"name\": dep, \"type\": \"Library\", \"properties\": {}, \"source\": \"pypi_requires_dist\"})\n", " seen_entities.add(dep)\n", " relationships.append({\"source\": pkg, \"target\": dep, \"type\": \"depends_on\"})\n", "\n", " print(f\"Extracted Entity: {pkg} (v{info.get('version')}) with {len(unique_deps)} dependencies (capped)\")\n", "\n", " for rel in [\n", " {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"built_on\"},\n", " {\"source\": \"scipy\", \"target\": \"numpy\", \"type\": \"built_on\"},\n", " {\"source\": \"matplotlib\", \"target\": \"numpy\", \"type\": \"built_on\"},\n", " {\"source\": \"scikit-learn\", \"target\": \"numpy\", \"type\": \"built_on\"},\n", " {\"source\": \"scikit-learn\", \"target\": \"scipy\", \"type\": \"built_on\"},\n", " ]:\n", " relationships.append(rel)\n", "\n", " source_api = {\n", " \"name\": \"PyPI Registry\",\n", " \"type\": \"structured_api\",\n", " \"entities\": entities,\n", " \"relationships\": relationships,\n", " }\n", "\n", "except Exception as e:\n", " print(f\"API Ingestion Failed: {e}\")\n", " source_api = {\n", " \"name\": \"PyPI Registry (Offline)\",\n", " \"type\": \"structured_api\",\n", " \"entities\": [\n", " {\"id\": \"PyPI\", \"name\": \"PyPI\", \"type\": \"PackageRegistry\", \"properties\": {\"url\": \"https://pypi.org/\"}, \"source\": \"offline\"},\n", " {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"offline\"},\n", " {\"id\": \"pandas\", \"name\": \"pandas\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"numpy\", \"name\": \"numpy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"scipy\", \"name\": \"scipy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"matplotlib\", \"name\": \"matplotlib\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"scikit-learn\", \"name\": \"scikit-learn\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"requests\", \"name\": \"requests\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n", " {\"id\": \"fastapi\", \"name\": \"fastapi\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n", " ],\n", " \"relationships\": [\n", " {\"source\": \"pandas\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n", " {\"source\": \"numpy\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n", " {\"source\": \"scipy\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n", " {\"source\": \"matplotlib\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n", " {\"source\": \"scikit-learn\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n", " {\"source\": \"requests\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n", " {\"source\": \"fastapi\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n", " {\"source\": \"pandas\", \"target\": \"Python\", \"type\": \"written_in\"},\n", " {\"source\": \"numpy\", \"target\": \"Python\", \"type\": \"written_in\"},\n", " {\"source\": \"scipy\", \"target\": \"Python\", \"type\": \"written_in\"},\n", " {\"source\": \"matplotlib\", \"target\": \"Python\", \"type\": \"written_in\"},\n", " {\"source\": \"scikit-learn\", \"target\": \"Python\", \"type\": \"written_in\"},\n", " {\"source\": \"requests\", \"target\": \"Python\", \"type\": \"written_in\"},\n", " {\"source\": \"fastapi\", \"target\": \"Python\", \"type\": \"written_in\"},\n", " {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"built_on\"},\n", " {\"source\": \"scipy\", \"target\": \"numpy\", \"type\": \"built_on\"},\n", " {\"source\": \"scikit-learn\", \"target\": \"numpy\", \"type\": \"built_on\"},\n", " {\"source\": \"scikit-learn\", \"target\": \"scipy\", \"type\": \"built_on\"}\n", " ],\n", " }\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Source 3: Source Code Repository (Raw Content)\n", "\n", "We fetch the raw `README.rst` from the official CPython GitHub repository. This represents unstructured technical documentation." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(\"\\n--- 3. Ingesting Repo Sources: CPython + pandas + NumPy + SciPy (GitHub) ---\")\n", "\n", "try:\n", " repo_ingestor = WebIngestor(delay=0.5)\n", "\n", " headers = {\"Accept\": \"application/vnd.github+json\", \"User-Agent\": \"semantica-cookbook\"}\n", "\n", " repo_targets = {\n", " \"python/cpython\": {\"readme_raw\": \"https://raw.githubusercontent.com/python/cpython/main/README.rst\", \"library\": \"Python\"},\n", " \"pandas-dev/pandas\": {\"readme_raw\": \"https://raw.githubusercontent.com/pandas-dev/pandas/main/README.md\", \"library\": \"pandas\"},\n", " \"numpy/numpy\": {\"readme_raw\": \"https://raw.githubusercontent.com/numpy/numpy/main/README.md\", \"library\": \"numpy\"},\n", " \"scipy/scipy\": {\"readme_raw\": \"https://raw.githubusercontent.com/scipy/scipy/main/README.rst\", \"library\": \"scipy\"},\n", " }\n", "\n", " entities = [\n", " {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"github_api\"}\n", " ]\n", " relationships = []\n", "\n", " for repo_full, cfg in repo_targets.items():\n", " readme_url = cfg[\"readme_raw\"]\n", " readme = repo_ingestor.ingest_url(readme_url)\n", " print(f\"Fetched {repo_full} README: {len(readme.text)} chars\")\n", "\n", " repo_url = f\"https://github.com/{repo_full}\"\n", " entities.append(\n", " {\n", " \"id\": repo_full,\n", " \"name\": repo_full,\n", " \"type\": \"Repository\",\n", " \"properties\": {\"repo_url\": repo_url, \"readme_url\": readme_url, \"readme_title\": getattr(readme, \"title\", \"\")},\n", " \"source\": \"github_raw\",\n", " }\n", " )\n", " relationships.append({\"source\": repo_full, \"target\": \"GitHub\", \"type\": \"hosted_on\"})\n", "\n", " owner = repo_full.split(\"/\")[0]\n", " org_id = f\"GitHubOrg::{owner}\"\n", " entities.append({\"id\": org_id, \"name\": owner, \"type\": \"Organization\", \"properties\": {\"url\": f\"https://github.com/{owner}\"}, \"source\": \"github_api\"})\n", " relationships.append({\"source\": repo_full, \"target\": org_id, \"type\": \"owned_by\"})\n", "\n", " lib = cfg.get(\"library\")\n", " if lib:\n", " relationships.append({\"source\": lib, \"target\": repo_full, \"type\": \"source_code_in\"})\n", " relationships.append({\"source\": repo_full, \"target\": lib, \"type\": \"source_code_for\"})\n", "\n", " repo_api_url = f\"https://api.github.com/repos/{repo_full}\"\n", " repo_resp = requests.get(repo_api_url, headers=headers, timeout=30)\n", " if repo_resp.status_code == 200:\n", " meta = repo_resp.json() or {}\n", " repo_props = {\n", " \"stars\": meta.get(\"stargazers_count\"),\n", " \"forks\": meta.get(\"forks_count\"),\n", " \"open_issues\": meta.get(\"open_issues_count\"),\n", " \"language\": meta.get(\"language\"),\n", " \"updated_at\": meta.get(\"updated_at\"),\n", " }\n", " for ent in entities:\n", " if isinstance(ent, dict) and ent.get(\"id\") == repo_full:\n", " ent.setdefault(\"properties\", {}).update({k: v for k, v in repo_props.items() if v is not None})\n", " break\n", " else:\n", " print(f\"Repo metadata unavailable for {repo_full} (status {repo_resp.status_code}).\")\n", "\n", " relationships.append({\"source\": \"python/cpython\", \"target\": \"CPython\", \"type\": \"repository_for\"})\n", " relationships.append({\"source\": \"python/cpython\", \"target\": \"Python\", \"type\": \"implements\"})\n", "\n", " gh_releases_url = \"https://api.github.com/repos/python/cpython/releases?per_page=8\"\n", " release_resp = requests.get(gh_releases_url, headers=headers, timeout=30)\n", " if release_resp.status_code == 200:\n", " releases = release_resp.json() or []\n", " for r in releases:\n", " tag = r.get(\"tag_name\")\n", " if not tag:\n", " continue\n", " release_id = f\"Release::python/cpython::{tag}\"\n", " entities.append(\n", " {\n", " \"id\": release_id,\n", " \"name\": f\"CPython {tag}\",\n", " \"type\": \"Release\",\n", " \"properties\": {\"tag\": tag, \"published_at\": r.get(\"published_at\"), \"url\": r.get(\"html_url\")},\n", " \"source\": \"github_api\",\n", " }\n", " )\n", " relationships.append({\"source\": release_id, \"target\": \"python/cpython\", \"type\": \"release_of\"})\n", " if tag.startswith(\"v\") and len(tag) >= 4:\n", " major_minor = \".\".join(tag.lstrip(\"v\").split(\".\")[:2])\n", " relationships.append({\"source\": release_id, \"target\": f\"Python {major_minor}\", \"type\": \"implements\"})\n", " else:\n", " print(f\"GitHub releases unavailable (status {release_resp.status_code}).\")\n", "\n", " source_repo = {\n", " \"name\": \"GitHub Repos + Metadata + Releases\",\n", " \"type\": \"unstructured_repo\",\n", " \"entities\": entities,\n", " \"relationships\": relationships,\n", " }\n", "\n", "except Exception as e:\n", " print(f\"Repo Ingestion Failed: {e}\")\n", " source_repo = {\n", " \"name\": \"GitHub Repos + Metadata + Releases (Offline)\",\n", " \"type\": \"unstructured_repo\",\n", " \"entities\": [\n", " {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"offline\"},\n", " {\"id\": \"python/cpython\", \"name\": \"python/cpython\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/python/cpython\"}, \"source\": \"offline\"},\n", " {\"id\": \"pandas-dev/pandas\", \"name\": \"pandas-dev/pandas\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/pandas-dev/pandas\"}, \"source\": \"offline\"},\n", " {\"id\": \"numpy/numpy\", \"name\": \"numpy/numpy\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/numpy/numpy\"}, \"source\": \"offline\"},\n", " {\"id\": \"scipy/scipy\", \"name\": \"scipy/scipy\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/scipy/scipy\"}, \"source\": \"offline\"},\n", " ],\n", " \"relationships\": [\n", " {\"source\": \"python/cpython\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n", " {\"source\": \"pandas-dev/pandas\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n", " {\"source\": \"numpy/numpy\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n", " {\"source\": \"scipy/scipy\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n", " {\"source\": \"python/cpython\", \"target\": \"Python\", \"type\": \"implements\"},\n", " {\"source\": \"pandas\", \"target\": \"pandas-dev/pandas\", \"type\": \"source_code_in\"},\n", " {\"source\": \"numpy\", \"target\": \"numpy/numpy\", \"type\": \"source_code_in\"},\n", " {\"source\": \"scipy\", \"target\": \"scipy/scipy\", \"type\": \"source_code_in\"}\n", " ],\n", " }\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Source 4: Database (SQLite via DBIngestor)\n", "\n", "We ingest structured data from a local SQLite database using `DBIngestor`. This demonstrates database connectivity and SQL query extraction.\n", "\n", "**Note:** This example creates a small SQLite database inside the temporary workspace so it works offline.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(\"\\n--- 4. Ingesting Database Source: SQLite (local) ---\")\n", "\n", "import sqlite3\n", "\n", "db_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem_metrics.sqlite\")\n", "\n", "try:\n", " conn = sqlite3.connect(db_path)\n", " cur = conn.cursor()\n", " cur.execute(\n", " \"\"\"\n", " CREATE TABLE IF NOT EXISTS library_metrics (\n", " library TEXT PRIMARY KEY,\n", " downloads INTEGER,\n", " stars INTEGER,\n", " last_updated TEXT\n", " )\n", " \"\"\"\n", " )\n", "\n", " cur.execute(\n", " \"\"\"\n", " CREATE TABLE IF NOT EXISTS library_categories (\n", " library TEXT,\n", " category TEXT,\n", " PRIMARY KEY (library, category)\n", " )\n", " \"\"\"\n", " )\n", "\n", " cur.execute(\n", " \"\"\"\n", " CREATE TABLE IF NOT EXISTS library_dependencies (\n", " library TEXT,\n", " dependency TEXT,\n", " relation TEXT,\n", " PRIMARY KEY (library, dependency, relation)\n", " )\n", " \"\"\"\n", " )\n", "\n", " sample_rows = [\n", " (\"numpy\", 80000000, 28000, datetime.utcnow().isoformat()),\n", " (\"pandas\", 50000000, 45000, datetime.utcnow().isoformat()),\n", " (\"scipy\", 20000000, 12000, datetime.utcnow().isoformat()),\n", " (\"matplotlib\", 25000000, 21000, datetime.utcnow().isoformat()),\n", " (\"scikit-learn\", 18000000, 60000, datetime.utcnow().isoformat()),\n", " (\"requests\", 90000000, 52000, datetime.utcnow().isoformat()),\n", " (\"fastapi\", 22000000, 80000, datetime.utcnow().isoformat()),\n", " ]\n", " cur.executemany(\n", " \"INSERT OR REPLACE INTO library_metrics (library, downloads, stars, last_updated) VALUES (?, ?, ?, ?)\",\n", " sample_rows,\n", " )\n", "\n", " category_rows = [\n", " (\"numpy\", \"Numerical\"),\n", " (\"pandas\", \"Data Analysis\"),\n", " (\"scipy\", \"Scientific Computing\"),\n", " (\"matplotlib\", \"Visualization\"),\n", " (\"scikit-learn\", \"Machine Learning\"),\n", " (\"requests\", \"Networking\"),\n", " (\"fastapi\", \"Web\"),\n", " ]\n", " cur.executemany(\n", " \"INSERT OR REPLACE INTO library_categories (library, category) VALUES (?, ?)\",\n", " category_rows,\n", " )\n", "\n", " dependency_rows = [\n", " (\"pandas\", \"numpy\", \"depends_on\"),\n", " (\"scipy\", \"numpy\", \"depends_on\"),\n", " (\"matplotlib\", \"numpy\", \"depends_on\"),\n", " (\"scikit-learn\", \"numpy\", \"depends_on\"),\n", " (\"scikit-learn\", \"scipy\", \"depends_on\"),\n", " (\"fastapi\", \"pydantic\", \"depends_on\"),\n", " (\"fastapi\", \"starlette\", \"built_on\"),\n", " (\"requests\", \"urllib3\", \"depends_on\"),\n", " (\"requests\", \"certifi\", \"depends_on\"),\n", " ]\n", " cur.executemany(\n", " \"INSERT OR REPLACE INTO library_dependencies (library, dependency, relation) VALUES (?, ?, ?)\",\n", " dependency_rows,\n", " )\n", " conn.commit()\n", " conn.close()\n", "\n", " db_ingestor = DBIngestor()\n", " sqlite_conn_str = f\"sqlite:///{db_path}\"\n", "\n", " rows = db_ingestor.execute_query(\n", " sqlite_conn_str,\n", " \"SELECT library, downloads, stars, last_updated FROM library_metrics WHERE downloads >= :min_downloads\",\n", " min_downloads=1000000,\n", " )\n", "\n", " cat_rows = db_ingestor.execute_query(\n", " sqlite_conn_str,\n", " \"SELECT library, category FROM library_categories\",\n", " )\n", "\n", " dep_rows = db_ingestor.execute_query(\n", " sqlite_conn_str,\n", " \"SELECT library, dependency, relation FROM library_dependencies\",\n", " )\n", "\n", " if not rows and not cat_rows and not dep_rows:\n", " raise RuntimeError(\"No data found in SQLite database\")\n", "\n", " entities = []\n", " relationships = []\n", "\n", " entities.append({\"id\": \"SQLite\", \"name\": \"SQLite\", \"type\": \"Database\", \"properties\": {\"path\": db_path}, \"source\": \"sqlite_db\"})\n", " for row in rows:\n", " lib = row.get(\"library\")\n", " if not lib:\n", " continue\n", " entities.append({\"id\": lib, \"name\": lib, \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db\"})\n", " metric_id = f\"{lib}::metrics\"\n", " entities.append(\n", " {\n", " \"id\": metric_id,\n", " \"name\": f\"{lib} metrics\",\n", " \"type\": \"LibraryMetrics\",\n", " \"properties\": {\n", " \"downloads\": row.get(\"downloads\"),\n", " \"stars\": row.get(\"stars\"),\n", " \"last_updated\": row.get(\"last_updated\"),\n", " },\n", " \"source\": \"sqlite_db\",\n", " }\n", " )\n", " relationships.append({\"source\": metric_id, \"target\": lib, \"type\": \"metrics_for\"})\n", " relationships.append({\"source\": lib, \"target\": \"Python\", \"type\": \"ecosystem_of\"})\n", " relationships.append({\"source\": metric_id, \"target\": \"SQLite\", \"type\": \"stored_in\"})\n", "\n", " for row in cat_rows:\n", " lib = row.get(\"library\")\n", " cat = row.get(\"category\")\n", " if not lib or not cat:\n", " continue\n", " cat_id = f\"Category::{cat}\"\n", " entities.append({\"id\": cat_id, \"name\": cat, \"type\": \"Category\", \"properties\": {}, \"source\": \"sqlite_db\"})\n", " relationships.append({\"source\": lib, \"target\": cat_id, \"type\": \"categorized_as\"})\n", "\n", " for row in dep_rows:\n", " lib = row.get(\"library\")\n", " dep = row.get(\"dependency\")\n", " rel = row.get(\"relation\") or \"depends_on\"\n", " if not lib or not dep:\n", " continue\n", " entities.append({\"id\": dep, \"name\": dep, \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db\"})\n", " relationships.append({\"source\": lib, \"target\": dep, \"type\": rel})\n", "\n", " print(f\"Ingested {len(entities)} rows from SQLite metrics table\")\n", " source_db = {\n", " \"name\": \"SQLite Metrics\",\n", " \"type\": \"database\",\n", " \"entities\": entities,\n", " \"relationships\": relationships,\n", " }\n", "\n", "except Exception as e:\n", " print(f\"Database ingestion via DBIngestor skipped: {e}\")\n", " entities = [\n", " {\"id\": \"SQLite\", \"name\": \"SQLite\", \"type\": \"Database\", \"properties\": {\"path\": db_path}, \"source\": \"sqlite_db_offline\"},\n", " {\"id\": \"pandas\", \"name\": \"pandas\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n", " {\"id\": \"pandas::metrics\", \"name\": \"pandas metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 50000000, \"stars\": 45000}, \"source\": \"sqlite_db_offline\"},\n", " {\"id\": \"numpy\", \"name\": \"numpy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n", " {\"id\": \"numpy::metrics\", \"name\": \"numpy metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 80000000, \"stars\": 28000}, \"source\": \"sqlite_db_offline\"},\n", " {\"id\": \"scipy\", \"name\": \"scipy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n", " {\"id\": \"scipy::metrics\", \"name\": \"scipy metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 20000000, \"stars\": 12000}, \"source\": \"sqlite_db_offline\"},\n", " {\"id\": \"fastapi\", \"name\": \"fastapi\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n", " {\"id\": \"fastapi::metrics\", \"name\": \"fastapi metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 22000000, \"stars\": 80000}, \"source\": \"sqlite_db_offline\"},\n", " ]\n", " relationships = [\n", " {\"source\": \"pandas::metrics\", \"target\": \"pandas\", \"type\": \"metrics_for\"},\n", " {\"source\": \"numpy::metrics\", \"target\": \"numpy\", \"type\": \"metrics_for\"},\n", " {\"source\": \"scipy::metrics\", \"target\": \"scipy\", \"type\": \"metrics_for\"},\n", " {\"source\": \"fastapi::metrics\", \"target\": \"fastapi\", \"type\": \"metrics_for\"},\n", " {\"source\": \"pandas\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n", " {\"source\": \"numpy\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n", " {\"source\": \"scipy\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n", " {\"source\": \"fastapi\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n", " {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"depends_on\"},\n", " {\"source\": \"fastapi\", \"target\": \"pydantic\", \"type\": \"depends_on\"}\n", " ]\n", " source_db = {\n", " \"name\": \"SQLite Metrics (Offline)\",\n", " \"type\": \"database\",\n", " \"entities\": entities,\n", " \"relationships\": relationships,\n", " }\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Source 5: Model Context Protocol (MCP)\n", "\n", "We attempt to connect to a local MCP server (e.g., a Web Search tool) to get live context. \n", "\n", "**Note:** If no MCP server is running at `localhost:8000`, this section will gracefully fallback to simulated data, but the code provided is production-ready for MCP integration.\n", "\n", "Useful MCP server directories / references:\n", "- https://github.com/modelcontextprotocol/servers\n", "- https://glama.ai/mcp/servers\n", "- https://github.com/punkpeye/awesome-mcp-servers\n", "- https://github.com/wong2/awesome-mcp-servers\n", "- https://mcp.so\n", "- Brave Search MCP Server: https://github.com/brave/brave-search-mcp-server" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(\"\\n--- 5. Ingesting via MCP (Model Context Protocol) ---\")\n", "\n", "mcp_server_urls = [\n", " \"http://localhost:8000/mcp\",\n", " \"http://localhost:8000/sse\",\n", " \"http://127.0.0.1:8000/mcp\",\n", " \"http://127.0.0.1:8000/sse\",\n", " \"http://localhost:8080/mcp\",\n", " \"http://localhost:8080/sse\",\n", " \"http://127.0.0.1:8080/mcp\",\n", " \"http://127.0.0.1:8080/sse\",\n", "]\n", "\n", "try:\n", " mcp_client_logger = logging.getLogger(\"semantica.mcp_client\")\n", " mcp_ingestor_logger = logging.getLogger(\"semantica.mcp_ingestor\")\n", " prev_client_level = mcp_client_logger.level\n", " prev_ingestor_level = mcp_ingestor_logger.level\n", " mcp_client_logger.setLevel(logging.CRITICAL)\n", " mcp_ingestor_logger.setLevel(logging.CRITICAL)\n", "\n", " mcp = MCPIngestor()\n", " connected_url = None\n", " for url in mcp_server_urls:\n", " try:\n", " mcp.connect(\"web_search\", url=url)\n", " connected_url = url\n", " break\n", " except Exception:\n", " continue\n", " if not connected_url:\n", " raise RuntimeError(\"No MCP server reachable on known local SSE endpoints\")\n", "\n", " print(f\"Connected to MCP Server at {connected_url}\")\n", " \n", " payload = {\"query\": \"latest python 3.13 features\"}\n", " mcp_result = None\n", " for tool_name in [\"search\", \"brave_web_search\", \"brave_local_search\"]:\n", " try:\n", " mcp_result = mcp.ingest_tool_output(\"web_search\", tool_name, payload)\n", " break\n", " except Exception:\n", " continue\n", " if mcp_result is None:\n", " raise RuntimeError(\"No compatible MCP search tool found\")\n", " search_results = getattr(mcp_result, \"content\", mcp_result)\n", " if not isinstance(search_results, dict):\n", " raise RuntimeError(\"MCP tool output content was not a dict\")\n", " \n", " print(\"Received Live Data from MCP.\")\n", " \n", " raw_entities = search_results.get(\"entities\", []) or []\n", " raw_relationships = search_results.get(\"relationships\", []) or []\n", " normalized_entities = []\n", " for ent in raw_entities:\n", " if not isinstance(ent, dict):\n", " continue\n", " name = ent.get(\"name\") or ent.get(\"text\") or ent.get(\"id\")\n", " if not name:\n", " continue\n", " normalized_entities.append(\n", " {\n", " \"id\": ent.get(\"id\") or name,\n", " \"name\": name,\n", " \"type\": ent.get(\"type\") or ent.get(\"label\") or \"Entity\",\n", " \"properties\": ent.get(\"properties\") or ent.get(\"metadata\") or {},\n", " \"source\": \"mcp_live\",\n", " }\n", " )\n", "\n", " normalized_relationships = []\n", " for rel in raw_relationships:\n", " if not isinstance(rel, dict):\n", " continue\n", " src = rel.get(\"source\") or rel.get(\"subject\")\n", " tgt = rel.get(\"target\") or rel.get(\"object\")\n", " rtype = rel.get(\"type\") or rel.get(\"label\") or rel.get(\"predicate\")\n", " if not src or not tgt or not rtype:\n", " continue\n", " normalized_relationships.append({\"source\": src, \"target\": tgt, \"type\": rtype, \"properties\": rel.get(\"properties\") or rel.get(\"metadata\") or {}})\n", "\n", " endpoint_ids = {e.get(\"id\") for e in normalized_entities if isinstance(e, dict) and e.get(\"id\")}\n", " for rel in normalized_relationships:\n", " src = rel.get(\"source\")\n", " tgt = rel.get(\"target\")\n", " for node_id in [src, tgt]:\n", " if node_id and node_id not in endpoint_ids:\n", " normalized_entities.append({\"id\": node_id, \"name\": node_id, \"type\": \"Entity\", \"properties\": {}, \"source\": \"mcp_live\"})\n", " endpoint_ids.add(node_id)\n", "\n", " source_mcp = {\n", " \"name\": \"MCP Search\",\n", " \"type\": \"agent_tool\",\n", " \"entities\": normalized_entities,\n", " \"relationships\": normalized_relationships,\n", " \"source\": \"mcp_live\"\n", " }\n", "\n", "except Exception as e:\n", " print(\"MCP Server not detected. Using simulated 'Live Search' data.\")\n", " print(\"To enable: start a local MCP server (commonly `http://localhost:8000/mcp` or `http://localhost:8000/sse`)\")\n", " \n", " source_mcp = {\n", " \"name\": \"MCP Search (Simulated)\",\n", " \"type\": \"agent_tool\",\n", " \"entities\": [\n", " {\n", " \"id\": \"Python 3.13\",\n", " \"name\": \"Python 3.13\",\n", " \"type\": \"SoftwareVersion\",\n", " \"properties\": {\"status\": \"In Development\", \"feature\": \"No-GIL Build\"},\n", " \"source\": \"mcp_simulated\"\n", " }\n", " ],\n", " \"relationships\": [\n", " {\"source\": \"Python 3.13\", \"target\": \"Python\", \"type\": \"version_of\"}\n", " ]\n", " }\n", "finally:\n", " if 'mcp_client_logger' in locals() and 'prev_client_level' in locals():\n", " mcp_client_logger.setLevel(prev_client_level)\n", " if 'mcp_ingestor_logger' in locals() and 'prev_ingestor_level' in locals():\n", " mcp_ingestor_logger.setLevel(prev_ingestor_level)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Phase 5: Knowledge Graph Construction\n", "\n", "We merge all these real-world data points into a single Knowledge Graph." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(\"\\n--- Building Knowledge Graph ---\")\n", "\n", "all_sources = [source_web, source_api, source_repo, source_db, source_mcp]\n", "\n", "builder = GraphBuilder(merge_entities=True, resolve_conflicts=True)\n", "\n", "kg = builder.build(sources=all_sources) or {}\n", "kg.setdefault(\"entities\", [])\n", "kg.setdefault(\"relationships\", [])\n", "\n", "entity_ids = set()\n", "for node in kg.get(\"entities\", []):\n", " if isinstance(node, dict):\n", " node_id = node.get(\"id\") or node.get(\"entity_id\") or node.get(\"name\")\n", " if node_id:\n", " entity_ids.add(node_id)\n", "\n", "missing_ids = set()\n", "for rel in kg.get(\"relationships\", []):\n", " if not isinstance(rel, dict):\n", " continue\n", " src = rel.get(\"source\") or rel.get(\"subject\")\n", " tgt = rel.get(\"target\") or rel.get(\"object\")\n", " if src and src not in entity_ids:\n", " missing_ids.add(src)\n", " if tgt and tgt not in entity_ids:\n", " missing_ids.add(tgt)\n", "\n", "for mid in sorted(missing_ids):\n", " kg[\"entities\"].append({\"id\": mid, \"name\": mid, \"type\": \"Entity\", \"properties\": {}, \"source\": \"auto\"})\n", " entity_ids.add(mid)\n", "\n", "print(f\"Graph Statistics:\")\n", "print(f\"Nodes: {len(kg.get('entities', []))}\")\n", "print(f\"Edges: {len(kg.get('relationships', []))}\")\n", "\n", "# List all nodes to verify integration\n", "print(\"\\nEntities in Graph:\")\n", "for node in kg.get('entities', []):\n", " name = node.get('name') or node.get('label') or node.get('id')\n", " ntype = node.get('type') or node.get('label') or 'Entity'\n", " print(f\"- {name} ({ntype})\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Phase 6: Advanced Graph Analytics\n", "\n", "We can perform network analysis on the constructed graph to find key entities. Here, we calculate **Degree Centrality** to identify the most connected nodes." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from semantica.kg import CentralityCalculator, CommunityDetector\n", "\n", "print(\"\\n--- Running Graph Analytics (Semantica) ---\")\n", "\n", "# 1. Centrality Analysis\n", "centrality_calc = CentralityCalculator()\n", "degree_centrality = centrality_calc.calculate_degree_centrality(kg)\n", "\n", "print(\"Top 5 Most Central Entities (Degree):\")\n", "for ranking in degree_centrality.get(\"rankings\", [])[:5]:\n", " print(f\"- {ranking['node']}: {ranking['score']:.4f}\")\n", "\n", "# 2. Community Detection\n", "try:\n", " detector = CommunityDetector()\n", " result = detector.detect_communities(kg, algorithm=\"louvain\") or {}\n", "\n", " communities_raw = result.get(\"communities\")\n", " if communities_raw is None:\n", " communities_raw = result.get(\"node_assignments\")\n", "\n", " communities = []\n", " if isinstance(communities_raw, list):\n", " for c in communities_raw:\n", " if isinstance(c, (list, tuple, set)):\n", " communities.append(list(c))\n", " elif isinstance(c, dict):\n", " communities.append(list(c.keys()))\n", " else:\n", " communities.append([str(c)])\n", " elif isinstance(communities_raw, dict):\n", " comm_map = {}\n", " for node_id, comm_id in communities_raw.items():\n", " comm_map.setdefault(comm_id, []).append(node_id)\n", " # Sort by community ID safely\n", " sorted_keys = sorted(comm_map.keys(), key=lambda x: str(x))\n", " communities = [comm_map[k] for k in sorted_keys]\n", "\n", " print(f\"\\nDetected {len(communities)} Communities:\")\n", " for i, comm in enumerate(communities[:3]):\n", " # Ensure elements are strings\n", " sample = [str(x) for x in list(comm)[:5]]\n", " print(f\"Community {i+1}: {', '.join(sample)}...\")\n", "except Exception as e:\n", " print(f\"\\nCommunity detection skipped: {e}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Phase 7: Semantic Querying\n", "\n", "We can query the graph to find specific relationships, such as tracing the lineage of Python versions or finding libraries related to Python." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from semantica.kg import ConnectivityAnalyzer\n", "\n", "print(\"\\n--- Semantic Querying & Path Finding ---\")\n", "\n", "kg = globals().get(\"kg\")\n", "kg = kg if isinstance(kg, dict) else {}\n", "kg.setdefault(\"entities\", [])\n", "kg.setdefault(\"relationships\", [])\n", "\n", "analyzer = ConnectivityAnalyzer()\n", "\n", "# 1. Check Connectivity\n", "connectivity = analyzer.analyze_connectivity(kg)\n", "print(f\"Graph Connected: {connectivity.get('is_connected')}\")\n", "print(f\"Connected Components: {connectivity.get('num_components')}\")\n", "\n", "# 2. Find Path between Entities\n", "source = \"pandas\"\n", "target = \"Python\"\n", "\n", "print(f\"\\nFinding path from '{source}' to '{target}':\")\n", "try:\n", " path_result = analyzer.calculate_shortest_paths(kg, source=source, target=target)\n", " \n", " if path_result.get(\"exists\"):\n", " path = path_result[\"path\"]\n", " print(f\"Path Found: {' -> '.join(path)}\")\n", " print(f\"Distance: {path_result['distance']}\")\n", " else:\n", " print(\"No path found.\")\n", "except Exception as e:\n", " print(f\"Path finding error: {e}\")\n", " # Fallback to simple neighbor check\n", " print(\"Falling back to direct neighbor check...\")\n", " found = False\n", " for rel in kg.get('relationships', []):\n", " if rel.get('source') == source and rel.get('target') == target:\n", " print(f\" - [{rel.get('type', 'related_to')}] -> {target}\")\n", " found = True\n", " if not found:\n", " print(\"No direct edge found.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Phase 8: Export & Persistence\n", "\n", "Finally, we save the constructed Knowledge Graph to a JSON file for external use or visualization in other tools." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from semantica.export import GraphExporter\n", "\n", "print(\"\\n--- Exporting Knowledge Graph ---\")\n", "\n", "# Use Semantica's GraphExporter for robust export\n", "exporter = GraphExporter(format=\"json\", include_attributes=True)\n", "export_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem_kg.json\")\n", "\n", "try:\n", " exporter.export_knowledge_graph(kg, export_path)\n", " print(f\"Graph saved to: {export_path}\")\n", " \n", " # Optional: Export to GraphML for Gephi\n", " graphml_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem.graphml\")\n", " exporter_ml = GraphExporter(format=\"graphml\")\n", " exporter_ml.export_knowledge_graph(kg, graphml_path)\n", " print(f\"GraphML saved to: {graphml_path} (Ready for Gephi/Cytoscape)\")\n", " \n", "except Exception as e:\n", " print(f\"Export failed: {e}\")\n", " # Fallback\n", " import json\n", " with open(export_path, \"w\") as f:\n", " json.dump(kg, f, default=str)\n", " print(\"Fallback export used.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Phase 9: Context Engineering for LLM Agents\n", "\n", "This is the **critical step** where we turn our Knowledge Graph into a queryable **Context** for AI Agents.\n", "We use the `AgentContext` module to ingest our graph and enable **Retrieval Augmented Generation (RAG)** capabilities." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(\"\\n--- Context Engineering ---\")\n", "\n", "import json\n", "from semantica.context import AgentContext, ContextGraph\n", "from semantica.vector_store import VectorStore\n", "\n", "kg = globals().get(\"kg\")\n", "kg = kg if isinstance(kg, dict) else {}\n", "kg.setdefault(\"entities\", [])\n", "kg.setdefault(\"relationships\", [])\n", "\n", "# 1. Initialize Vector Store (with FastEmbed support)\n", "# We try to use the high-performance 'fastembed' model if available\n", "vs = VectorStore(backend=\"inmemory\", dimension=384)\n", "try:\n", " if hasattr(vs, \"embedder\") and vs.embedder:\n", " print(\"Initializing FastEmbed model (BAAI/bge-small-en-v1.5)...\")\n", " vs.embedder.set_text_model(method=\"fastembed\", model_name=\"BAAI/bge-small-en-v1.5\")\n", "except Exception as e:\n", " print(f\"FastEmbed not available ({e}). Using fallback keyword/random embedding.\")\n", " print(\"Tip: Run '!pip install fastembed' and restart kernel for better results.\")\n", "\n", "# 2. Initialize Context Graph\n", "cg = ContextGraph()\n", "\n", "# 3. Create the Agent Context\n", "# This binds the Vector Store (Content) and Knowledge Graph (Structure) together\n", "context = AgentContext(vector_store=vs, knowledge_graph=cg)\n", "\n", "# 4. Ingest Graph Structure\n", "# We map our generic KG data to the specific structure ContextGraph expects\n", "print(\"Building Context Graph structure...\")\n", "\n", "kg_entities = kg.get(\"entities\", []) if isinstance(kg, dict) else []\n", "kg_relationships = kg.get(\"relationships\", []) if isinstance(kg, dict) else []\n", "\n", "context_entities = []\n", "for node in kg_entities:\n", " if not isinstance(node, dict):\n", " continue\n", " name = node.get(\"name\") or node.get(\"id\")\n", " if not name:\n", " continue\n", " context_entities.append(\n", " {\n", " \"id\": node.get(\"id\") or name,\n", " \"text\": name,\n", " \"type\": node.get(\"type\") or \"Entity\",\n", " \"metadata\": node.get(\"properties\") or {},\n", " }\n", " )\n", "\n", "context_relationships = []\n", "for rel in kg_relationships:\n", " if not isinstance(rel, dict):\n", " continue\n", " src = rel.get(\"source\")\n", " tgt = rel.get(\"target\")\n", " rtype = rel.get(\"type\")\n", " if not src or not tgt or not rtype:\n", " continue\n", " context_relationships.append({\"source_id\": src, \"target_id\": tgt, \"type\": rtype})\n", "\n", "cg.build_from_entities_and_relationships(context_entities, context_relationships)\n", "print(f\"Context Graph: {cg.stats()['node_count']} nodes, {cg.stats()['edge_count']} edges\")\n", "\n", "# 5. Index Entities for Vector Retrieval (Batch Store)\n", "# We transform entities into \"documents\" so the Vector Store can index them.\n", "# This allows the Agent to \"find\" the graph nodes using semantic search.\n", "print(\"Indexing entities into Vector Store...\")\n", "\n", "entity_documents = []\n", "for node in kg_entities:\n", " # Create a rich textual description for the embedding\n", " if not isinstance(node, dict):\n", " continue\n", " name = node.get(\"name\") or node.get(\"id\")\n", " if not name:\n", " continue\n", " description = f\"{name} is a {node.get('type', 'Entity')}.\"\n", " props = node.get('properties', {})\n", " if props:\n", " # Flatten properties into string for better semantic context\n", " prop_str = \", \".join([f\"{k}: {v}\" for k,v in props.items() if isinstance(v, (str, int, float))])\n", " description += f\" Properties: {prop_str}\"\n", " \n", " # Create document object\n", " entity_documents.append({\n", " \"content\": description,\n", " \"metadata\": {\n", " \"source\": \"knowledge_graph\",\n", " \"original_id\": node.get(\"id\") or name,\n", " \"type\": node.get('type', 'Entity')\n", " }\n", " })\n", "\n", "# Batch store all entity descriptions\n", "# extract_entities=False because we are storing the entities themselves\n", "context.store(entity_documents, extract_entities=False)\n", "print(f\"Successfully indexed {len(entity_documents)} entities.\")\n", "\n", "# 6. Simulate an Agent Query (GraphRAG)\n", "query = \"pandas library\"\n", "print(f\"\\nAgent Query: '{query}'\")\n", "\n", "# Retrieve context using Hybrid Search (Vector + Graph)\n", "results = context.retrieve(\n", " query,\n", " use_graph=True,\n", " expand_graph=True, # Follow edges to get related context (e.g. pandas -> Python)\n", " max_results=3\n", ")\n", "\n", "print(\"\\n--- Retrieved Context for LLM ---\")\n", "if results:\n", " for res in results:\n", " # Access dictionary keys instead of attributes\n", " print(f\"Content: {res['content']}\")\n", " print(f\"Score: {res['score']:.4f}\")\n", " \n", " # Check for related entities in the dictionary\n", " if 'related_entities' in res and res['related_entities']:\n", " # related_entities is a list of dicts, we want the 'text' or 'id'\n", " related = [e.get('text', e.get('id', 'Unknown')) for e in res['related_entities']]\n", " print(f\"Graph Expansion: {', '.join(related)}\")\n", " print(\"-\" * 30)\n", "else:\n", " print(\"No context retrieved.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(\"\\n--- Visualizing Graph ---\")\n", "visualizer = KGVisualizer(layout=\"force\", color_scheme=\"vibrant\")\n", "fig = visualizer.visualize_network(kg, output=\"interactive\")\n", "fig.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.9" } }, "nbformat": 4, "nbformat_minor": 4 }