{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Real-World Multi-Source Integration: Python Ecosystem Analysis\n",
    "\n",
    "This cookbook demonstrates how to build a **Knowledge Graph of the Python Ecosystem** using **real-world data** from live sources.\n",
    "\n",
    "We will ingest data from:\n",
    "1.  **Official Website (Web)**: `https://www.python.org/` (using `WebIngestor`)\n",
    "2.  **Package Registry (API)**: PyPI JSON API for `pandas` (using `RESTIngestor` + `FileIngestor`)\n",
    "3.  **Source Code (Repo)**: CPython GitHub Repository (using Raw Content)\n",
    "4.  **Database (DB)**: Local SQLite metrics (using `DBIngestor`)\n",
    "5.  **Live Search (MCP)**: Real-time search via Model Context Protocol (using `MCPIngestor`)\n",
    "\n",
    "**Goal**: Construct a unified graph linking Python, key libraries, source code, and live context."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Installation & Setup\n",
    "!pip install -q semantica requests beautifulsoup4 fastmcp networkx fastembed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import requests\n",
    "import tempfile\n",
    "import logging\n",
    "from datetime import datetime\n",
    "\n",
    "# Semantica Imports\n",
    "from semantica.ingest import WebIngestor, FileIngestor, RESTIngestor, MCPIngestor, DBIngestor\n",
    "from semantica.kg import GraphBuilder\n",
    "from semantica.visualization import KGVisualizer\n",
    "\n",
    "# Setup Workspace\n",
    "WORKSPACE_DIR = tempfile.mkdtemp()\n",
    "print(f\"Workspace created at: {WORKSPACE_DIR}\")\n",
    "\n",
    "# Configure Logging to show ingestion progress\n",
    "logging.basicConfig(level=logging.INFO, format='%(name)s - %(levelname)s - %(message)s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Source 1: Official Website (Web Ingestion)\n",
    "\n",
    "We use `WebIngestor` to crawl the official Python homepage. This demonstrates handling unstructured HTML content."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- 1. Ingesting Web Sources: Python.org + Docs + PEPs ---\")\n",
    "\n",
    "try:\n",
    "    web_ingestor = WebIngestor(delay=0.5)\n",
    "\n",
    "    web_targets = {\n",
    "        \"python_home\": \"https://www.python.org/\",\n",
    "        \"psf\": \"https://www.python.org/psf/\",\n",
    "        \"python_docs\": \"https://docs.python.org/3/\",\n",
    "        \"whatsnew_313\": \"https://docs.python.org/3/whatsnew/3.13.html\",\n",
    "        \"asyncio_docs\": \"https://docs.python.org/3/library/asyncio.html\",\n",
    "        \"typing_docs\": \"https://docs.python.org/3/library/typing.html\",\n",
    "        \"pep_703\": \"https://peps.python.org/pep-0703/\",\n",
    "        \"pep_8\": \"https://peps.python.org/pep-0008/\",\n",
    "        \"pep_484\": \"https://peps.python.org/pep-0484/\",\n",
    "        \"packaging_guide\": \"https://packaging.python.org/en/latest/\",\n",
    "    }\n",
    "\n",
    "    web_pages = {}\n",
    "    for key, url in web_targets.items():\n",
    "        page = web_ingestor.ingest_url(url)\n",
    "        web_pages[key] = page\n",
    "        title = getattr(page, \"title\", \"\")\n",
    "        text = getattr(page, \"text\", \"\")\n",
    "        print(f\"Ingested {key}: {url}\")\n",
    "        print(f\"  Title: {title}\")\n",
    "        print(f\"  Content Length: {len(text)} characters\")\n",
    "\n",
    "    entities = [\n",
    "        {\n",
    "            \"id\": \"Python\",\n",
    "            \"name\": \"Python\",\n",
    "            \"type\": \"ProgrammingLanguage\",\n",
    "            \"properties\": {\n",
    "                \"website\": web_targets[\"python_home\"],\n",
    "                \"docs\": web_targets[\"python_docs\"],\n",
    "            },\n",
    "            \"source\": \"python_web\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"CPython\",\n",
    "            \"name\": \"CPython\",\n",
    "            \"type\": \"Interpreter\",\n",
    "            \"properties\": {\"repo_url\": \"https://github.com/python/cpython\"},\n",
    "            \"source\": \"python_web\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"Python Software Foundation\",\n",
    "            \"name\": \"Python Software Foundation\",\n",
    "            \"type\": \"Organization\",\n",
    "            \"properties\": {\"url\": web_targets[\"psf\"]},\n",
    "            \"source\": \"python_web\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"Python Documentation\",\n",
    "            \"name\": \"Python Documentation\",\n",
    "            \"type\": \"Documentation\",\n",
    "            \"properties\": {\"url\": web_targets[\"python_docs\"]},\n",
    "            \"source\": \"python_docs\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"Python Packaging User Guide\",\n",
    "            \"name\": \"Python Packaging User Guide\",\n",
    "            \"type\": \"Documentation\",\n",
    "            \"properties\": {\"url\": web_targets[\"packaging_guide\"]},\n",
    "            \"source\": \"python_docs\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"Python 3.13\",\n",
    "            \"name\": \"Python 3.13\",\n",
    "            \"type\": \"SoftwareVersion\",\n",
    "            \"properties\": {\"release_notes\": web_targets[\"whatsnew_313\"]},\n",
    "            \"source\": \"python_docs\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"asyncio\",\n",
    "            \"name\": \"asyncio\",\n",
    "            \"type\": \"StdlibModule\",\n",
    "            \"properties\": {\"docs_url\": web_targets[\"asyncio_docs\"]},\n",
    "            \"source\": \"python_docs\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"typing\",\n",
    "            \"name\": \"typing\",\n",
    "            \"type\": \"StdlibModule\",\n",
    "            \"properties\": {\"docs_url\": web_targets[\"typing_docs\"]},\n",
    "            \"source\": \"python_docs\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"PEP 703\",\n",
    "            \"name\": \"PEP 703\",\n",
    "            \"type\": \"PEP\",\n",
    "            \"properties\": {\n",
    "                \"url\": web_targets[\"pep_703\"],\n",
    "                \"title\": getattr(web_pages.get(\"pep_703\"), \"title\", \"\"),\n",
    "            },\n",
    "            \"source\": \"pep_site\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"PEP 8\",\n",
    "            \"name\": \"PEP 8\",\n",
    "            \"type\": \"PEP\",\n",
    "            \"properties\": {\"url\": web_targets[\"pep_8\"], \"title\": getattr(web_pages.get(\"pep_8\"), \"title\", \"\")},\n",
    "            \"source\": \"pep_site\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"PEP 484\",\n",
    "            \"name\": \"PEP 484\",\n",
    "            \"type\": \"PEP\",\n",
    "            \"properties\": {\"url\": web_targets[\"pep_484\"], \"title\": getattr(web_pages.get(\"pep_484\"), \"title\", \"\")},\n",
    "            \"source\": \"pep_site\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"Global Interpreter Lock\",\n",
    "            \"name\": \"Global Interpreter Lock\",\n",
    "            \"type\": \"Concept\",\n",
    "            \"properties\": {\"abbrev\": \"GIL\"},\n",
    "            \"source\": \"pep_site\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"Type Hints\",\n",
    "            \"name\": \"Type Hints\",\n",
    "            \"type\": \"Concept\",\n",
    "            \"properties\": {},\n",
    "            \"source\": \"pep_site\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"No-GIL Build\",\n",
    "            \"name\": \"No-GIL Build\",\n",
    "            \"type\": \"Feature\",\n",
    "            \"properties\": {\"description\": \"CPython build configuration without the GIL\"},\n",
    "            \"source\": \"pep_site\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"Free-Threaded Python\",\n",
    "            \"name\": \"Free-Threaded Python\",\n",
    "            \"type\": \"Feature\",\n",
    "            \"properties\": {\"description\": \"Python builds that allow threads without a global lock\"},\n",
    "            \"source\": \"pep_site\",\n",
    "        },\n",
    "        {\n",
    "            \"id\": \"Packaging\",\n",
    "            \"name\": \"Packaging\",\n",
    "            \"type\": \"Concept\",\n",
    "            \"properties\": {},\n",
    "            \"source\": \"python_docs\",\n",
    "        },\n",
    "    ]\n",
    "\n",
    "    relationships = [\n",
    "        {\"source\": \"Python Software Foundation\", \"target\": \"Python\", \"type\": \"governs\"},\n",
    "        {\"source\": \"Python Software Foundation\", \"target\": \"CPython\", \"type\": \"supports\"},\n",
    "        {\"source\": \"CPython\", \"target\": \"Python\", \"type\": \"implements\"},\n",
    "        {\"source\": \"Python Documentation\", \"target\": \"Python\", \"type\": \"documents\"},\n",
    "        {\"source\": \"Python Documentation\", \"target\": \"Python 3.13\", \"type\": \"documents\"},\n",
    "        {\"source\": \"Python Documentation\", \"target\": \"asyncio\", \"type\": \"documents\"},\n",
    "        {\"source\": \"Python Documentation\", \"target\": \"typing\", \"type\": \"documents\"},\n",
    "        {\"source\": \"Python 3.13\", \"target\": \"Python\", \"type\": \"version_of\"},\n",
    "        {\"source\": \"asyncio\", \"target\": \"Python\", \"type\": \"stdlib_of\"},\n",
    "        {\"source\": \"typing\", \"target\": \"Python\", \"type\": \"stdlib_of\"},\n",
    "        {\"source\": \"Python Packaging User Guide\", \"target\": \"Packaging\", \"type\": \"documents\"},\n",
    "        {\"source\": \"Python Packaging User Guide\", \"target\": \"PyPI\", \"type\": \"mentions\"},\n",
    "        {\"source\": \"PEP 703\", \"target\": \"No-GIL Build\", \"type\": \"proposes\"},\n",
    "        {\"source\": \"PEP 703\", \"target\": \"Free-Threaded Python\", \"type\": \"proposes\"},\n",
    "        {\"source\": \"PEP 703\", \"target\": \"Global Interpreter Lock\", \"type\": \"discusses\"},\n",
    "        {\"source\": \"No-GIL Build\", \"target\": \"Python 3.13\", \"type\": \"planned_for\"},\n",
    "        {\"source\": \"PEP 703\", \"target\": \"CPython\", \"type\": \"targets\"},\n",
    "        {\"source\": \"PEP 8\", \"target\": \"Python\", \"type\": \"style_guide_for\"},\n",
    "        {\"source\": \"PEP 484\", \"target\": \"Type Hints\", \"type\": \"introduces\"},\n",
    "        {\"source\": \"Type Hints\", \"target\": \"typing\", \"type\": \"implemented_by\"},\n",
    "    ]\n",
    "\n",
    "    source_web = {\n",
    "        \"name\": \"Python Web + Docs + PEPs + Packaging\",\n",
    "        \"type\": \"unstructured_web\",\n",
    "        \"entities\": entities,\n",
    "        \"relationships\": relationships,\n",
    "    }\n",
    "\n",
    "except Exception as e:\n",
    "    print(f\"Web Ingestion Failed: {e}\")\n",
    "    source_web = {\n",
    "        \"name\": \"Python Web + Docs + PEPs + Packaging (Offline)\",\n",
    "        \"type\": \"unstructured_web\",\n",
    "        \"entities\": [\n",
    "            {\"id\": \"Python\", \"name\": \"Python\", \"type\": \"ProgrammingLanguage\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"Python Documentation\", \"name\": \"Python Documentation\", \"type\": \"Documentation\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"CPython\", \"name\": \"CPython\", \"type\": \"Interpreter\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"Python Software Foundation\", \"name\": \"Python Software Foundation\", \"type\": \"Organization\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"PEP 703\", \"name\": \"PEP 703\", \"type\": \"PEP\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"Global Interpreter Lock\", \"name\": \"Global Interpreter Lock\", \"type\": \"Concept\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"Type Hints\", \"name\": \"Type Hints\", \"type\": \"Concept\", \"properties\": {}, \"source\": \"offline\"},\n",
    "        ],\n",
    "        \"relationships\": [\n",
    "            {\"source\": \"Python Software Foundation\", \"target\": \"Python\", \"type\": \"governs\"},\n",
    "            {\"source\": \"CPython\", \"target\": \"Python\", \"type\": \"implements\"},\n",
    "            {\"source\": \"Python Documentation\", \"target\": \"Python\", \"type\": \"documents\"},\n",
    "            {\"source\": \"PEP 703\", \"target\": \"Global Interpreter Lock\", \"type\": \"discusses\"},\n",
    "            {\"source\": \"Type Hints\", \"target\": \"Python\", \"type\": \"feature_of\"}\n",
    "        ],\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Source 2: Package Registry (API -> File Ingestion)\n",
    "\n",
    "We fetch live metadata for the `pandas` library from PyPI's JSON API. We save this as a JSON file and then ingest it using `FileIngestor` to demonstrate structured file handling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n--- 2. Ingesting API Sources: PyPI (pandas, numpy, scipy, matplotlib, scikit-learn, requests, fastapi) ---\")\n",
    "\n",
    "import re\n",
    "from urllib.parse import urlparse\n",
    "\n",
    "packages = [\"pandas\", \"numpy\", \"scipy\", \"matplotlib\", \"scikit-learn\", \"requests\", \"fastapi\"]\n",
    "file_ingestor = FileIngestor()\n",
    "api_ingestor = RESTIngestor(timeout=30)\n",
    "\n",
    "entities = [\n",
    "    {\"id\": \"PyPI\", \"name\": \"PyPI\", \"type\": \"PackageRegistry\", \"properties\": {\"url\": \"https://pypi.org/\"}, \"source\": \"pypi_api\"},\n",
    "    {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"pypi_api\"},\n",
    "]\n",
    "relationships = []\n",
    "\n",
    "seen_entities = {e[\"id\"] for e in entities}\n",
    "\n",
    "def _dep_name(req: str) -> str:\n",
    "    if not req:\n",
    "        return \"\"\n",
    "    req = req.split(\";\")[0].strip()\n",
    "    m = re.match(r\"^([A-Za-z0-9_.-]+)\", req)\n",
    "    return (m.group(1) if m else \"\").strip()\n",
    "\n",
    "def _url_entity_id(url: str) -> str:\n",
    "    return f\"URL::{url.strip()}\"\n",
    "\n",
    "def _add_url_entity(url: str, label: str, source: str) -> str:\n",
    "    url = (url or \"\").strip()\n",
    "    if not url:\n",
    "        return \"\"\n",
    "    uid = _url_entity_id(url)\n",
    "    if uid not in seen_entities:\n",
    "        entities.append({\"id\": uid, \"name\": label or url, \"type\": \"WebResource\", \"properties\": {\"url\": url}, \"source\": source})\n",
    "        seen_entities.add(uid)\n",
    "    return uid\n",
    "\n",
    "def _is_github(url: str) -> bool:\n",
    "    try:\n",
    "        return urlparse(url).netloc.lower().endswith(\"github.com\")\n",
    "    except Exception:\n",
    "        return False\n",
    "\n",
    "try:\n",
    "    for pkg in packages:\n",
    "        pypi_url = f\"https://pypi.org/pypi/{pkg}/json\"\n",
    "        local_json_path = os.path.join(WORKSPACE_DIR, f\"{pkg}_pypi.json\")\n",
    "\n",
    "        api_data = api_ingestor.ingest_endpoint(pypi_url)\n",
    "        data = api_data.data if isinstance(api_data.data, dict) else {}\n",
    "\n",
    "        with open(local_json_path, \"w\", encoding=\"utf-8\") as f:\n",
    "            json.dump(data, f)\n",
    "\n",
    "        ingested_file = file_ingestor.ingest_file(local_json_path)\n",
    "        print(f\"Ingested File: {ingested_file.name} ({ingested_file.size} bytes)\")\n",
    "\n",
    "        info = data.get(\"info\", {})\n",
    "        project_urls = info.get(\"project_urls\") or {}\n",
    "\n",
    "        if pkg not in seen_entities:\n",
    "            entities.append(\n",
    "                {\n",
    "                    \"id\": pkg,\n",
    "                    \"name\": pkg,\n",
    "                    \"type\": \"Library\",\n",
    "                    \"properties\": {\n",
    "                        \"version\": info.get(\"version\"),\n",
    "                        \"summary\": info.get(\"summary\"),\n",
    "                        \"license\": info.get(\"license\"),\n",
    "                        \"requires_python\": info.get(\"requires_python\"),\n",
    "                    },\n",
    "                    \"source\": \"pypi_api\",\n",
    "                }\n",
    "            )\n",
    "            seen_entities.add(pkg)\n",
    "\n",
    "        relationships.append({\"source\": pkg, \"target\": \"PyPI\", \"type\": \"published_on\"})\n",
    "        relationships.append({\"source\": pkg, \"target\": \"Python\", \"type\": \"written_in\"})\n",
    "\n",
    "        home_page = (info.get(\"home_page\") or \"\").strip()\n",
    "        if home_page:\n",
    "            hp_id = _add_url_entity(home_page, f\"{pkg} homepage\", \"pypi_api\")\n",
    "            if hp_id:\n",
    "                relationships.append({\"source\": pkg, \"target\": hp_id, \"type\": \"has_homepage\"})\n",
    "\n",
    "        normalized_project_urls = {}\n",
    "        for k, v in project_urls.items():\n",
    "            if not k or not v:\n",
    "                continue\n",
    "            normalized_project_urls[str(k).strip().lower()] = str(v).strip()\n",
    "\n",
    "        for key, url in normalized_project_urls.items():\n",
    "            if not url:\n",
    "                continue\n",
    "            label = f\"{pkg} {key}\"\n",
    "            url_id = _add_url_entity(url, label, \"pypi_api\")\n",
    "            if not url_id:\n",
    "                continue\n",
    "            if \"doc\" in key:\n",
    "                relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"has_documentation\"})\n",
    "            elif \"bug\" in key or \"issue\" in key:\n",
    "                relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"issues_at\"})\n",
    "            elif \"source\" in key or \"github\" in key:\n",
    "                relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"has_source\"})\n",
    "            else:\n",
    "                relationships.append({\"source\": pkg, \"target\": url_id, \"type\": \"related_resource\"})\n",
    "            if _is_github(url):\n",
    "                relationships.append({\"source\": url_id, \"target\": \"GitHub\", \"type\": \"hosted_on\"})\n",
    "\n",
    "        requires_dist = info.get(\"requires_dist\") or []\n",
    "        dep_names = []\n",
    "        for req in requires_dist:\n",
    "            name = _dep_name(req)\n",
    "            if name:\n",
    "                dep_names.append(name)\n",
    "\n",
    "        unique_deps = sorted(set(dep_names))[:12]\n",
    "        for dep in unique_deps:\n",
    "            if dep not in seen_entities:\n",
    "                entities.append({\"id\": dep, \"name\": dep, \"type\": \"Library\", \"properties\": {}, \"source\": \"pypi_requires_dist\"})\n",
    "                seen_entities.add(dep)\n",
    "            relationships.append({\"source\": pkg, \"target\": dep, \"type\": \"depends_on\"})\n",
    "\n",
    "        print(f\"Extracted Entity: {pkg} (v{info.get('version')}) with {len(unique_deps)} dependencies (capped)\")\n",
    "\n",
    "    for rel in [\n",
    "        {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
    "        {\"source\": \"scipy\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
    "        {\"source\": \"matplotlib\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
    "        {\"source\": \"scikit-learn\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
    "        {\"source\": \"scikit-learn\", \"target\": \"scipy\", \"type\": \"built_on\"},\n",
    "    ]:\n",
    "        relationships.append(rel)\n",
    "\n",
    "    source_api = {\n",
    "        \"name\": \"PyPI Registry\",\n",
    "        \"type\": \"structured_api\",\n",
    "        \"entities\": entities,\n",
    "        \"relationships\": relationships,\n",
    "    }\n",
    "\n",
    "except Exception as e:\n",
    "    print(f\"API Ingestion Failed: {e}\")\n",
    "    source_api = {\n",
    "        \"name\": \"PyPI Registry (Offline)\",\n",
    "        \"type\": \"structured_api\",\n",
    "        \"entities\": [\n",
    "            {\"id\": \"PyPI\", \"name\": \"PyPI\", \"type\": \"PackageRegistry\", \"properties\": {\"url\": \"https://pypi.org/\"}, \"source\": \"offline\"},\n",
    "            {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"offline\"},\n",
    "            {\"id\": \"pandas\", \"name\": \"pandas\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"numpy\", \"name\": \"numpy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"scipy\", \"name\": \"scipy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"matplotlib\", \"name\": \"matplotlib\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"scikit-learn\", \"name\": \"scikit-learn\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"requests\", \"name\": \"requests\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
    "            {\"id\": \"fastapi\", \"name\": \"fastapi\", \"type\": \"Library\", \"properties\": {}, \"source\": \"offline\"},\n",
    "        ],\n",
    "        \"relationships\": [\n",
    "            {\"source\": \"pandas\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
    "            {\"source\": \"numpy\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
    "            {\"source\": \"scipy\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
    "            {\"source\": \"matplotlib\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
    "            {\"source\": \"scikit-learn\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
    "            {\"source\": \"requests\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
    "            {\"source\": \"fastapi\", \"target\": \"PyPI\", \"type\": \"published_on\"},\n",
    "            {\"source\": \"pandas\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
    "            {\"source\": \"numpy\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
    "            {\"source\": \"scipy\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
    "            {\"source\": \"matplotlib\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
    "            {\"source\": \"scikit-learn\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
    "            {\"source\": \"requests\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
    "            {\"source\": \"fastapi\", \"target\": \"Python\", \"type\": \"written_in\"},\n",
    "            {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
    "            {\"source\": \"scipy\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
    "            {\"source\": \"scikit-learn\", \"target\": \"numpy\", \"type\": \"built_on\"},\n",
    "            {\"source\": \"scikit-learn\", \"target\": \"scipy\", \"type\": \"built_on\"}\n",
    "        ],\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Source 3: Source Code Repository (Raw Content)\n",
    "\n",
    "We fetch the raw `README.rst` from the official CPython GitHub repository. This represents unstructured technical documentation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n--- 3. Ingesting Repo Sources: CPython + pandas + NumPy + SciPy (GitHub) ---\")\n",
    "\n",
    "try:\n",
    "    repo_ingestor = WebIngestor(delay=0.5)\n",
    "\n",
    "    headers = {\"Accept\": \"application/vnd.github+json\", \"User-Agent\": \"semantica-cookbook\"}\n",
    "\n",
    "    repo_targets = {\n",
    "        \"python/cpython\": {\"readme_raw\": \"https://raw.githubusercontent.com/python/cpython/main/README.rst\", \"library\": \"Python\"},\n",
    "        \"pandas-dev/pandas\": {\"readme_raw\": \"https://raw.githubusercontent.com/pandas-dev/pandas/main/README.md\", \"library\": \"pandas\"},\n",
    "        \"numpy/numpy\": {\"readme_raw\": \"https://raw.githubusercontent.com/numpy/numpy/main/README.md\", \"library\": \"numpy\"},\n",
    "        \"scipy/scipy\": {\"readme_raw\": \"https://raw.githubusercontent.com/scipy/scipy/main/README.rst\", \"library\": \"scipy\"},\n",
    "    }\n",
    "\n",
    "    entities = [\n",
    "        {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"github_api\"}\n",
    "    ]\n",
    "    relationships = []\n",
    "\n",
    "    for repo_full, cfg in repo_targets.items():\n",
    "        readme_url = cfg[\"readme_raw\"]\n",
    "        readme = repo_ingestor.ingest_url(readme_url)\n",
    "        print(f\"Fetched {repo_full} README: {len(readme.text)} chars\")\n",
    "\n",
    "        repo_url = f\"https://github.com/{repo_full}\"\n",
    "        entities.append(\n",
    "            {\n",
    "                \"id\": repo_full,\n",
    "                \"name\": repo_full,\n",
    "                \"type\": \"Repository\",\n",
    "                \"properties\": {\"repo_url\": repo_url, \"readme_url\": readme_url, \"readme_title\": getattr(readme, \"title\", \"\")},\n",
    "                \"source\": \"github_raw\",\n",
    "            }\n",
    "        )\n",
    "        relationships.append({\"source\": repo_full, \"target\": \"GitHub\", \"type\": \"hosted_on\"})\n",
    "\n",
    "        owner = repo_full.split(\"/\")[0]\n",
    "        org_id = f\"GitHubOrg::{owner}\"\n",
    "        entities.append({\"id\": org_id, \"name\": owner, \"type\": \"Organization\", \"properties\": {\"url\": f\"https://github.com/{owner}\"}, \"source\": \"github_api\"})\n",
    "        relationships.append({\"source\": repo_full, \"target\": org_id, \"type\": \"owned_by\"})\n",
    "\n",
    "        lib = cfg.get(\"library\")\n",
    "        if lib:\n",
    "            relationships.append({\"source\": lib, \"target\": repo_full, \"type\": \"source_code_in\"})\n",
    "            relationships.append({\"source\": repo_full, \"target\": lib, \"type\": \"source_code_for\"})\n",
    "\n",
    "        repo_api_url = f\"https://api.github.com/repos/{repo_full}\"\n",
    "        repo_resp = requests.get(repo_api_url, headers=headers, timeout=30)\n",
    "        if repo_resp.status_code == 200:\n",
    "            meta = repo_resp.json() or {}\n",
    "            repo_props = {\n",
    "                \"stars\": meta.get(\"stargazers_count\"),\n",
    "                \"forks\": meta.get(\"forks_count\"),\n",
    "                \"open_issues\": meta.get(\"open_issues_count\"),\n",
    "                \"language\": meta.get(\"language\"),\n",
    "                \"updated_at\": meta.get(\"updated_at\"),\n",
    "            }\n",
    "            for ent in entities:\n",
    "                if isinstance(ent, dict) and ent.get(\"id\") == repo_full:\n",
    "                    ent.setdefault(\"properties\", {}).update({k: v for k, v in repo_props.items() if v is not None})\n",
    "                    break\n",
    "        else:\n",
    "            print(f\"Repo metadata unavailable for {repo_full} (status {repo_resp.status_code}).\")\n",
    "\n",
    "    relationships.append({\"source\": \"python/cpython\", \"target\": \"CPython\", \"type\": \"repository_for\"})\n",
    "    relationships.append({\"source\": \"python/cpython\", \"target\": \"Python\", \"type\": \"implements\"})\n",
    "\n",
    "    gh_releases_url = \"https://api.github.com/repos/python/cpython/releases?per_page=8\"\n",
    "    release_resp = requests.get(gh_releases_url, headers=headers, timeout=30)\n",
    "    if release_resp.status_code == 200:\n",
    "        releases = release_resp.json() or []\n",
    "        for r in releases:\n",
    "            tag = r.get(\"tag_name\")\n",
    "            if not tag:\n",
    "                continue\n",
    "            release_id = f\"Release::python/cpython::{tag}\"\n",
    "            entities.append(\n",
    "                {\n",
    "                    \"id\": release_id,\n",
    "                    \"name\": f\"CPython {tag}\",\n",
    "                    \"type\": \"Release\",\n",
    "                    \"properties\": {\"tag\": tag, \"published_at\": r.get(\"published_at\"), \"url\": r.get(\"html_url\")},\n",
    "                    \"source\": \"github_api\",\n",
    "                }\n",
    "            )\n",
    "            relationships.append({\"source\": release_id, \"target\": \"python/cpython\", \"type\": \"release_of\"})\n",
    "            if tag.startswith(\"v\") and len(tag) >= 4:\n",
    "                major_minor = \".\".join(tag.lstrip(\"v\").split(\".\")[:2])\n",
    "                relationships.append({\"source\": release_id, \"target\": f\"Python {major_minor}\", \"type\": \"implements\"})\n",
    "    else:\n",
    "        print(f\"GitHub releases unavailable (status {release_resp.status_code}).\")\n",
    "\n",
    "    source_repo = {\n",
    "        \"name\": \"GitHub Repos + Metadata + Releases\",\n",
    "        \"type\": \"unstructured_repo\",\n",
    "        \"entities\": entities,\n",
    "        \"relationships\": relationships,\n",
    "    }\n",
    "\n",
    "except Exception as e:\n",
    "    print(f\"Repo Ingestion Failed: {e}\")\n",
    "    source_repo = {\n",
    "        \"name\": \"GitHub Repos + Metadata + Releases (Offline)\",\n",
    "        \"type\": \"unstructured_repo\",\n",
    "        \"entities\": [\n",
    "            {\"id\": \"GitHub\", \"name\": \"GitHub\", \"type\": \"Platform\", \"properties\": {\"url\": \"https://github.com/\"}, \"source\": \"offline\"},\n",
    "            {\"id\": \"python/cpython\", \"name\": \"python/cpython\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/python/cpython\"}, \"source\": \"offline\"},\n",
    "            {\"id\": \"pandas-dev/pandas\", \"name\": \"pandas-dev/pandas\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/pandas-dev/pandas\"}, \"source\": \"offline\"},\n",
    "            {\"id\": \"numpy/numpy\", \"name\": \"numpy/numpy\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/numpy/numpy\"}, \"source\": \"offline\"},\n",
    "            {\"id\": \"scipy/scipy\", \"name\": \"scipy/scipy\", \"type\": \"Repository\", \"properties\": {\"repo_url\": \"https://github.com/scipy/scipy\"}, \"source\": \"offline\"},\n",
    "        ],\n",
    "        \"relationships\": [\n",
    "            {\"source\": \"python/cpython\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n",
    "            {\"source\": \"pandas-dev/pandas\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n",
    "            {\"source\": \"numpy/numpy\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n",
    "            {\"source\": \"scipy/scipy\", \"target\": \"GitHub\", \"type\": \"hosted_on\"},\n",
    "            {\"source\": \"python/cpython\", \"target\": \"Python\", \"type\": \"implements\"},\n",
    "            {\"source\": \"pandas\", \"target\": \"pandas-dev/pandas\", \"type\": \"source_code_in\"},\n",
    "            {\"source\": \"numpy\", \"target\": \"numpy/numpy\", \"type\": \"source_code_in\"},\n",
    "            {\"source\": \"scipy\", \"target\": \"scipy/scipy\", \"type\": \"source_code_in\"}\n",
    "        ],\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Source 4: Database (SQLite via DBIngestor)\n",
    "\n",
    "We ingest structured data from a local SQLite database using `DBIngestor`. This demonstrates database connectivity and SQL query extraction.\n",
    "\n",
    "**Note:** This example creates a small SQLite database inside the temporary workspace so it works offline.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n--- 4. Ingesting Database Source: SQLite (local) ---\")\n",
    "\n",
    "import sqlite3\n",
    "\n",
    "db_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem_metrics.sqlite\")\n",
    "\n",
    "try:\n",
    "    conn = sqlite3.connect(db_path)\n",
    "    cur = conn.cursor()\n",
    "    cur.execute(\n",
    "        \"\"\"\n",
    "        CREATE TABLE IF NOT EXISTS library_metrics (\n",
    "            library TEXT PRIMARY KEY,\n",
    "            downloads INTEGER,\n",
    "            stars INTEGER,\n",
    "            last_updated TEXT\n",
    "        )\n",
    "        \"\"\"\n",
    "    )\n",
    "\n",
    "    cur.execute(\n",
    "        \"\"\"\n",
    "        CREATE TABLE IF NOT EXISTS library_categories (\n",
    "            library TEXT,\n",
    "            category TEXT,\n",
    "            PRIMARY KEY (library, category)\n",
    "        )\n",
    "        \"\"\"\n",
    "    )\n",
    "\n",
    "    cur.execute(\n",
    "        \"\"\"\n",
    "        CREATE TABLE IF NOT EXISTS library_dependencies (\n",
    "            library TEXT,\n",
    "            dependency TEXT,\n",
    "            relation TEXT,\n",
    "            PRIMARY KEY (library, dependency, relation)\n",
    "        )\n",
    "        \"\"\"\n",
    "    )\n",
    "\n",
    "    sample_rows = [\n",
    "        (\"numpy\", 80000000, 28000, datetime.utcnow().isoformat()),\n",
    "        (\"pandas\", 50000000, 45000, datetime.utcnow().isoformat()),\n",
    "        (\"scipy\", 20000000, 12000, datetime.utcnow().isoformat()),\n",
    "        (\"matplotlib\", 25000000, 21000, datetime.utcnow().isoformat()),\n",
    "        (\"scikit-learn\", 18000000, 60000, datetime.utcnow().isoformat()),\n",
    "        (\"requests\", 90000000, 52000, datetime.utcnow().isoformat()),\n",
    "        (\"fastapi\", 22000000, 80000, datetime.utcnow().isoformat()),\n",
    "    ]\n",
    "    cur.executemany(\n",
    "        \"INSERT OR REPLACE INTO library_metrics (library, downloads, stars, last_updated) VALUES (?, ?, ?, ?)\",\n",
    "        sample_rows,\n",
    "    )\n",
    "\n",
    "    category_rows = [\n",
    "        (\"numpy\", \"Numerical\"),\n",
    "        (\"pandas\", \"Data Analysis\"),\n",
    "        (\"scipy\", \"Scientific Computing\"),\n",
    "        (\"matplotlib\", \"Visualization\"),\n",
    "        (\"scikit-learn\", \"Machine Learning\"),\n",
    "        (\"requests\", \"Networking\"),\n",
    "        (\"fastapi\", \"Web\"),\n",
    "    ]\n",
    "    cur.executemany(\n",
    "        \"INSERT OR REPLACE INTO library_categories (library, category) VALUES (?, ?)\",\n",
    "        category_rows,\n",
    "    )\n",
    "\n",
    "    dependency_rows = [\n",
    "        (\"pandas\", \"numpy\", \"depends_on\"),\n",
    "        (\"scipy\", \"numpy\", \"depends_on\"),\n",
    "        (\"matplotlib\", \"numpy\", \"depends_on\"),\n",
    "        (\"scikit-learn\", \"numpy\", \"depends_on\"),\n",
    "        (\"scikit-learn\", \"scipy\", \"depends_on\"),\n",
    "        (\"fastapi\", \"pydantic\", \"depends_on\"),\n",
    "        (\"fastapi\", \"starlette\", \"built_on\"),\n",
    "        (\"requests\", \"urllib3\", \"depends_on\"),\n",
    "        (\"requests\", \"certifi\", \"depends_on\"),\n",
    "    ]\n",
    "    cur.executemany(\n",
    "        \"INSERT OR REPLACE INTO library_dependencies (library, dependency, relation) VALUES (?, ?, ?)\",\n",
    "        dependency_rows,\n",
    "    )\n",
    "    conn.commit()\n",
    "    conn.close()\n",
    "\n",
    "    db_ingestor = DBIngestor()\n",
    "    sqlite_conn_str = f\"sqlite:///{db_path}\"\n",
    "\n",
    "    rows = db_ingestor.execute_query(\n",
    "        sqlite_conn_str,\n",
    "        \"SELECT library, downloads, stars, last_updated FROM library_metrics WHERE downloads >= :min_downloads\",\n",
    "        min_downloads=1000000,\n",
    "    )\n",
    "\n",
    "    cat_rows = db_ingestor.execute_query(\n",
    "        sqlite_conn_str,\n",
    "        \"SELECT library, category FROM library_categories\",\n",
    "    )\n",
    "\n",
    "    dep_rows = db_ingestor.execute_query(\n",
    "        sqlite_conn_str,\n",
    "        \"SELECT library, dependency, relation FROM library_dependencies\",\n",
    "    )\n",
    "\n",
    "    if not rows and not cat_rows and not dep_rows:\n",
    "        raise RuntimeError(\"No data found in SQLite database\")\n",
    "\n",
    "    entities = []\n",
    "    relationships = []\n",
    "\n",
    "    entities.append({\"id\": \"SQLite\", \"name\": \"SQLite\", \"type\": \"Database\", \"properties\": {\"path\": db_path}, \"source\": \"sqlite_db\"})\n",
    "    for row in rows:\n",
    "        lib = row.get(\"library\")\n",
    "        if not lib:\n",
    "            continue\n",
    "        entities.append({\"id\": lib, \"name\": lib, \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db\"})\n",
    "        metric_id = f\"{lib}::metrics\"\n",
    "        entities.append(\n",
    "            {\n",
    "                \"id\": metric_id,\n",
    "                \"name\": f\"{lib} metrics\",\n",
    "                \"type\": \"LibraryMetrics\",\n",
    "                \"properties\": {\n",
    "                    \"downloads\": row.get(\"downloads\"),\n",
    "                    \"stars\": row.get(\"stars\"),\n",
    "                    \"last_updated\": row.get(\"last_updated\"),\n",
    "                },\n",
    "                \"source\": \"sqlite_db\",\n",
    "            }\n",
    "        )\n",
    "        relationships.append({\"source\": metric_id, \"target\": lib, \"type\": \"metrics_for\"})\n",
    "        relationships.append({\"source\": lib, \"target\": \"Python\", \"type\": \"ecosystem_of\"})\n",
    "        relationships.append({\"source\": metric_id, \"target\": \"SQLite\", \"type\": \"stored_in\"})\n",
    "\n",
    "    for row in cat_rows:\n",
    "        lib = row.get(\"library\")\n",
    "        cat = row.get(\"category\")\n",
    "        if not lib or not cat:\n",
    "            continue\n",
    "        cat_id = f\"Category::{cat}\"\n",
    "        entities.append({\"id\": cat_id, \"name\": cat, \"type\": \"Category\", \"properties\": {}, \"source\": \"sqlite_db\"})\n",
    "        relationships.append({\"source\": lib, \"target\": cat_id, \"type\": \"categorized_as\"})\n",
    "\n",
    "    for row in dep_rows:\n",
    "        lib = row.get(\"library\")\n",
    "        dep = row.get(\"dependency\")\n",
    "        rel = row.get(\"relation\") or \"depends_on\"\n",
    "        if not lib or not dep:\n",
    "            continue\n",
    "        entities.append({\"id\": dep, \"name\": dep, \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db\"})\n",
    "        relationships.append({\"source\": lib, \"target\": dep, \"type\": rel})\n",
    "\n",
    "    print(f\"Ingested {len(entities)} rows from SQLite metrics table\")\n",
    "    source_db = {\n",
    "        \"name\": \"SQLite Metrics\",\n",
    "        \"type\": \"database\",\n",
    "        \"entities\": entities,\n",
    "        \"relationships\": relationships,\n",
    "    }\n",
    "\n",
    "except Exception as e:\n",
    "    print(f\"Database ingestion via DBIngestor skipped: {e}\")\n",
    "    entities = [\n",
    "        {\"id\": \"SQLite\", \"name\": \"SQLite\", \"type\": \"Database\", \"properties\": {\"path\": db_path}, \"source\": \"sqlite_db_offline\"},\n",
    "        {\"id\": \"pandas\", \"name\": \"pandas\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n",
    "        {\"id\": \"pandas::metrics\", \"name\": \"pandas metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 50000000, \"stars\": 45000}, \"source\": \"sqlite_db_offline\"},\n",
    "        {\"id\": \"numpy\", \"name\": \"numpy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n",
    "        {\"id\": \"numpy::metrics\", \"name\": \"numpy metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 80000000, \"stars\": 28000}, \"source\": \"sqlite_db_offline\"},\n",
    "        {\"id\": \"scipy\", \"name\": \"scipy\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n",
    "        {\"id\": \"scipy::metrics\", \"name\": \"scipy metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 20000000, \"stars\": 12000}, \"source\": \"sqlite_db_offline\"},\n",
    "        {\"id\": \"fastapi\", \"name\": \"fastapi\", \"type\": \"Library\", \"properties\": {}, \"source\": \"sqlite_db_offline\"},\n",
    "        {\"id\": \"fastapi::metrics\", \"name\": \"fastapi metrics\", \"type\": \"LibraryMetrics\", \"properties\": {\"downloads\": 22000000, \"stars\": 80000}, \"source\": \"sqlite_db_offline\"},\n",
    "    ]\n",
    "    relationships = [\n",
    "        {\"source\": \"pandas::metrics\", \"target\": \"pandas\", \"type\": \"metrics_for\"},\n",
    "        {\"source\": \"numpy::metrics\", \"target\": \"numpy\", \"type\": \"metrics_for\"},\n",
    "        {\"source\": \"scipy::metrics\", \"target\": \"scipy\", \"type\": \"metrics_for\"},\n",
    "        {\"source\": \"fastapi::metrics\", \"target\": \"fastapi\", \"type\": \"metrics_for\"},\n",
    "        {\"source\": \"pandas\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n",
    "        {\"source\": \"numpy\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n",
    "        {\"source\": \"scipy\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n",
    "        {\"source\": \"fastapi\", \"target\": \"Python\", \"type\": \"ecosystem_of\"},\n",
    "        {\"source\": \"pandas\", \"target\": \"numpy\", \"type\": \"depends_on\"},\n",
    "        {\"source\": \"fastapi\", \"target\": \"pydantic\", \"type\": \"depends_on\"}\n",
    "    ]\n",
    "    source_db = {\n",
    "        \"name\": \"SQLite Metrics (Offline)\",\n",
    "        \"type\": \"database\",\n",
    "        \"entities\": entities,\n",
    "        \"relationships\": relationships,\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Source 5: Model Context Protocol (MCP)\n",
    "\n",
    "We attempt to connect to a local MCP server (e.g., a Web Search tool) to get live context. \n",
    "\n",
    "**Note:** If no MCP server is running at `localhost:8000`, this section will gracefully fallback to simulated data, but the code provided is production-ready for MCP integration.\n",
    "\n",
    "Useful MCP server directories / references:\n",
    "- https://github.com/modelcontextprotocol/servers\n",
    "- https://glama.ai/mcp/servers\n",
    "- https://github.com/punkpeye/awesome-mcp-servers\n",
    "- https://github.com/wong2/awesome-mcp-servers\n",
    "- https://mcp.so\n",
    "- Brave Search MCP Server: https://github.com/brave/brave-search-mcp-server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n--- 5. Ingesting via MCP (Model Context Protocol) ---\")\n",
    "\n",
    "mcp_server_urls = [\n",
    "    \"http://localhost:8000/mcp\",\n",
    "    \"http://localhost:8000/sse\",\n",
    "    \"http://127.0.0.1:8000/mcp\",\n",
    "    \"http://127.0.0.1:8000/sse\",\n",
    "    \"http://localhost:8080/mcp\",\n",
    "    \"http://localhost:8080/sse\",\n",
    "    \"http://127.0.0.1:8080/mcp\",\n",
    "    \"http://127.0.0.1:8080/sse\",\n",
    "]\n",
    "\n",
    "try:\n",
    "    mcp_client_logger = logging.getLogger(\"semantica.mcp_client\")\n",
    "    mcp_ingestor_logger = logging.getLogger(\"semantica.mcp_ingestor\")\n",
    "    prev_client_level = mcp_client_logger.level\n",
    "    prev_ingestor_level = mcp_ingestor_logger.level\n",
    "    mcp_client_logger.setLevel(logging.CRITICAL)\n",
    "    mcp_ingestor_logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "    mcp = MCPIngestor()\n",
    "    connected_url = None\n",
    "    for url in mcp_server_urls:\n",
    "        try:\n",
    "            mcp.connect(\"web_search\", url=url)\n",
    "            connected_url = url\n",
    "            break\n",
    "        except Exception:\n",
    "            continue\n",
    "    if not connected_url:\n",
    "        raise RuntimeError(\"No MCP server reachable on known local SSE endpoints\")\n",
    "\n",
    "    print(f\"Connected to MCP Server at {connected_url}\")\n",
    "    \n",
    "    payload = {\"query\": \"latest python 3.13 features\"}\n",
    "    mcp_result = None\n",
    "    for tool_name in [\"search\", \"brave_web_search\", \"brave_local_search\"]:\n",
    "        try:\n",
    "            mcp_result = mcp.ingest_tool_output(\"web_search\", tool_name, payload)\n",
    "            break\n",
    "        except Exception:\n",
    "            continue\n",
    "    if mcp_result is None:\n",
    "        raise RuntimeError(\"No compatible MCP search tool found\")\n",
    "    search_results = getattr(mcp_result, \"content\", mcp_result)\n",
    "    if not isinstance(search_results, dict):\n",
    "        raise RuntimeError(\"MCP tool output content was not a dict\")\n",
    "    \n",
    "    print(\"Received Live Data from MCP.\")\n",
    "    \n",
    "    raw_entities = search_results.get(\"entities\", []) or []\n",
    "    raw_relationships = search_results.get(\"relationships\", []) or []\n",
    "    normalized_entities = []\n",
    "    for ent in raw_entities:\n",
    "        if not isinstance(ent, dict):\n",
    "            continue\n",
    "        name = ent.get(\"name\") or ent.get(\"text\") or ent.get(\"id\")\n",
    "        if not name:\n",
    "            continue\n",
    "        normalized_entities.append(\n",
    "            {\n",
    "                \"id\": ent.get(\"id\") or name,\n",
    "                \"name\": name,\n",
    "                \"type\": ent.get(\"type\") or ent.get(\"label\") or \"Entity\",\n",
    "                \"properties\": ent.get(\"properties\") or ent.get(\"metadata\") or {},\n",
    "                \"source\": \"mcp_live\",\n",
    "            }\n",
    "        )\n",
    "\n",
    "    normalized_relationships = []\n",
    "    for rel in raw_relationships:\n",
    "        if not isinstance(rel, dict):\n",
    "            continue\n",
    "        src = rel.get(\"source\") or rel.get(\"subject\")\n",
    "        tgt = rel.get(\"target\") or rel.get(\"object\")\n",
    "        rtype = rel.get(\"type\") or rel.get(\"label\") or rel.get(\"predicate\")\n",
    "        if not src or not tgt or not rtype:\n",
    "            continue\n",
    "        normalized_relationships.append({\"source\": src, \"target\": tgt, \"type\": rtype, \"properties\": rel.get(\"properties\") or rel.get(\"metadata\") or {}})\n",
    "\n",
    "    endpoint_ids = {e.get(\"id\") for e in normalized_entities if isinstance(e, dict) and e.get(\"id\")}\n",
    "    for rel in normalized_relationships:\n",
    "        src = rel.get(\"source\")\n",
    "        tgt = rel.get(\"target\")\n",
    "        for node_id in [src, tgt]:\n",
    "            if node_id and node_id not in endpoint_ids:\n",
    "                normalized_entities.append({\"id\": node_id, \"name\": node_id, \"type\": \"Entity\", \"properties\": {}, \"source\": \"mcp_live\"})\n",
    "                endpoint_ids.add(node_id)\n",
    "\n",
    "    source_mcp = {\n",
    "        \"name\": \"MCP Search\",\n",
    "        \"type\": \"agent_tool\",\n",
    "        \"entities\": normalized_entities,\n",
    "        \"relationships\": normalized_relationships,\n",
    "        \"source\": \"mcp_live\"\n",
    "    }\n",
    "\n",
    "except Exception as e:\n",
    "    print(\"MCP Server not detected. Using simulated 'Live Search' data.\")\n",
    "    print(\"To enable: start a local MCP server (commonly `http://localhost:8000/mcp` or `http://localhost:8000/sse`)\")\n",
    "    \n",
    "    source_mcp = {\n",
    "        \"name\": \"MCP Search (Simulated)\",\n",
    "        \"type\": \"agent_tool\",\n",
    "        \"entities\": [\n",
    "            {\n",
    "                \"id\": \"Python 3.13\",\n",
    "                \"name\": \"Python 3.13\",\n",
    "                \"type\": \"SoftwareVersion\",\n",
    "                \"properties\": {\"status\": \"In Development\", \"feature\": \"No-GIL Build\"},\n",
    "                \"source\": \"mcp_simulated\"\n",
    "            }\n",
    "        ],\n",
    "        \"relationships\": [\n",
    "            {\"source\": \"Python 3.13\", \"target\": \"Python\", \"type\": \"version_of\"}\n",
    "        ]\n",
    "    }\n",
    "finally:\n",
    "    if 'mcp_client_logger' in locals() and 'prev_client_level' in locals():\n",
    "        mcp_client_logger.setLevel(prev_client_level)\n",
    "    if 'mcp_ingestor_logger' in locals() and 'prev_ingestor_level' in locals():\n",
    "        mcp_ingestor_logger.setLevel(prev_ingestor_level)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Phase 5: Knowledge Graph Construction\n",
    "\n",
    "We merge all these real-world data points into a single Knowledge Graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n--- Building Knowledge Graph ---\")\n",
    "\n",
    "all_sources = [source_web, source_api, source_repo, source_db, source_mcp]\n",
    "\n",
    "builder = GraphBuilder(merge_entities=True, resolve_conflicts=True)\n",
    "\n",
    "kg = builder.build(sources=all_sources) or {}\n",
    "kg.setdefault(\"entities\", [])\n",
    "kg.setdefault(\"relationships\", [])\n",
    "\n",
    "entity_ids = set()\n",
    "for node in kg.get(\"entities\", []):\n",
    "    if isinstance(node, dict):\n",
    "        node_id = node.get(\"id\") or node.get(\"entity_id\") or node.get(\"name\")\n",
    "        if node_id:\n",
    "            entity_ids.add(node_id)\n",
    "\n",
    "missing_ids = set()\n",
    "for rel in kg.get(\"relationships\", []):\n",
    "    if not isinstance(rel, dict):\n",
    "        continue\n",
    "    src = rel.get(\"source\") or rel.get(\"subject\")\n",
    "    tgt = rel.get(\"target\") or rel.get(\"object\")\n",
    "    if src and src not in entity_ids:\n",
    "        missing_ids.add(src)\n",
    "    if tgt and tgt not in entity_ids:\n",
    "        missing_ids.add(tgt)\n",
    "\n",
    "for mid in sorted(missing_ids):\n",
    "    kg[\"entities\"].append({\"id\": mid, \"name\": mid, \"type\": \"Entity\", \"properties\": {}, \"source\": \"auto\"})\n",
    "    entity_ids.add(mid)\n",
    "\n",
    "print(f\"Graph Statistics:\")\n",
    "print(f\"Nodes: {len(kg.get('entities', []))}\")\n",
    "print(f\"Edges: {len(kg.get('relationships', []))}\")\n",
    "\n",
    "# List all nodes to verify integration\n",
    "print(\"\\nEntities in Graph:\")\n",
    "for node in kg.get('entities', []):\n",
    "    name = node.get('name') or node.get('label') or node.get('id')\n",
    "    ntype = node.get('type') or node.get('label') or 'Entity'\n",
    "    print(f\"- {name} ({ntype})\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Phase 6: Advanced Graph Analytics\n",
    "\n",
    "We can perform network analysis on the constructed graph to find key entities. Here, we calculate **Degree Centrality** to identify the most connected nodes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from semantica.kg import CentralityCalculator, CommunityDetector\n",
    "\n",
    "print(\"\\n--- Running Graph Analytics (Semantica) ---\")\n",
    "\n",
    "# 1. Centrality Analysis\n",
    "centrality_calc = CentralityCalculator()\n",
    "degree_centrality = centrality_calc.calculate_degree_centrality(kg)\n",
    "\n",
    "print(\"Top 5 Most Central Entities (Degree):\")\n",
    "for ranking in degree_centrality.get(\"rankings\", [])[:5]:\n",
    "    print(f\"- {ranking['node']}: {ranking['score']:.4f}\")\n",
    "\n",
    "# 2. Community Detection\n",
    "try:\n",
    "    detector = CommunityDetector()\n",
    "    result = detector.detect_communities(kg, algorithm=\"louvain\") or {}\n",
    "\n",
    "    communities_raw = result.get(\"communities\")\n",
    "    if communities_raw is None:\n",
    "        communities_raw = result.get(\"node_assignments\")\n",
    "\n",
    "    communities = []\n",
    "    if isinstance(communities_raw, list):\n",
    "        for c in communities_raw:\n",
    "            if isinstance(c, (list, tuple, set)):\n",
    "                communities.append(list(c))\n",
    "            elif isinstance(c, dict):\n",
    "                communities.append(list(c.keys()))\n",
    "            else:\n",
    "                communities.append([str(c)])\n",
    "    elif isinstance(communities_raw, dict):\n",
    "        comm_map = {}\n",
    "        for node_id, comm_id in communities_raw.items():\n",
    "            comm_map.setdefault(comm_id, []).append(node_id)\n",
    "        # Sort by community ID safely\n",
    "        sorted_keys = sorted(comm_map.keys(), key=lambda x: str(x))\n",
    "        communities = [comm_map[k] for k in sorted_keys]\n",
    "\n",
    "    print(f\"\\nDetected {len(communities)} Communities:\")\n",
    "    for i, comm in enumerate(communities[:3]):\n",
    "        # Ensure elements are strings\n",
    "        sample = [str(x) for x in list(comm)[:5]]\n",
    "        print(f\"Community {i+1}: {', '.join(sample)}...\")\n",
    "except Exception as e:\n",
    "    print(f\"\\nCommunity detection skipped: {e}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Phase 7: Semantic Querying\n",
    "\n",
    "We can query the graph to find specific relationships, such as tracing the lineage of Python versions or finding libraries related to Python."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from semantica.kg import ConnectivityAnalyzer\n",
    "\n",
    "print(\"\\n--- Semantic Querying & Path Finding ---\")\n",
    "\n",
    "kg = globals().get(\"kg\")\n",
    "kg = kg if isinstance(kg, dict) else {}\n",
    "kg.setdefault(\"entities\", [])\n",
    "kg.setdefault(\"relationships\", [])\n",
    "\n",
    "analyzer = ConnectivityAnalyzer()\n",
    "\n",
    "# 1. Check Connectivity\n",
    "connectivity = analyzer.analyze_connectivity(kg)\n",
    "print(f\"Graph Connected: {connectivity.get('is_connected')}\")\n",
    "print(f\"Connected Components: {connectivity.get('num_components')}\")\n",
    "\n",
    "# 2. Find Path between Entities\n",
    "source = \"pandas\"\n",
    "target = \"Python\"\n",
    "\n",
    "print(f\"\\nFinding path from '{source}' to '{target}':\")\n",
    "try:\n",
    "    path_result = analyzer.calculate_shortest_paths(kg, source=source, target=target)\n",
    "    \n",
    "    if path_result.get(\"exists\"):\n",
    "        path = path_result[\"path\"]\n",
    "        print(f\"Path Found: {' -> '.join(path)}\")\n",
    "        print(f\"Distance: {path_result['distance']}\")\n",
    "    else:\n",
    "        print(\"No path found.\")\n",
    "except Exception as e:\n",
    "    print(f\"Path finding error: {e}\")\n",
    "    # Fallback to simple neighbor check\n",
    "    print(\"Falling back to direct neighbor check...\")\n",
    "    found = False\n",
    "    for rel in kg.get('relationships', []):\n",
    "        if rel.get('source') == source and rel.get('target') == target:\n",
    "            print(f\" - [{rel.get('type', 'related_to')}] -> {target}\")\n",
    "            found = True\n",
    "    if not found:\n",
    "        print(\"No direct edge found.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Phase 8: Export & Persistence\n",
    "\n",
    "Finally, we save the constructed Knowledge Graph to a JSON file for external use or visualization in other tools."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from semantica.export import GraphExporter\n",
    "\n",
    "print(\"\\n--- Exporting Knowledge Graph ---\")\n",
    "\n",
    "# Use Semantica's GraphExporter for robust export\n",
    "exporter = GraphExporter(format=\"json\", include_attributes=True)\n",
    "export_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem_kg.json\")\n",
    "\n",
    "try:\n",
    "    exporter.export_knowledge_graph(kg, export_path)\n",
    "    print(f\"Graph saved to: {export_path}\")\n",
    "    \n",
    "    # Optional: Export to GraphML for Gephi\n",
    "    graphml_path = os.path.join(WORKSPACE_DIR, \"python_ecosystem.graphml\")\n",
    "    exporter_ml = GraphExporter(format=\"graphml\")\n",
    "    exporter_ml.export_knowledge_graph(kg, graphml_path)\n",
    "    print(f\"GraphML saved to: {graphml_path} (Ready for Gephi/Cytoscape)\")\n",
    "    \n",
    "except Exception as e:\n",
    "    print(f\"Export failed: {e}\")\n",
    "    # Fallback\n",
    "    import json\n",
    "    with open(export_path, \"w\") as f:\n",
    "        json.dump(kg, f, default=str)\n",
    "    print(\"Fallback export used.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Phase 9: Context Engineering for LLM Agents\n",
    "\n",
    "This is the **critical step** where we turn our Knowledge Graph into a queryable **Context** for AI Agents.\n",
    "We use the `AgentContext` module to ingest our graph and enable **Retrieval Augmented Generation (RAG)** capabilities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n--- Context Engineering ---\")\n",
    "\n",
    "import json\n",
    "from semantica.context import AgentContext, ContextGraph\n",
    "from semantica.vector_store import VectorStore\n",
    "\n",
    "kg = globals().get(\"kg\")\n",
    "kg = kg if isinstance(kg, dict) else {}\n",
    "kg.setdefault(\"entities\", [])\n",
    "kg.setdefault(\"relationships\", [])\n",
    "\n",
    "# 1. Initialize Vector Store (with FastEmbed support)\n",
    "# We try to use the high-performance 'fastembed' model if available\n",
    "vs = VectorStore(backend=\"inmemory\", dimension=384)\n",
    "try:\n",
    "    if hasattr(vs, \"embedder\") and vs.embedder:\n",
    "        print(\"Initializing FastEmbed model (BAAI/bge-small-en-v1.5)...\")\n",
    "        vs.embedder.set_text_model(method=\"fastembed\", model_name=\"BAAI/bge-small-en-v1.5\")\n",
    "except Exception as e:\n",
    "    print(f\"FastEmbed not available ({e}). Using fallback keyword/random embedding.\")\n",
    "    print(\"Tip: Run '!pip install fastembed' and restart kernel for better results.\")\n",
    "\n",
    "# 2. Initialize Context Graph\n",
    "cg = ContextGraph()\n",
    "\n",
    "# 3. Create the Agent Context\n",
    "# This binds the Vector Store (Content) and Knowledge Graph (Structure) together\n",
    "context = AgentContext(vector_store=vs, knowledge_graph=cg)\n",
    "\n",
    "# 4. Ingest Graph Structure\n",
    "# We map our generic KG data to the specific structure ContextGraph expects\n",
    "print(\"Building Context Graph structure...\")\n",
    "\n",
    "kg_entities = kg.get(\"entities\", []) if isinstance(kg, dict) else []\n",
    "kg_relationships = kg.get(\"relationships\", []) if isinstance(kg, dict) else []\n",
    "\n",
    "context_entities = []\n",
    "for node in kg_entities:\n",
    "    if not isinstance(node, dict):\n",
    "        continue\n",
    "    name = node.get(\"name\") or node.get(\"id\")\n",
    "    if not name:\n",
    "        continue\n",
    "    context_entities.append(\n",
    "        {\n",
    "            \"id\": node.get(\"id\") or name,\n",
    "            \"text\": name,\n",
    "            \"type\": node.get(\"type\") or \"Entity\",\n",
    "            \"metadata\": node.get(\"properties\") or {},\n",
    "        }\n",
    "    )\n",
    "\n",
    "context_relationships = []\n",
    "for rel in kg_relationships:\n",
    "    if not isinstance(rel, dict):\n",
    "        continue\n",
    "    src = rel.get(\"source\")\n",
    "    tgt = rel.get(\"target\")\n",
    "    rtype = rel.get(\"type\")\n",
    "    if not src or not tgt or not rtype:\n",
    "        continue\n",
    "    context_relationships.append({\"source_id\": src, \"target_id\": tgt, \"type\": rtype})\n",
    "\n",
    "cg.build_from_entities_and_relationships(context_entities, context_relationships)\n",
    "print(f\"Context Graph: {cg.stats()['node_count']} nodes, {cg.stats()['edge_count']} edges\")\n",
    "\n",
    "# 5. Index Entities for Vector Retrieval (Batch Store)\n",
    "# We transform entities into \"documents\" so the Vector Store can index them.\n",
    "# This allows the Agent to \"find\" the graph nodes using semantic search.\n",
    "print(\"Indexing entities into Vector Store...\")\n",
    "\n",
    "entity_documents = []\n",
    "for node in kg_entities:\n",
    "    # Create a rich textual description for the embedding\n",
    "    if not isinstance(node, dict):\n",
    "        continue\n",
    "    name = node.get(\"name\") or node.get(\"id\")\n",
    "    if not name:\n",
    "        continue\n",
    "    description = f\"{name} is a {node.get('type', 'Entity')}.\"\n",
    "    props = node.get('properties', {})\n",
    "    if props:\n",
    "        # Flatten properties into string for better semantic context\n",
    "        prop_str = \", \".join([f\"{k}: {v}\" for k,v in props.items() if isinstance(v, (str, int, float))])\n",
    "        description += f\" Properties: {prop_str}\"\n",
    "    \n",
    "    # Create document object\n",
    "    entity_documents.append({\n",
    "        \"content\": description,\n",
    "        \"metadata\": {\n",
    "            \"source\": \"knowledge_graph\",\n",
    "            \"original_id\": node.get(\"id\") or name,\n",
    "            \"type\": node.get('type', 'Entity')\n",
    "        }\n",
    "    })\n",
    "\n",
    "# Batch store all entity descriptions\n",
    "# extract_entities=False because we are storing the entities themselves\n",
    "context.store(entity_documents, extract_entities=False)\n",
    "print(f\"Successfully indexed {len(entity_documents)} entities.\")\n",
    "\n",
    "# 6. Simulate an Agent Query (GraphRAG)\n",
    "query = \"pandas library\"\n",
    "print(f\"\\nAgent Query: '{query}'\")\n",
    "\n",
    "# Retrieve context using Hybrid Search (Vector + Graph)\n",
    "results = context.retrieve(\n",
    "    query,\n",
    "    use_graph=True,\n",
    "    expand_graph=True,  # Follow edges to get related context (e.g. pandas -> Python)\n",
    "    max_results=3\n",
    ")\n",
    "\n",
    "print(\"\\n--- Retrieved Context for LLM ---\")\n",
    "if results:\n",
    "    for res in results:\n",
    "        # Access dictionary keys instead of attributes\n",
    "        print(f\"Content: {res['content']}\")\n",
    "        print(f\"Score: {res['score']:.4f}\")\n",
    "        \n",
    "        # Check for related entities in the dictionary\n",
    "        if 'related_entities' in res and res['related_entities']:\n",
    "            # related_entities is a list of dicts, we want the 'text' or 'id'\n",
    "            related = [e.get('text', e.get('id', 'Unknown')) for e in res['related_entities']]\n",
    "            print(f\"Graph Expansion: {', '.join(related)}\")\n",
    "        print(\"-\" * 30)\n",
    "else:\n",
    "    print(\"No context retrieved.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n--- Visualizing Graph ---\")\n",
    "visualizer = KGVisualizer(layout=\"force\", color_scheme=\"vibrant\")\n",
    "fig = visualizer.visualize_network(kg, output=\"interactive\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
