From d8f78cd49efab6bfa4827fbc7330d62d3c3b07a4 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Tue, 30 Dec 2025 18:19:57 +0530 Subject: [PATCH] Refactor Criminal Network Analysis notebook: simplify code, use Semantica modules effectively, add interactive visualization, fix GraphRAG queries --- .../01_Criminal_Network_Analysis.ipynb | 8078 ++++++++++++++++- .../intelligence/criminal_network.graphml | 2163 +++++ .../intelligence/criminal_network.json | 4956 ++++++++++ .../criminal_network_entities.csv | 290 + .../criminal_network_relationships.csv | 177 + 5 files changed, 15359 insertions(+), 305 deletions(-) create mode 100644 cookbook/use_cases/intelligence/criminal_network.graphml create mode 100644 cookbook/use_cases/intelligence/criminal_network.json create mode 100644 cookbook/use_cases/intelligence/criminal_network_entities.csv create mode 100644 cookbook/use_cases/intelligence/criminal_network_relationships.csv diff --git a/cookbook/use_cases/intelligence/01_Criminal_Network_Analysis.ipynb b/cookbook/use_cases/intelligence/01_Criminal_Network_Analysis.ipynb index 51d9b9c7..a17dcb46 100644 --- a/cookbook/use_cases/intelligence/01_Criminal_Network_Analysis.ipynb +++ b/cookbook/use_cases/intelligence/01_Criminal_Network_Analysis.ipynb @@ -57,9 +57,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n" + ] + } + ], "source": [ "%pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq sentence-transformers scikit-learn\n" ] @@ -77,13 +100,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", - "os.environ[\"GROQ_API_KEY\"] = os.getenv(\"GROQ_API_KEY\", \"your-key-here\")\n", + "os.environ[\"GROQ_API_KEY\"] = os.getenv(\"GROQ_API_KEY\", \"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\")\n", "\n", "# Configuration constants\n", "EMBEDDING_DIMENSION = 384\n", @@ -105,9 +128,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://www.justice.gov/opa/pressreleases ❌πŸ“₯ (1.6s) | 🧠 Semantica is ingesting: 403 Client Error: Forbidden for url: https://www.dea.gov/press-releases ❌πŸ“₯ (0.4s) [1/12] Feed: 10 documents\n", + "🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://www.justice.gov/opa/pressreleases ❌πŸ“₯ (1.6s) | 🧠 Semantica is ingesting: 403 Client Error: Forbidden for url: https://www.dea.gov/press-releases ❌πŸ“₯ (0.4s) [2/12] Feed: 10 documents\n", + "🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://www.justice.gov/opa/pressreleases ❌πŸ“₯ (1.6s) | 🧠 Semantica is ingesting: 403 Client Error: Forbidden for url: https://www.dea.gov/press-releases ❌πŸ“₯ (0.4s) [3/12] Feed failed: Failed to parse feed: not well-formed (invalid tok\n", + "🧠 Semantica is ingesting: 403 Client Error: Forbidden for url: https://www.dea.gov/press-releases ❌πŸ“₯ (0.4s) | 🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://feeds.feedburner.com/oreilly/radar ❌πŸ“₯ (0.7s) [4/12] Feed failed: Failed to fetch feed: 404 Client Error: Not Found \n", + "🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://feeds.feedburner.com/oreilly/radar ❌πŸ“₯ (0.7s) | 🧠 Semantica is ingesting: Ingested 10 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ [5/12] Feed: 10 documents\n", + "🧠 Semantica is ingesting: Ingested 10 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: Ingested 10 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ [6/12] Feed: 10 documents\n", + "🧠 Semantica is ingesting: Ingested 10 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: Ingested 50 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ [7/12] Feed: 50 documents\n", + "🧠 Semantica is ingesting: Ingested 50 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: Ingested 10 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ [8/12] Feed: 10 documents\n", + "🧠 Semantica is ingesting: Ingested 10 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: 403 Client Error: Forbidden for url: https://www.bleepingcomputer.com/feed/ ❌πŸ“₯ (0.5s) [9/12] Feed failed: Failed to fetch feed: 403 Client Error: Forbidden \n", + "🧠 Semantica is ingesting: 403 Client Error: Forbidden for url: https://www.bleepingcomputer.com/feed/ ❌πŸ“₯ (0.5s) | 🧠 Semantica is ingesting: 403 Client Error: Forbidden for url: https://www.securityweek.com/rss ❌πŸ“₯ (0.4s) [10/12] Feed failed: Failed to fetch feed: 403 Client Error: Forbidden \n", + "🧠 Semantica is ingesting: 403 Client Error: Forbidden for url: https://www.securityweek.com/rss ❌πŸ“₯ (0.4s) | 🧠 Semantica is ingesting: Ingested 12 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ [11/12] Feed: 12 documents\n", + "🧠 Semantica is ingesting: Ingested 12 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://www.csoonline.com/index.rss ❌πŸ“₯ (1.3s) [12/12] Feed failed: Failed to fetch feed: 404 Client Error: Not Found \n", + "🧠 Semantica is ingesting: Ingested 12 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://www.csoonline.com/index.rss ❌πŸ“₯ (1.3s) [1/7] Web: 3133 characters\n", + "🧠 Semantica is ingesting: Ingested 12 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://www.csoonline.com/index.rss ❌πŸ“₯ (1.3s) [2/7] Web: 3861 characters\n", + "🧠 Semantica is ingesting: Ingested 12 items |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://www.csoonline.com/index.rss ❌πŸ“₯ (1.3s) [3/7] Web: 8501 characters\n", + "🧠 Semantica is ingesting: 404 Client Error: Not Found for url: https://www.csoonline.com/index.rss ❌πŸ“₯ (1.3s) | 🧠 Semantica is ingesting: Ingested https://www.us-cert.gov/ncas/alerts (200) |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ [4/7] Web: 8501 characters\n", + "🧠 Semantica is ingesting: Ingested https://www.us-cert.gov/ncas/alerts (200) |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: Ingested https://www.europol.europa.eu/newsroom (200) |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ [5/7] Web: 96 characters\n", + "🧠 Semantica is ingesting: Ingested https://www.europol.europa.eu/newsroom (200) |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: Ingested https://www.ncsc.gov.uk/news (200) |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ [6/7] Web: 50 characters\n", + "🧠 Semantica is ingesting: Ingested https://www.ncsc.gov.uk/news (200) |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100.0% βœ…πŸ“₯ | 🧠 Semantica is ingesting: HTTPSConnectionPool(host='www.cyber.gov.au', port=443): Max retries exceeded with url: /news (Caused by ReadTimeoutError(\"HTTPSConnectionPool(host='www.cyber.gov.au', port=443): Read timed out. (read timeout=30)\")) ❌πŸ“₯ (119.3s) [7/7] Web failed: Failed to fetch URL: HTTPSConnectionPool(host='www\n", + "Ingested 118 documents\n" + ] + } + ], "source": [ "from semantica.ingest import FeedIngestor, WebIngestor, FileIngestor\n", "from contextlib import redirect_stderr\n", @@ -121,46 +171,196 @@ "# Ingest from OSINT RSS feeds\n", "osint_feeds = [\n", " \"https://www.us-cert.gov/ncas/alerts.xml\",\n", - " \"https://www.fbi.gov/feeds/news\"\n", + " \"https://www.europol.europa.eu/rss.xml\",\n", + " \"https://www.treasury.gov/resource-center/sanctions/OFAC-Enforcement/Pages/rss.xml\",\n", + " \"https://feeds.feedburner.com/oreilly/radar\",\n", + " \"https://krebsonsecurity.com/feed/\",\n", + " \"https://www.schneier.com/feed/\",\n", + " \"https://www.darkreading.com/rss.xml\",\n", + " \"https://threatpost.com/feed/\",\n", + " \"https://www.bleepingcomputer.com/feed/\",\n", + " \"https://www.securityweek.com/rss\",\n", + " \"https://www.infosecurity-magazine.com/rss/news/\",\n", + " \"https://www.csoonline.com/index.rss\"\n", "]\n", "\n", - "for feed_url in osint_feeds:\n", + "feed_ingestor = FeedIngestor()\n", + "for i, feed_url in enumerate(osint_feeds, 1):\n", " try:\n", " with redirect_stderr(StringIO()):\n", - " feed_ingestor = FeedIngestor()\n", - " feed_docs = feed_ingestor.ingest(feed_url, method=\"rss\")\n", - " documents.extend(feed_docs)\n", - " except Exception:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url, validate=False)\n", + " \n", + " feed_count = 0\n", + " for item in feed_data.items:\n", + " if not item.content:\n", + " item.content = item.description or item.title or \"\"\n", + " if item.content:\n", + " if not hasattr(item, 'metadata'):\n", + " item.metadata = {}\n", + " item.metadata['source'] = feed_url\n", + " documents.append(item)\n", + " feed_count += 1\n", + " \n", + " if feed_count > 0:\n", + " print(f\" [{i}/{len(osint_feeds)}] Feed: {feed_count} documents\")\n", + " except Exception as e:\n", + " print(f\" [{i}/{len(osint_feeds)}] Feed failed: {str(e)[:50]}\")\n", + " pass\n", + "\n", + "# Web ingestion from various intelligence and security sources\n", + "web_links = [\n", + " \"https://www.interpol.int/en/How-we-work/Notices/View-Red-Notices\",\n", + " \"https://www.unodc.org/unodc/en/data-and-analysis/index.html\",\n", + " \"https://www.cisa.gov/news-events/cybersecurity-advisories\",\n", + " \"https://www.us-cert.gov/ncas/alerts\",\n", + " \"https://www.europol.europa.eu/newsroom\",\n", + " \"https://www.ncsc.gov.uk/news\",\n", + " \"https://www.cyber.gov.au/news\"\n", + "]\n", + "\n", + "web_ingestor = WebIngestor(respect_robots=False, delay=1.0)\n", + "for i, web_url in enumerate(web_links, 1):\n", + " try:\n", + " with redirect_stderr(StringIO()):\n", + " web_content = web_ingestor.ingest_url(web_url)\n", + " if web_content and web_content.text:\n", + " # Add content attribute for compatibility with parser\n", + " web_content.content = web_content.text\n", + " if not hasattr(web_content, 'metadata'):\n", + " web_content.metadata = {}\n", + " web_content.metadata['source'] = web_url\n", + " documents.append(web_content)\n", + " print(f\" [{i}/{len(web_links)}] Web: {len(web_content.text)} characters\")\n", + " except Exception as e:\n", + " print(f\" [{i}/{len(web_links)}] Web failed: {str(e)[:50]}\")\n", " pass\n", "\n", "# Example: Web ingestion from FBI API (commented - requires authentication)\n", "# web_ingestor = WebIngestor()\n", - "# fbi_docs = web_ingestor.ingest(\"https://api.fbi.gov/wanted/v1/list\", method=\"api\")\n", - "\n", - "# Fallback: Sample criminal network data\n", - "if not documents:\n", - " network_data = \"\"\"\n", - " John Smith is associated with criminal organization XYZ.\n", - " Jane Doe has connections to John Smith and organization XYZ.\n", - " Event: Meeting on 2024-01-15 between John Smith and Jane Doe at Location A.\n", - " Organization XYZ is linked to multiple criminal activities.\n", - " Person: Mike Johnson connected to organization XYZ.\n", - " Location A is a known meeting point for criminal activities.\n", - " Relationship: John Smith and Jane Doe have a business relationship.\n", - " \"\"\"\n", - " with open(\"data/criminal_network.txt\", \"w\", encoding=\"utf-8\") as f:\n", - " f.write(network_data)\n", - " file_ingestor = FileIngestor()\n", - " documents = file_ingestor.ingest(\"data/criminal_network.txt\")\n", + "# fbi_docs = web_ingestor.ingest_url(\"https://api.fbi.gov/wanted/v1/list\")\n", "\n", "print(f\"Ingested {len(documents)} documents\")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Parsing 118 documents...\n", + "🧠 Semantica is ingesting: HTTPSConnectionPool(host='www.cyber.gov.au', port=443): Max retries exceeded with url: /news (Caused by ReadTimeoutError(\"HTTPSConnectionPool(host='www.cyber.gov.au', port=443): Read timed out. (read timeout=30)\")) ❌πŸ“₯ (119.3s) | 🧠 Semantica is parsing: Document: p>\n", + "🧠 Semantica is parsing: Document: p> \n", + " πŸ”„πŸ” (0.0s) | 🧠 Semantica is parsing: Document file not found:
\n", + "

Summary<\\strong><\\h2>\n", + "<\\div>\n", + "

Unpatched Pulse Secure VPN servers continue to be an attractive target for malicious actors. Affected organizations that have not applied the software patch to fix an arbitrary file reading vulnerability, known as CVE-2019-11510, can become compromised in an attack.[1<\\a>]<\\p>\n", + "

Although Pulse Secure [2] disclosed the vulnerability and provided software patches for the various affected products in April 2019, the Cybersecurity and Infrastructure Security Agency (CISA) continues to observe wide exploitation of CVE-2019-11510.[3<\\a>],[4<\\a>],[5<\\a>]<\\p>\n", + "

CISA expects to see continued attacks exploiting unpatched Pulse Secure VPN environments and strongly urges users and administrators to upgrade to the corresponding fixes.[2]<\\p>\n", + "

Timelines of Specific Events<\\h3>\n", + "