fix: Update ingestion URLs and logic in GraphRAG notebook and restructure script

cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb

@@ -26,13 +26,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Environment Setup\n",
-    "import os\n",
-    "import sys\n",
-    "\n",
-    "# Ensure Groq API Key is set if using Groq\n",
-    "# os.environ['GROQ_API_KEY'] = 'your-key'\n",
-    "\n",
     "%pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu"
    ]
   },
@@ -58,35 +51,41 @@
     "normalizer = TextNormalizer()\n",
     "all_content = []\n",
     "\n",
-    "# 1. Global News Feeds (RSS)\n",
+    "# 1. Global News Feeds (RSS) - Using more robust and accessible feeds\n",
     "feeds = [\n",
     "    \"http://feeds.bbci.co.uk/news/world/rss.xml\",\n",
-    "    \"https://www.reutersagency.com/feed/\" \n",
+    "    \"https://www.aljazeera.com/xml/rss/all.xml\",\n",
+    "    \"https://news.google.com/rss/search?q=site%3Areuters.com&hl=en-US&gl=US&ceid=US%3Aen\" # Reuters workaround\n",
     "]\n",
     "feed_ingestor = FeedIngestor()\n",
     "for f in feeds:\n",
     "    try:\n",
-    "        data = feed_ingestor.ingest_feed(f) # FIXED: Use ingest_feed, not ingest\n",
-    "        items = data.items[:5]\n",
-    "        all_content.extend([getattr(item, 'content', getattr(item, 'text', str(item))) for item in items])\n",
+    "        data = feed_ingestor.ingest_feed(f)\n",
+    "        items = data.items[:10]\n",
+    "        for item in items:\n",
+    "            # Fallback chain: content -> description -> title\n",
+    "            text = item.content or item.description or item.title\n",
+    "            if text:\n",
+    "                all_content.append(text)\n",
     "    except Exception as e:\n",
     "        print(f\"Warning: Failed to ingest feed {f}: {e}\")\n",
     "\n",
-    "# 2. Strategic Overviews (Web)\n",
+    "# 2. Strategic Overviews (Web) - Using pages with more permissive robots.txt\n",
     "web_urls = [\n",
     "    \"https://www.cia.gov/the-world-factbook/\",\n",
-    "    \"https://www.un.org/en/observances/security-council-day\"\n",
+    "    \"https://www.cfr.org/backgrounders\" \n",
     "]\n",
     "web_ingestor = WebIngestor()\n",
     "for url in web_urls:\n",
     "    try:\n",
-    "        content = web_ingestor.ingest_url(url) # FIXED: Use ingest_url, not ingest\n",
-    "        all_content.append(getattr(content, 'text', getattr(content, 'content', str(content))))\n",
+    "        content = web_ingestor.ingest_url(url)\n",
+    "        if content.text:\n",
+    "            all_content.append(content.text)\n",
     "    except Exception as e:\n",
     "        print(f\"Warning: Failed to ingest URL {url}: {e}\")\n",
     "\n",
     "# Clean and normalize\n",
-    "clean_docs = [normalizer.normalize(text) for text in all_content if len(text) > 50]\n",
+    "clean_docs = [normalizer.normalize(text) for text in all_content if len(text) > 100] # Increased threshold for higher quality\n",
     "print(f\"Intelligence Knowledge Hub Populated with {len(clean_docs)} reports.\")"
    ]
   },
@@ -233,7 +232,11 @@
    ]
   }
  ],
- "metadata": {},
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
  "nbformat": 4,
  "nbformat_minor": 5
 }

restructure_utf8.py

@@ -39,35 +39,41 @@ from semantica.normalize import TextNormalizer
 normalizer = TextNormalizer()
 all_content = []
 
-# 1. Global News Feeds (RSS)
+# 1. Global News Feeds (RSS) - Using more robust and accessible feeds
 feeds = [
     "http://feeds.bbci.co.uk/news/world/rss.xml",
-    "https://www.reutersagency.com/feed/" 
+    "https://www.aljazeera.com/xml/rss/all.xml",
+    "https://news.google.com/rss/search?q=site%3Areuters.com&hl=en-US&gl=US&ceid=US%3Aen" # Reuters workaround
 ]
 feed_ingestor = FeedIngestor()
 for f in feeds:
     try:
-        data = feed_ingestor.ingest_feed(f) # FIXED: Use ingest_feed, not ingest
-        items = data.items[:5]
-        all_content.extend([getattr(item, 'content', getattr(item, 'text', str(item))) for item in items])
+        data = feed_ingestor.ingest_feed(f)
+        items = data.items[:10]
+        for item in items:
+            # Fallback chain: content -> description -> title
+            text = item.content or item.description or item.title
+            if text:
+                all_content.append(text)
     except Exception as e:
         print(f"Warning: Failed to ingest feed {f}: {e}")
 
-# 2. Strategic Overviews (Web)
+# 2. Strategic Overviews (Web) - Using pages with more permissive robots.txt
 web_urls = [
     "https://www.cia.gov/the-world-factbook/",
-    "https://www.un.org/en/observances/security-council-day"
+    "https://www.cfr.org/backgrounders" 
 ]
 web_ingestor = WebIngestor()
 for url in web_urls:
     try:
-        content = web_ingestor.ingest_url(url) # FIXED: Use ingest_url, not ingest
-        all_content.append(getattr(content, 'text', getattr(content, 'content', str(content))))
+        content = web_ingestor.ingest_url(url)
+        if content.text:
+            all_content.append(content.text)
     except Exception as e:
         print(f"Warning: Failed to ingest URL {url}: {e}")
 
 # Clean and normalize
-clean_docs = [normalizer.normalize(text) for text in all_content if len(text) > 50]
+clean_docs = [normalizer.normalize(text) for text in all_content if len(text) > 100] # Increased threshold for higher quality
 print(f"Intelligence Knowledge Hub Populated with {len(clean_docs)} reports.")"""))
 
 # Phase 2: Vector RAG