From 2cf2733d5bd53feba9d377f491f44b610da4ca67 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Mon, 22 Dec 2025 20:43:51 +0530 Subject: [PATCH] fix: Update ingestion URLs and logic in GraphRAG notebook and restructure script --- .../02_RAG_vs_GraphRAG_Comparison.ipynb | 39 ++++++++++--------- restructure_utf8.py | 26 ++++++++----- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb b/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb index 84b467c6..f917bf83 100644 --- a/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb +++ b/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb @@ -26,13 +26,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Environment Setup\n", - "import os\n", - "import sys\n", - "\n", - "# Ensure Groq API Key is set if using Groq\n", - "# os.environ['GROQ_API_KEY'] = 'your-key'\n", - "\n", "%pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu" ] }, @@ -58,35 +51,41 @@ "normalizer = TextNormalizer()\n", "all_content = []\n", "\n", - "# 1. Global News Feeds (RSS)\n", + "# 1. Global News Feeds (RSS) - Using more robust and accessible feeds\n", "feeds = [\n", " \"http://feeds.bbci.co.uk/news/world/rss.xml\",\n", - " \"https://www.reutersagency.com/feed/\" \n", + " \"https://www.aljazeera.com/xml/rss/all.xml\",\n", + " \"https://news.google.com/rss/search?q=site%3Areuters.com&hl=en-US&gl=US&ceid=US%3Aen\" # Reuters workaround\n", "]\n", "feed_ingestor = FeedIngestor()\n", "for f in feeds:\n", " try:\n", - " data = feed_ingestor.ingest_feed(f) # FIXED: Use ingest_feed, not ingest\n", - " items = data.items[:5]\n", - " all_content.extend([getattr(item, 'content', getattr(item, 'text', str(item))) for item in items])\n", + " data = feed_ingestor.ingest_feed(f)\n", + " items = data.items[:10]\n", + " for item in items:\n", + " # Fallback chain: content -> description -> title\n", + " text = item.content or item.description or item.title\n", + " if text:\n", + " all_content.append(text)\n", " except Exception as e:\n", " print(f\"Warning: Failed to ingest feed {f}: {e}\")\n", "\n", - "# 2. Strategic Overviews (Web)\n", + "# 2. Strategic Overviews (Web) - Using pages with more permissive robots.txt\n", "web_urls = [\n", " \"https://www.cia.gov/the-world-factbook/\",\n", - " \"https://www.un.org/en/observances/security-council-day\"\n", + " \"https://www.cfr.org/backgrounders\" \n", "]\n", "web_ingestor = WebIngestor()\n", "for url in web_urls:\n", " try:\n", - " content = web_ingestor.ingest_url(url) # FIXED: Use ingest_url, not ingest\n", - " all_content.append(getattr(content, 'text', getattr(content, 'content', str(content))))\n", + " content = web_ingestor.ingest_url(url)\n", + " if content.text:\n", + " all_content.append(content.text)\n", " except Exception as e:\n", " print(f\"Warning: Failed to ingest URL {url}: {e}\")\n", "\n", "# Clean and normalize\n", - "clean_docs = [normalizer.normalize(text) for text in all_content if len(text) > 50]\n", + "clean_docs = [normalizer.normalize(text) for text in all_content if len(text) > 100] # Increased threshold for higher quality\n", "print(f\"Intelligence Knowledge Hub Populated with {len(clean_docs)} reports.\")" ] }, @@ -233,7 +232,11 @@ ] } ], - "metadata": {}, + "metadata": { + "language_info": { + "name": "python" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/restructure_utf8.py b/restructure_utf8.py index a6b4dc63..ed2d0147 100644 --- a/restructure_utf8.py +++ b/restructure_utf8.py @@ -39,35 +39,41 @@ from semantica.normalize import TextNormalizer normalizer = TextNormalizer() all_content = [] -# 1. Global News Feeds (RSS) +# 1. Global News Feeds (RSS) - Using more robust and accessible feeds feeds = [ "http://feeds.bbci.co.uk/news/world/rss.xml", - "https://www.reutersagency.com/feed/" + "https://www.aljazeera.com/xml/rss/all.xml", + "https://news.google.com/rss/search?q=site%3Areuters.com&hl=en-US&gl=US&ceid=US%3Aen" # Reuters workaround ] feed_ingestor = FeedIngestor() for f in feeds: try: - data = feed_ingestor.ingest_feed(f) # FIXED: Use ingest_feed, not ingest - items = data.items[:5] - all_content.extend([getattr(item, 'content', getattr(item, 'text', str(item))) for item in items]) + data = feed_ingestor.ingest_feed(f) + items = data.items[:10] + for item in items: + # Fallback chain: content -> description -> title + text = item.content or item.description or item.title + if text: + all_content.append(text) except Exception as e: print(f"Warning: Failed to ingest feed {f}: {e}") -# 2. Strategic Overviews (Web) +# 2. Strategic Overviews (Web) - Using pages with more permissive robots.txt web_urls = [ "https://www.cia.gov/the-world-factbook/", - "https://www.un.org/en/observances/security-council-day" + "https://www.cfr.org/backgrounders" ] web_ingestor = WebIngestor() for url in web_urls: try: - content = web_ingestor.ingest_url(url) # FIXED: Use ingest_url, not ingest - all_content.append(getattr(content, 'text', getattr(content, 'content', str(content)))) + content = web_ingestor.ingest_url(url) + if content.text: + all_content.append(content.text) except Exception as e: print(f"Warning: Failed to ingest URL {url}: {e}") # Clean and normalize -clean_docs = [normalizer.normalize(text) for text in all_content if len(text) > 50] +clean_docs = [normalizer.normalize(text) for text in all_content if len(text) > 100] # Increased threshold for higher quality print(f"Intelligence Knowledge Hub Populated with {len(clean_docs)} reports.")""")) # Phase 2: Vector RAG