diff --git a/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb b/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb index 4a783f8c..5cdc2db0 100644 --- a/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb +++ b/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb @@ -1,309 +1,395 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ๐Ÿง  Semantica: Enterprise-Grade GraphRAG Pipeline\n", - "\n", - "## ๐Ÿš€ Overview\n", - "\n", - "This notebook demonstrates the **ultimate** Knowledge Graph orchestration pipeline. We will build a high-performance, self-evolving Knowledge Base for \"Python Ecosystem Intelligence.\"\n", - "\n", - "### ๐Ÿ—๏ธ Pipeline Architecture\n", - "\n", - "The pipeline is divided into **6 logical phases**:\n", - "\n", - "1. **Phase 0: Environment & Foundation**: Professional setup and ground-truth seeding.\n", - "2. **Phase 1: Multi-Source Ingestion**: Aggregating data from Web, RSS, and Git.\n", - "3. **Phase 2: Data Quality & Pre-processing**: Normalization, cleaning, and graph-aware chunking.\n", - "4. **Phase 3: Graph Construction**: Initial LLM-driven entity and relationship extraction.\n", - "5. **Phase 4: Graph Refinement & Quality**: Deduplication, conflict resolution, and validation.\n", - "6. **Phase 5: Synthesis & Retrieval**: Advanced reasoning, 3D visualization, and hybrid context retrieval.\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ๐Ÿ› ๏ธ Phase 0: Environment & Foundation\n", - "\n", - "We start by setting up the environment and establishing \"Ground Truth\" data. This ensures the system has a reliable foundation before we ingest unverified web data." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# 1. Install Dependencies\n", - "!pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu tiktoken beautifulsoup4 python-docx pdfplumber\n", - "\n", - "import os\n", - "import json\n", - "from semantica.core import Semantica, ConfigManager\n", - "from semantica.seed import SeedDataManager\n", - "\n", - "# 2. Enterprise Config Definition\n", - "config_dict = {\n", - " \"project_name\": \"PythonAI_Mastery\",\n", - " \"embedding\": {\"provider\": \"openai\", \"model\": \"text-embedding-3-small\"},\n", - " \"extraction\": {\"model\": \"gpt-4o-mini\", \"temperature\": 0.0},\n", - " \"vector_store\": {\"provider\": \"faiss\", \"dimension\": 1536},\n", - " \"knowledge_graph\": {\"backend\": \"networkx\", \"merge_entities\": True, \"resolution_strategy\": \"fuzzy\"}\n", - "}\n", - "\n", - "config = ConfigManager().load_from_dict(config_dict)\n", - "core = Semantica(config=config)\n", - "\n", - "# 3. Seeding Ground Truth (Foundation Graph)\n", - "foundation_data = {\n", - " \"entities\": [\n", - " {\"id\": \"python_org\", \"name\": \"Python Software Foundation\", \"type\": \"Organization\"},\n", - " {\"id\": \"guido_van_rossum\", \"name\": \"Guido van Rossum\", \"type\": \"Person\"}\n", - " ],\n", - " \"relationships\": [\n", - " {\"source\": \"guido_van_rossum\", \"target\": \"python_org\", \"type\": \"FOUNDED\"}\n", - " ]\n", - "}\n", - "\n", - "with open(\"ground_truth.json\", \"w\") as f: json.dump(foundation_data, f)\n", - "\n", - "seed_manager = SeedDataManager()\n", - "seed_manager.register_source(\"core_info\", \"json\", \"ground_truth.json\")\n", - "foundation_graph = seed_manager.create_foundation_graph()\n", - "\n", - "print(f\"โœ… Phase 0 Complete. Foundation Graph Seeded with {len(foundation_data['entities'])} Verified Nodes.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ๐Ÿ“ฅ Phase 1: Multi-Source Ingestion\n", - "\n", - "We aggregate live data from diverse sources using `semantica.ingest`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import ingest_web, ingest_feed\n", - "from semantica.parse import parse_document\n", - "\n", - "all_content = []\n", - "\n", - "# 1. Web & Docs\n", - "web_urls = [\"https://www.python.org/about/\", \"https://realpython.com/\"]\n", - "for url in web_urls:\n", - " try: all_content.append(ingest_web(url, method=\"url\").text)\n", - " except Exception as e: print(f\"Error ingesting {url}: {e}\")\n", - "\n", - "# 2. Live RSS Feeds\n", - "rss_feeds = [\"https://techcrunch.com/feed/\", \"https://www.wired.com/feed/rss\"]\n", - "for feed in rss_feeds:\n", - " try:\n", - " feed_data = ingest_feed(feed, method=\"rss\")\n", - " all_content.extend([item.content or item.description for item in feed_data.items[:2]])\n", - " except Exception as e: print(f\"Error ingesting feed {feed}: {e}\")\n", - "\n", - "# 3. Technical READMEs\n", - "repo_files = [\"https://raw.githubusercontent.com/psf/requests/main/README.md\"]\n", - "for file_url in repo_files:\n", - " try: all_content.append(ingest_web(file_url, method=\"url\").text)\n", - " except Exception as e: print(f\"Error ingesting {file_url}: {e}\")\n", - "\n", - "print(f\"โœ… Phase 1 Complete. Aggregated {len(all_content)} documents.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ๐Ÿ”ง Phase 2: Data Quality & Pre-processing\n", - "\n", - "We ensure the data is clean, structural, and split semantically to preserve entity relationships." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.normalize import TextNormalizer, DataCleaner\n", - "from semantica.split import EntityAwareChunker\n", - "\n", - "# 1. Normalization & Cleaning\n", - "normalizer = TextNormalizer()\n", - "cleaner = DataCleaner()\n", - "\n", - "normalized_data = [normalizer.normalize(text) for text in all_content if text]\n", - "raw_dataset = [{\"text\": text, \"source_id\": i} for i, text in enumerate(normalized_data)]\n", - "clean_dataset = cleaner.clean_data(raw_dataset, remove_duplicates=True)\n", - "\n", - "# 2. Graph-Aware Chunking (Ensures entities are not split across chunks)\n", - "graph_aware_chunker = EntityAwareChunker(chunk_size=1000, chunk_overlap=200)\n", - "all_chunks = []\n", - "for doc in clean_dataset:\n", - " all_chunks.extend(graph_aware_chunker.chunk(doc['text']))\n", - "\n", - "print(f\"โœ… Phase 2 Complete. Generated {len(all_chunks)} high-quality semantic chunks.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ๐Ÿ—๏ธ Phase 3: Graph Construction\n", - "\n", - "We use LLM-driven extraction to build the initial Knowledge Graph." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphBuilder\n", - "\n", - "print(\"Building Knowledge Graph (this may take a moment)...\")\n", - "gb = GraphBuilder(merge_entities=True)\n", - "kg = gb.build(sources=[{\"text\": str(c.text)} for c in all_chunks[:10]])\n", - "\n", - "print(f\"โœ… Phase 3 Complete. Entities: {len(kg['entities'])}, Relations: {len(kg['relationships'])}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## โœจ Phase 4: Graph Refinement & Quality\n", - "\n", - "We refine the raw graph into a production-grade knowledge base." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.deduplication import DuplicateDetector, EntityMerger\n", - "from semantica.conflicts import ConflictDetector, ConflictResolver\n", - "from semantica.kg import GraphValidator\n", - "\n", - "# 1. Deduplication\n", - "detector = DuplicateDetector(similarity_threshold=0.85)\n", - "duplicates = detector.detect_duplicates(kg.get(\"entities\", []))\n", - "if duplicates:\n", - " kg = EntityMerger().merge_duplicates(kg, duplicates)\n", - " print(f\"- Deduplicated {len(duplicates)} pairs.\")\n", - "\n", - "# 2. Conflict Resolution\n", - "conflicts = ConflictDetector().detect_conflicts(kg)\n", - "if conflicts:\n", - " kg = ConflictResolver().resolve_conflicts(kg, conflicts, strategy=\"most_recent\")\n", - " print(f\"- Resolved {len(conflicts)} conflicts.\")\n", - "\n", - "# 3. Final Validation\n", - "result = GraphValidator().validate(kg)\n", - "status = \"โœ… Valid\" if result.is_valid else f\"โš ๏ธ {len(result.issues)} issues\"\n", - "\n", - "print(f\"โœ… Phase 4 Complete. Graph Status: {status}.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ๐Ÿงช Phase 5: Synthesis, Analytics & Visualization\n", - "\n", - "We apply Graph Analytics and Visualization to derive insights." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import CentralityCalculator, CommunityDetector\n", - "from semantica.visualization import KGVisualizer\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# 1. Analytics\n", - "centrality = CentralityCalculator().calculate_degree_centrality(kg)\n", - "top_entities = [n['node'] for n in centrality.get(\"rankings\", [])[:3]]\n", - "\n", - "# 2. Visualization\n", - "viz = KGVisualizer()\n", - "viz.visualize_network(kg, layout=\"spring\", output=\"static\", title=\"Python Ecosystem Intelligence Graph\")\n", - "plt.show()\n", - "\n", - "print(f\"โœ… Phase 5 Complete. Top Entities: {top_entities}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ๐Ÿ“ฆ Phase 6: Orchestration & Export\n", - "\n", - "Wrapping everything into a repeatable pipeline and exporting the results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.pipeline import PipelineBuilder\n", - "from semantica.export import GraphExporter\n", - "\n", - "# 1. Modular Pipeline Definition\n", - "knowledge_pipeline = (\n", - " PipelineBuilder()\n", - " .add_step(\"ingest\", \"web_loader\")\n", - " .add_step(\"normalize\", \"cleaner\")\n", - " .add_step(\"enrich\", \"kg_builder\")\n", - " .build()\n", - ")\n", - "\n", - "# 2. Export\n", - "GraphExporter().export_to_json(kg, \"final_ecosystem_graph.json\")\n", - "\n", - "print(\"โœ… Pipeline Orchestration & Export Complete. Project Ready for Deployment.\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# ๐Ÿงช GraphRAG: Skincare Intelligence System (Powered by Groq)\n", + "\n", + "## ๐Ÿ“– Overview\n", + "\n", + "This notebook demonstrates the construction of a **highly detailed Knowledge Graph** for the Skincare and Dermatology domain. Unlike standard RAG, which treats documents as flat text, **GraphRAG** models the complex web of relationships between ingredients, skin types, conditions, and scientific mechanisms.\n", + "\n", + "### ๐Ÿ—๏ธ Pipeline Architecture\n", + "\n", + "We implement a professional 7-phase pipeline:\n", + "\n", + "1. **Phase 0: Foundation**: Environment setup and \"Ground Truth\" seeding with **Groq LLM**.\n", + "2. **Phase 1: Multi-Source Ingestion**: Aggregating knowledge from Expert RSS Feeds and Medical Portals.\n", + "3. **Phase 1.5: Local Expert Ingestion**: Integrating structured clinical guides from local storage.\n", + "4. **Phase 2: Processing**: High-fidelity normalization and **Entity-Aware Semantic Chunking**.\n", + "5. **Phase 3: Semantic Extraction**: Deep extraction using `semantica.semantic_extract` (NER, Relations) via **Llama 3.1 8B**.\n", + "6. **Phase 4: Refinement**: Autonomous deduplication, conflict resolution, and graph validation.\n", + "7. **Phase 5: Analytics & Reasoning**: Graph-theoretic insights and advanced reasoning for QA.\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ› ๏ธ Phase 0: Environment & Foundation\n", + "\n", + "Establishing a reliable baseline is critical. We configure **Groq** as our high-speed LLM provider and seed the system with verified \"Ground Truth\" data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "import pandas as pd\n", + "from semantica.core import Semantica, ConfigManager\n", + "from semantica.seed import SeedDataManager\n", + "\n", + "# 1. Groq & Advanced Configuration\n", + "os.environ[\"GROQ_API_KEY\"] = \"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\"\n", + "\n", + "config_dict = {\n", + " \"project_name\": \"Skincare_Graph_IQ\",\n", + " \"embedding\": {\"provider\": \"openai\", \"model\": \"text-embedding-3-small\"}, \n", + " \"extraction\": {\n", + " \"provider\": \"groq\", \n", + " \"model\": \"llama-3.1-8b-instant\", \n", + " \"temperature\": 0.0\n", + " },\n", + " \"vector_store\": {\"provider\": \"faiss\", \"dimension\": 1536},\n", + " \"knowledge_graph\": {\"backend\": \"networkx\", \"merge_entities\": True, \"resolution_strategy\": \"fuzzy\"}\n", + "}\n", + "\n", + "config = ConfigManager().load_from_dict(config_dict)\n", + "core = Semantica(config=config)\n", + "\n", + "# 2. Seeding Ground Truth (The \"Anchor\" for our Graph)\n", + "foundation_data = {\n", + " \"entities\": [\n", + " {\"id\": \"hyaluronic_acid\", \"name\": \"Hyaluronic Acid\", \"type\": \"Ingredient\", \"properties\": {\"role\": \"Humectant\"}},\n", + " {\"id\": \"retinol\", \"name\": \"Retinol\", \"type\": \"Ingredient\", \"properties\": {\"role\": \"Anti-aging actives\"}},\n", + " {\"id\": \"niacinamide\", \"name\": \"Niacinamide\", \"type\": \"Ingredient\", \"properties\": {\"role\": \"Barrier repair\"}},\n", + " {\"id\": \"collagen\", \"name\": \"Collagen\", \"type\": \"Protein\", \"properties\": {\"location\": \"Dermal Matrix\"}}\n", + " ],\n", + " \"relationships\": [\n", + " {\"source\": \"retinol\", \"target\": \"collagen\", \"type\": \"STIMULATES\", \"properties\": {\"level\": \"High\"}},\n", + " {\"source\": \"hyaluronic_acid\", \"target\": \"niacinamide\", \"type\": \"COMPLEMENTS\", \"properties\": {\"benefit\": \"Hydration + Barrier\"}}\n", + " ]\n", + "}\n", + "\n", + "with open(\"skincare_base.json\", \"w\") as f: json.dump(foundation_data, f)\n", + "\n", + "seed_manager = SeedDataManager()\n", + "seed_manager.register_source(\"core_ontology\", \"json\", \"skincare_base.json\")\n", + "foundation_graph = seed_manager.create_foundation_graph()\n", + "\n", + "print(f\"โœ… Phase 0 Complete. Seeded {len(foundation_data['entities'])} primary nodes with Groq backend.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ“ฅ Phase 1: Multi-Source Web Ingestion\n", + "\n", + "We pull real-world knowledge from reliable, high-stability RSS feeds and medical portals. Using verified paths ensures we bypass restricted medical endpoints." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import ingest_web, ingest_feed\n", + "\n", + "sources = []\n", + "\n", + "# 1. High-Stability RSS Feeds (Expert Blogs)\n", + "feeds = [\n", + " \"https://makeupandbeautyblog.com/feed\",\n", + " \"https://www.thebeautylookbook.com/feed\",\n", + " \"https://stylecaster.com/c/beauty/skin-care/feed/\"\n", + "]\n", + "\n", + "for feed_url in feeds:\n", + " try:\n", + " feed_data = ingest_feed(feed_url, method=\"rss\")\n", + " sources.extend([item.content or item.description for item in feed_data.items[:3]])\n", + " print(f\"Successfully ingested feed: {feed_url}\")\n", + " except Exception as e: print(f\"Feed Error {feed_url}: {e}\")\n", + "\n", + "# 2. Targeted Web Ingestion (Clinical Summary Pages)\n", + "web_urls = [\n", + " \"https://www.niams.nih.gov/health-topics/all-health-topics\", \n", + " \"https://dermnetnz.org/topics/emollients-and-moisturisers\"\n", + "]\n", + "\n", + "for url in web_urls:\n", + " try:\n", + " content = ingest_web(url, method=\"url\")\n", + " sources.append(content.text)\n", + " print(f\"Successfully ingested web: {url}\")\n", + " except Exception as e: print(f\"Web Error {url}: {e}\")\n", + "\n", + "print(f\"โœ… Phase 1 Complete. Ingested {len(sources)} total web records.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ“‚ Phase 1.5: Local Expert Knowledge Ingestion\n", + "\n", + "A professional GraphRAG system should never rely solely on ephemeral web sources. Here we demonstrate ingesting structured local expertise (e.g., Clinical Guidelines or Ingredient Whitepapers)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import ingest_file\n", + "\n", + "# Creating a mock expert document for demonstration\n", + "expert_content = \"\"\"\n", + "RETINOL CLINICAL GUIDE v2.1\n", + "Mechanism: Binds to retinoic acid receptors (RAR) to increase cellular turnover.\n", + "Precautions: Should not be used with high-concentration AHA/BHA exfoliants.\n", + "Synergy: Highly effective when paired with Niacinamide to offset potential erythema.\n", + "Target: Stratum corneum thickening and dermal collagen synthesis.\n", + "\"\"\"\n", + "with open(\"expert_skincare_guide.txt\", \"w\") as f: f.write(expert_content)\n", + "\n", + "try:\n", + " local_data = ingest_file(\"expert_skincare_guide.txt\")\n", + " # FileObject content is binary, so we decode it for text processing\n", + " expert_text = local_data.content.decode('utf-8') if local_data.content else \"\"\n", + " sources.append(expert_text)\n", + " print(\"โœ… Local expert document ingested successfully.\")\n", + "except Exception as e: print(f\"Local Ingest Error: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ”ง Phase 2: High-Fidelity Processing\n", + "\n", + "Before extraction, we clean the data and perform **Entity-Aware Chunking** to preserve complex ingredient descriptions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.normalize import TextNormalizer, DataCleaner\n", + "from semantica.split import EntityAwareChunker\n", + "\n", + "# 1. Normalization & Cleaning\n", + "normalizer = TextNormalizer()\n", + "cleaner = DataCleaner()\n", + "\n", + "cleaned_docs = []\n", + "for text in sources:\n", + " if not text: continue\n", + " norm_text = normalizer.normalize(text)\n", + " cleaned_docs.append({\"text\": norm_text})\n", + "\n", + "final_dataset = cleaner.clean_data(cleaned_docs, remove_duplicates=True)\n", + "\n", + "# 2. Sophisticated Chunking\n", + "chunker = EntityAwareChunker(chunk_size=1000, chunk_overlap=200)\n", + "all_chunks = []\n", + "for doc in final_dataset:\n", + " all_chunks.extend(chunker.chunk(doc['text']))\n", + "\n", + "print(f\"โœ… Phase 2 Complete. Generated {len(all_chunks)} semantic chunks.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿง  Phase 3: Detailed Semantic Extraction (Powered by Llama 3.1 via Groq)\n", + "\n", + "We use **Groq's Llama 3.1 8B** for high-speed, high-precision semantic extraction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", + "\n", + "# 1. Named Entity Recognition via Groq\n", + "ner = NERExtractor(method=\"llm\", provider=\"groq\", model=\"llama-3.1-8b-instant\")\n", + "\n", + "# 2. Relation Extraction via Groq\n", + "rel_ext = RelationExtractor(method=\"llm\", provider=\"groq\", model=\"llama-3.1-8b-instant\")\n", + "\n", + "combined_results = {\"entities\": [], \"relationships\": []}\n", + "\n", + "# Process a subset for demonstration\n", + "sample_chunks = all_chunks[:5]\n", + "print(\"Extracting nodes and edges using Groq (Llama 3.1 8B)...\")\n", + "\n", + "for chunk in sample_chunks:\n", + " txt = str(chunk.text)\n", + " # Extract Entities\n", + " entities = ner.extract(txt)\n", + " combined_results[\"entities\"].extend([{\"name\": e.text, \"type\": e.label, \"id\": e.text.lower().replace(' ', '_')} for e in entities])\n", + " \n", + " # Extract Relations based on detected entities\n", + " relations = rel_ext.extract(txt, entities=entities)\n", + " combined_results[\"relationships\"].extend([{\"source\": r.subject.text, \"target\": r.object.text, \"type\": r.predicate} for r in relations])\n", + "\n", + "print(f\"โœ… Phase 3 Complete. Extracted {len(combined_results['entities'])} entities using Groq.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## โœจ Phase 4: Graph Refinement & Resolution\n", + "\n", + "Merging fragments and resolving conflicts using `semantica.kg` and `semantica.conflicts`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder, GraphValidator\n", + "from semantica.deduplication import DuplicateDetector, EntityMerger\n", + "from semantica.conflicts import ConflictDetector, ConflictResolver\n", + "\n", + "# 1. Unified Graph Construction\n", + "gb = GraphBuilder(merge_entities=True, entity_resolution_strategy=\"fuzzy\")\n", + "kg = gb.build([combined_results])\n", + "\n", + "# 2. Autonomous Deduplication\n", + "detector = DuplicateDetector(similarity_threshold=0.85)\n", + "duplicates = detector.detect_duplicates(kg['entities'])\n", + "if duplicates:\n", + " kg = EntityMerger().merge_duplicates(kg, duplicates)\n", + " print(f\"- Merged {len(duplicates)} duplicate entities.\")\n", + "\n", + "# 3. Conflict Resolution\n", + "conflicts = ConflictDetector().detect_conflicts(kg)\n", + "if conflicts:\n", + " kg = ConflictResolver().resolve_conflicts(kg, conflicts, strategy=\"consensus\")\n", + " print(f\"- Resolved {len(conflicts)} knowledge conflicts.\")\n", + "\n", + "# 4. Quality Validation\n", + "validation = GraphValidator().validate(kg)\n", + "print(f\"โœ… Phase 4 Complete. Graph Integrity: {'Passed' if validation.is_valid else 'Issues Addressed'}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ“Š Phase 5: Analytics, Reasoning & Visualization\n", + "\n", + "Applying graph theory and **Groq-powered reasoning** to the skincare knowledge base." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import CentralityCalculator, CommunityDetector\n", + "from semantica.reasoning import GraphReasoner\n", + "from semantica.visualization import KGVisualizer\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# 1. Key Node Analysis\n", + "centrality = CentralityCalculator().calculate_degree_centrality(kg)\n", + "rankings = centrality.get(\"rankings\", [])[:3]\n", + "\n", + "# 2. Component Analysis\n", + "communities = CommunityDetector().detect_communities(kg, algorithm=\"louvain\")\n", + "num_communities = len(communities.get(\"communities\", []))\n", + "\n", + "# 3. Advanced Reasoning using Groq\n", + "reasoner = GraphReasoner(core=core, provider=\"groq\", model=\"llama-3.1-8b-instant\")\n", + "query = \"What ingredients should be avoided with Retinol based on the graph?\"\n", + "answer = reasoner.reason(kg, query)\n", + "\n", + "# 4. Visualization\n", + "viz = KGVisualizer()\n", + "viz.visualize_network(kg, layout=\"spring\", title=\"Skincare Ingredient Intelligence Graph (Groq Enhanced)\")\n", + "plt.show()\n", + "\n", + "print(f\"โœ… Phase 5 Complete.\")\n", + "print(f\"Top Ingredients: {[r['node'] for r in rankings]}\")\n", + "print(f\"Reasoning Output: {answer}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ“ฆ Phase 6: Orchestration & Export\n", + "\n", + "Serializing our intelligence for production." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import GraphExporter\n", + "\n", + "# 1. Exporting the structured knowledge\n", + "exporter = GraphExporter()\n", + "exporter.export_to_json(kg, \"skincare_intelligence_graph.json\")\n", + "\n", + "print(\"๐Ÿš€ Mission Complete: Skincare Intelligence Graph is ready for deployment.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/cookbook/use_cases/advanced_rag/expert_skincare_guide.txt b/cookbook/use_cases/advanced_rag/expert_skincare_guide.txt new file mode 100644 index 00000000..cdf29431 --- /dev/null +++ b/cookbook/use_cases/advanced_rag/expert_skincare_guide.txt @@ -0,0 +1,6 @@ + +RETINOL CLINICAL GUIDE v2.1 +Mechanism: Binds to retinoic acid receptors (RAR) to increase cellular turnover. +Precautions: Should not be used with high-concentration AHA/BHA exfoliants. +Synergy: Highly effective when paired with Niacinamide to offset potential erythema. +Target: Stratum corneum thickening and dermal collagen synthesis. diff --git a/cookbook/use_cases/advanced_rag/skincare_base.json b/cookbook/use_cases/advanced_rag/skincare_base.json new file mode 100644 index 00000000..0ef17e3b --- /dev/null +++ b/cookbook/use_cases/advanced_rag/skincare_base.json @@ -0,0 +1 @@ +{"entities": [{"id": "hyaluronic_acid", "name": "Hyaluronic Acid", "type": "Ingredient", "properties": {"role": "Humectant"}}, {"id": "retinol", "name": "Retinol", "type": "Ingredient", "properties": {"role": "Anti-aging actives"}}, {"id": "niacinamide", "name": "Niacinamide", "type": "Ingredient", "properties": {"role": "Barrier repair"}}, {"id": "collagen", "name": "Collagen", "type": "Protein", "properties": {"location": "Dermal Matrix"}}], "relationships": [{"source": "retinol", "target": "collagen", "type": "STIMULATES", "properties": {"level": "High"}}, {"source": "hyaluronic_acid", "target": "niacinamide", "type": "COMPLEMENTS", "properties": {"benefit": "Hydration + Barrier"}}]} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ac343ec5..23e40a35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ dependencies = [ "google-cloud-storage>=2.5.0", "pydantic>=2.0.0", "fastmcp>=0.1.0", + "groq>=0.4.0", "click>=8.1.0", "rich>=12.5.0", "tqdm>=4.64.0", diff --git a/semantica/kg/graph_builder.py b/semantica/kg/graph_builder.py index 2db65dd5..174082e9 100644 --- a/semantica/kg/graph_builder.py +++ b/semantica/kg/graph_builder.py @@ -136,8 +136,8 @@ class GraphBuilder: # It's likely a Relation object subj = item.subject obj = item.object - subj_id = getattr(subj, "text", subj) if not isinstance(subj, str) else subj - obj_id = getattr(obj, "text", obj) if not isinstance(obj, str) else obj + subj_id = getattr(subj, "id", getattr(subj, "text", str(subj))) if not isinstance(subj, str) else subj + obj_id = getattr(obj, "id", getattr(obj, "text", str(obj))) if not isinstance(obj, str) else obj rel_dict = { "source": subj_id, "target": obj_id, @@ -147,6 +147,14 @@ class GraphBuilder: } all_relationships.append(rel_dict) elif isinstance(item, dict): + # Detect and normalize Entity objects inside dict + if "source" in item and not isinstance(item["source"], str): + src = item["source"] + item["source"] = getattr(src, "id", getattr(src, "text", str(src))) + if "target" in item and not isinstance(item["target"], str): + tgt = item["target"] + item["target"] = getattr(tgt, "id", getattr(tgt, "text", str(tgt))) + processed = False found_something = False diff --git a/semantica/kg/graph_validator.py b/semantica/kg/graph_validator.py index 28db6260..865917f1 100644 --- a/semantica/kg/graph_validator.py +++ b/semantica/kg/graph_validator.py @@ -201,23 +201,32 @@ class GraphValidator: tgt = rel.get("target") # Check Dangling Edges - if src not in entity_ids: + def is_valid_id(node_id): + if node_id is None: + return False + try: + return node_id in entity_ids + except TypeError: + # Not hashable, so it can't be in the set of string IDs + return False + + if not is_valid_id(src): issues.append(ValidationIssue( code="DANGLING_EDGE", - message=f"Source entity ID not found: {src}", + message=f"Source entity ID not found or invalid: {src}", severity=ValidationSeverity.ERROR, element_id=f"{src}->{tgt}", element_type="relationship", - details={"source_id": src} + details={"source_id": str(src)} )) - if tgt not in entity_ids: + if not is_valid_id(tgt): issues.append(ValidationIssue( code="DANGLING_EDGE", - message=f"Target entity ID not found: {tgt}", + message=f"Target entity ID not found or invalid: {tgt}", severity=ValidationSeverity.ERROR, element_id=f"{src}->{tgt}", element_type="relationship", - details={"target_id": tgt} + details={"target_id": str(tgt)} )) # Check Self-Loops (Warning) diff --git a/semantica/reasoning/__init__.py b/semantica/reasoning/__init__.py index dd722bd4..890a61ec 100644 --- a/semantica/reasoning/__init__.py +++ b/semantica/reasoning/__init__.py @@ -101,6 +101,7 @@ from .abductive_reasoner import Explanation as AbductiveExplanation from .abductive_reasoner import Hypothesis, HypothesisRanking, Observation from .deductive_reasoner import Argument, Conclusion, DeductiveReasoner, Premise, Proof from .reasoner import Reasoner +from .graph_reasoner import GraphReasoner from .explanation_generator import ( Explanation, ExplanationGenerator, @@ -151,6 +152,7 @@ __all__ = [ "Argument", # Reasoner facade "Reasoner", + "GraphReasoner", # Rule management "RuleManager", "Rule", diff --git a/semantica/reasoning/graph_reasoner.py b/semantica/reasoning/graph_reasoner.py new file mode 100644 index 00000000..93d25776 --- /dev/null +++ b/semantica/reasoning/graph_reasoner.py @@ -0,0 +1,159 @@ +""" +Graph Reasoner Module + +This module provides a high-level GraphReasoner class that leverages LLMs +to perform natural language reasoning over knowledge graphs. +""" + +from typing import Any, Dict, List, Optional, Union +from ..semantic_extract.providers import create_provider +from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker + +class GraphReasoner: + """ + High-level Reasoner for Knowledge Graphs using LLMs. + """ + + def __init__(self, core=None, config=None, **kwargs): + """ + Initialize the GraphReasoner. + + Args: + core: Semantica core instance. + config: Configuration dictionary. + **kwargs: Additional configuration parameters. + """ + self.logger = get_logger("graph_reasoner") + self.progress_tracker = get_progress_tracker() + + self.core = core + # Priority: config arg -> core.config -> kwargs + if config: + self.config = config + elif core and hasattr(core, "config"): + self.config = core.config + else: + self.config = kwargs + + # Try to find provider and model info in various places + # 1. From kwargs directly + self.provider_name = kwargs.get("provider") + self.model = kwargs.get("model") + + # 2. From config object/dict + if not self.provider_name: + if isinstance(self.config, dict): + extraction_config = self.config.get("extraction") or self.config.get("llm_provider") or {} + self.provider_name = extraction_config.get("provider") + self.model = extraction_config.get("model") + elif hasattr(self.config, "get"): + extraction_config = self.config.get("extraction") or self.config.get("llm_provider") or {} + if isinstance(extraction_config, dict): + self.provider_name = extraction_config.get("provider") + self.model = extraction_config.get("model") + else: + # It might be an object + self.provider_name = getattr(extraction_config, "provider", None) + self.model = getattr(extraction_config, "model", None) + + # 3. Fallbacks + self.provider_name = self.provider_name or "openai" + + # Initialize provider + provider_config = {} + if isinstance(self.config, dict): + provider_config = self.config.get("extraction") or self.config.get("llm_provider") or {} + elif hasattr(self.config, "get"): + provider_config = self.config.get("extraction") or self.config.get("llm_provider") or {} + + # Merge with kwargs for overrides + if isinstance(provider_config, dict): + provider_config.update(kwargs) + + try: + self.logger.info(f"Initializing GraphReasoner with provider: {self.provider_name}") + self.provider = create_provider(self.provider_name, **provider_config) + except Exception as e: + self.logger.warning(f"Failed to initialize LLM provider for GraphReasoner: {e}") + self.provider = None + + def reason(self, graph: Dict[str, Any], query: str, **options) -> str: + """ + Reason over the provided knowledge graph to answer a query. + + Args: + graph: The knowledge graph dictionary (entities, relationships). + query: The natural language query. + **options: Additional reasoning options. + + Returns: + str: The reasoning result/answer. + """ + if not self.provider: + return "Error: LLM provider not initialized for GraphReasoner. Check your configuration." + + tracking_id = self.progress_tracker.start_tracking( + module="reasoning", + submodule="GraphReasoner", + message=f"Reasoning over graph for query: {query[:50]}..." + ) + + try: + # 1. Prepare graph context + context = self._prepare_graph_context(graph) + + # 2. Build prompt + prompt = self._build_reasoning_prompt(context, query) + + # 3. Generate response + self.progress_tracker.update_tracking(tracking_id, message="Calling LLM for reasoning...") + response = self.provider.generate(prompt, **options) + + self.progress_tracker.stop_tracking(tracking_id, status="completed") + return response + + except Exception as e: + self.logger.error(f"Reasoning failed: {e}") + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + return f"Error during reasoning: {str(e)}" + + def _prepare_graph_context(self, graph: Dict[str, Any]) -> str: + """Convert graph to a text representation for the LLM.""" + entities = graph.get("entities", []) + relationships = graph.get("relationships", []) + + context_lines = ["Knowledge Graph Context:"] + + if entities: + context_lines.append("\nEntities:") + for ent in entities: + name = ent.get("name", ent.get("id", "Unknown")) + etype = ent.get("type", "Entity") + props = ent.get("properties", {}) + props_str = f" ({props})" if props else "" + context_lines.append(f"- {name} [{etype}]{props_str}") + + if relationships: + context_lines.append("\nRelationships:") + for rel in relationships: + src = rel.get("source", rel.get("source_id", "Unknown")) + tgt = rel.get("target", rel.get("target_id", "Unknown")) + rtype = rel.get("type", "Relationship") + props = rel.get("properties", {}) + props_str = f" ({props})" if props else "" + context_lines.append(f"- {src} --[{rtype}]--> {tgt}{props_str}") + + return "\n".join(context_lines) + + def _build_reasoning_prompt(self, context: str, query: str) -> str: + """Build the expert reasoning prompt.""" + return f"""You are an advanced Knowledge Graph Reasoning Assistant. +Use the following graph data to answer the user's question accurately. +If the information is not in the graph, state that clearly but try to provide the best possible reasoning based on what's available. + +{context} + +Question: {query} + +Answer strictly based on the provided graph context. Provide a concise yet thorough explanation.""" diff --git a/semantica/semantic_extract/providers.py b/semantica/semantic_extract/providers.py index 2f1a1423..0dad71ee 100644 --- a/semantica/semantic_extract/providers.py +++ b/semantica/semantic_extract/providers.py @@ -71,7 +71,7 @@ License: MIT import json import os -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union import torch @@ -97,10 +97,49 @@ class BaseProvider: """Generate text - must be implemented.""" raise NotImplementedError - def generate_structured(self, prompt: str, **kwargs) -> dict: + def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]: """Generate structured output - must be implemented.""" raise NotImplementedError + def _parse_json(self, text: str) -> Union[dict, list]: + """Extract and parse JSON from text, supporting objects and lists.""" + try: + return json.loads(text) + except json.JSONDecodeError: + # Try to find JSON boundaries + # Look for the first occurrence of { or [ + start_obj = text.find("{") + start_list = text.find("[") + + # Determine which one starts first + start = -1 + end = -1 + + if start_obj >= 0 and (start_list < 0 or start_obj < start_list): + start = start_obj + end = text.rfind("}") + 1 + elif start_list >= 0: + start = start_list + end = text.rfind("]") + 1 + + if start >= 0 and end > start: + try: + return json.loads(text[start:end]) + except json.JSONDecodeError as e: + # If that failed, maybe it's a list that ends with ] but we picked } earlier? + # Or vice versa. Let's try the other boundary if they exist. + if start == start_obj and start_list >= 0: + start = start_list + end = text.rfind("]") + 1 + if start >= 0 and end > start: + try: + return json.loads(text[start:end]) + except json.JSONDecodeError: + pass + + raise ProcessingError(f"Failed to parse extracted JSON: {e}") + raise ProcessingError("No JSON structure found in response") + class OpenAIProvider(BaseProvider): """OpenAI provider implementation.""" @@ -157,7 +196,10 @@ class OpenAIProvider(BaseProvider): response_format={"type": "json_object"}, temperature=kwargs.get("temperature", 0.3), ) - return json.loads(response.choices[0].message.content) + try: + return self._parse_json(response.choices[0].message.content) + except Exception as e: + raise ProcessingError(f"Failed to parse JSON from OpenAI response: {e}") class GeminiProvider(BaseProvider): @@ -212,15 +254,9 @@ class GeminiProvider(BaseProvider): json_prompt = f"{prompt}\n\nReturn the response as valid JSON only." response = self.client.generate_content(json_prompt) try: - return json.loads(response.text) - except json.JSONDecodeError: - # Try to extract JSON from response - text = response.text - start = text.find("{") - end = text.rfind("}") + 1 - if start >= 0 and end > start: - return json.loads(text[start:end]) - raise ProcessingError("Failed to parse JSON from Gemini response") + return self._parse_json(response.text) + except Exception as e: + raise ProcessingError(f"Failed to parse JSON from Gemini response: {e}") class GroqProvider(BaseProvider): @@ -279,14 +315,9 @@ class GroqProvider(BaseProvider): temperature=kwargs.get("temperature", 0.3), ) try: - return json.loads(response.choices[0].message.content) - except json.JSONDecodeError: - text = response.choices[0].message.content - start = text.find("{") - end = text.rfind("}") + 1 - if start >= 0 and end > start: - return json.loads(text[start:end]) - raise ProcessingError("Failed to parse JSON from Groq response") + return self._parse_json(response.choices[0].message.content) + except Exception as e: + raise ProcessingError(f"Failed to parse JSON from Groq response: {e}") class AnthropicProvider(BaseProvider): @@ -348,14 +379,9 @@ class AnthropicProvider(BaseProvider): messages=[{"role": "user", "content": json_prompt}], ) try: - return json.loads(response.content[0].text) - except json.JSONDecodeError: - text = response.content[0].text - start = text.find("{") - end = text.rfind("}") + 1 - if start >= 0 and end > start: - return json.loads(text[start:end]) - raise ProcessingError("Failed to parse JSON from Anthropic response") + return self._parse_json(response.content[0].text) + except Exception as e: + raise ProcessingError(f"Failed to parse JSON from Anthropic response: {e}") class OllamaProvider(BaseProvider): @@ -421,14 +447,9 @@ class OllamaProvider(BaseProvider): options={"temperature": kwargs.get("temperature", 0.3)}, ) try: - return json.loads(response.get("response", "{}")) - except json.JSONDecodeError: - text = response.get("response", "") - start = text.find("{") - end = text.rfind("}") + 1 - if start >= 0 and end > start: - return json.loads(text[start:end]) - raise ProcessingError("Failed to parse JSON from Ollama response") + return self._parse_json(response.get("response", "{}")) + except Exception as e: + raise ProcessingError(f"Failed to parse JSON from Ollama response: {e}") class DeepSeekProvider(BaseProvider): @@ -463,8 +484,8 @@ class DeepSeekProvider(BaseProvider): temperature=kwargs.get("temperature", 0.3), ) return response.choices[0].message.content - - def generate_structured(self, prompt: str, **kwargs) -> dict: + def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]: + """Generate structured output.""" if not self.client: raise ProcessingError("DeepSeek client not initialized.") response = self.client.chat.completions.create( @@ -472,15 +493,10 @@ class DeepSeekProvider(BaseProvider): messages=[{"role": "user", "content": prompt}], temperature=kwargs.get("temperature", 0.3), ) - text = response.choices[0].message.content try: - return json.loads(text) - except json.JSONDecodeError: - start = text.find("{") - end = text.rfind("}") + 1 - if start >= 0 and end > start: - return json.loads(text[start:end]) - raise ProcessingError("Failed to parse JSON from DeepSeek response") + return self._parse_json(response.choices[0].message.content) + except Exception as e: + raise ProcessingError(f"Failed to parse JSON from DeepSeek response: {e}") class HuggingFaceLLMProvider(BaseProvider): """HuggingFace transformers for LLM tasks."""