mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Implement GraphReasoner, fix KG validation and normalization, and update RAG cookbook
This commit is contained in:
KaifAhmad1
@@ -1,309 +1,395 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 🧠 Semantica: Enterprise-Grade GraphRAG Pipeline\n",
|
||||
"\n",
|
||||
"## 🚀 Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates the **ultimate** Knowledge Graph orchestration pipeline. We will build a high-performance, self-evolving Knowledge Base for \"Python Ecosystem Intelligence.\"\n",
|
||||
"\n",
|
||||
"### 🏗️ Pipeline Architecture\n",
|
||||
"\n",
|
||||
"The pipeline is divided into **6 logical phases**:\n",
|
||||
"\n",
|
||||
"1. **Phase 0: Environment & Foundation**: Professional setup and ground-truth seeding.\n",
|
||||
"2. **Phase 1: Multi-Source Ingestion**: Aggregating data from Web, RSS, and Git.\n",
|
||||
"3. **Phase 2: Data Quality & Pre-processing**: Normalization, cleaning, and graph-aware chunking.\n",
|
||||
"4. **Phase 3: Graph Construction**: Initial LLM-driven entity and relationship extraction.\n",
|
||||
"5. **Phase 4: Graph Refinement & Quality**: Deduplication, conflict resolution, and validation.\n",
|
||||
"6. **Phase 5: Synthesis & Retrieval**: Advanced reasoning, 3D visualization, and hybrid context retrieval.\n",
|
||||
"\n",
|
||||
"---"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🛠️ Phase 0: Environment & Foundation\n",
|
||||
"\n",
|
||||
"We start by setting up the environment and establishing \"Ground Truth\" data. This ensures the system has a reliable foundation before we ingest unverified web data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 1. Install Dependencies\n",
|
||||
"!pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu tiktoken beautifulsoup4 python-docx pdfplumber\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import json\n",
|
||||
"from semantica.core import Semantica, ConfigManager\n",
|
||||
"from semantica.seed import SeedDataManager\n",
|
||||
"\n",
|
||||
"# 2. Enterprise Config Definition\n",
|
||||
"config_dict = {\n",
|
||||
" \"project_name\": \"PythonAI_Mastery\",\n",
|
||||
" \"embedding\": {\"provider\": \"openai\", \"model\": \"text-embedding-3-small\"},\n",
|
||||
" \"extraction\": {\"model\": \"gpt-4o-mini\", \"temperature\": 0.0},\n",
|
||||
" \"vector_store\": {\"provider\": \"faiss\", \"dimension\": 1536},\n",
|
||||
" \"knowledge_graph\": {\"backend\": \"networkx\", \"merge_entities\": True, \"resolution_strategy\": \"fuzzy\"}\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"config = ConfigManager().load_from_dict(config_dict)\n",
|
||||
"core = Semantica(config=config)\n",
|
||||
"\n",
|
||||
"# 3. Seeding Ground Truth (Foundation Graph)\n",
|
||||
"foundation_data = {\n",
|
||||
" \"entities\": [\n",
|
||||
" {\"id\": \"python_org\", \"name\": \"Python Software Foundation\", \"type\": \"Organization\"},\n",
|
||||
" {\"id\": \"guido_van_rossum\", \"name\": \"Guido van Rossum\", \"type\": \"Person\"}\n",
|
||||
" ],\n",
|
||||
" \"relationships\": [\n",
|
||||
" {\"source\": \"guido_van_rossum\", \"target\": \"python_org\", \"type\": \"FOUNDED\"}\n",
|
||||
" ]\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"with open(\"ground_truth.json\", \"w\") as f: json.dump(foundation_data, f)\n",
|
||||
"\n",
|
||||
"seed_manager = SeedDataManager()\n",
|
||||
"seed_manager.register_source(\"core_info\", \"json\", \"ground_truth.json\")\n",
|
||||
"foundation_graph = seed_manager.create_foundation_graph()\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 0 Complete. Foundation Graph Seeded with {len(foundation_data['entities'])} Verified Nodes.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📥 Phase 1: Multi-Source Ingestion\n",
|
||||
"\n",
|
||||
"We aggregate live data from diverse sources using `semantica.ingest`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import ingest_web, ingest_feed\n",
|
||||
"from semantica.parse import parse_document\n",
|
||||
"\n",
|
||||
"all_content = []\n",
|
||||
"\n",
|
||||
"# 1. Web & Docs\n",
|
||||
"web_urls = [\"https://www.python.org/about/\", \"https://realpython.com/\"]\n",
|
||||
"for url in web_urls:\n",
|
||||
" try: all_content.append(ingest_web(url, method=\"url\").text)\n",
|
||||
" except Exception as e: print(f\"Error ingesting {url}: {e}\")\n",
|
||||
"\n",
|
||||
"# 2. Live RSS Feeds\n",
|
||||
"rss_feeds = [\"https://techcrunch.com/feed/\", \"https://www.wired.com/feed/rss\"]\n",
|
||||
"for feed in rss_feeds:\n",
|
||||
" try:\n",
|
||||
" feed_data = ingest_feed(feed, method=\"rss\")\n",
|
||||
" all_content.extend([item.content or item.description for item in feed_data.items[:2]])\n",
|
||||
" except Exception as e: print(f\"Error ingesting feed {feed}: {e}\")\n",
|
||||
"\n",
|
||||
"# 3. Technical READMEs\n",
|
||||
"repo_files = [\"https://raw.githubusercontent.com/psf/requests/main/README.md\"]\n",
|
||||
"for file_url in repo_files:\n",
|
||||
" try: all_content.append(ingest_web(file_url, method=\"url\").text)\n",
|
||||
" except Exception as e: print(f\"Error ingesting {file_url}: {e}\")\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 1 Complete. Aggregated {len(all_content)} documents.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🔧 Phase 2: Data Quality & Pre-processing\n",
|
||||
"\n",
|
||||
"We ensure the data is clean, structural, and split semantically to preserve entity relationships."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.normalize import TextNormalizer, DataCleaner\n",
|
||||
"from semantica.split import EntityAwareChunker\n",
|
||||
"\n",
|
||||
"# 1. Normalization & Cleaning\n",
|
||||
"normalizer = TextNormalizer()\n",
|
||||
"cleaner = DataCleaner()\n",
|
||||
"\n",
|
||||
"normalized_data = [normalizer.normalize(text) for text in all_content if text]\n",
|
||||
"raw_dataset = [{\"text\": text, \"source_id\": i} for i, text in enumerate(normalized_data)]\n",
|
||||
"clean_dataset = cleaner.clean_data(raw_dataset, remove_duplicates=True)\n",
|
||||
"\n",
|
||||
"# 2. Graph-Aware Chunking (Ensures entities are not split across chunks)\n",
|
||||
"graph_aware_chunker = EntityAwareChunker(chunk_size=1000, chunk_overlap=200)\n",
|
||||
"all_chunks = []\n",
|
||||
"for doc in clean_dataset:\n",
|
||||
" all_chunks.extend(graph_aware_chunker.chunk(doc['text']))\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 2 Complete. Generated {len(all_chunks)} high-quality semantic chunks.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🏗️ Phase 3: Graph Construction\n",
|
||||
"\n",
|
||||
"We use LLM-driven extraction to build the initial Knowledge Graph."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"\n",
|
||||
"print(\"Building Knowledge Graph (this may take a moment)...\")\n",
|
||||
"gb = GraphBuilder(merge_entities=True)\n",
|
||||
"kg = gb.build(sources=[{\"text\": str(c.text)} for c in all_chunks[:10]])\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 3 Complete. Entities: {len(kg['entities'])}, Relations: {len(kg['relationships'])}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## ✨ Phase 4: Graph Refinement & Quality\n",
|
||||
"\n",
|
||||
"We refine the raw graph into a production-grade knowledge base."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.deduplication import DuplicateDetector, EntityMerger\n",
|
||||
"from semantica.conflicts import ConflictDetector, ConflictResolver\n",
|
||||
"from semantica.kg import GraphValidator\n",
|
||||
"\n",
|
||||
"# 1. Deduplication\n",
|
||||
"detector = DuplicateDetector(similarity_threshold=0.85)\n",
|
||||
"duplicates = detector.detect_duplicates(kg.get(\"entities\", []))\n",
|
||||
"if duplicates:\n",
|
||||
" kg = EntityMerger().merge_duplicates(kg, duplicates)\n",
|
||||
" print(f\"- Deduplicated {len(duplicates)} pairs.\")\n",
|
||||
"\n",
|
||||
"# 2. Conflict Resolution\n",
|
||||
"conflicts = ConflictDetector().detect_conflicts(kg)\n",
|
||||
"if conflicts:\n",
|
||||
" kg = ConflictResolver().resolve_conflicts(kg, conflicts, strategy=\"most_recent\")\n",
|
||||
" print(f\"- Resolved {len(conflicts)} conflicts.\")\n",
|
||||
"\n",
|
||||
"# 3. Final Validation\n",
|
||||
"result = GraphValidator().validate(kg)\n",
|
||||
"status = \"✅ Valid\" if result.is_valid else f\"⚠️ {len(result.issues)} issues\"\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 4 Complete. Graph Status: {status}.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🧪 Phase 5: Synthesis, Analytics & Visualization\n",
|
||||
"\n",
|
||||
"We apply Graph Analytics and Visualization to derive insights."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import CentralityCalculator, CommunityDetector\n",
|
||||
"from semantica.visualization import KGVisualizer\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"# 1. Analytics\n",
|
||||
"centrality = CentralityCalculator().calculate_degree_centrality(kg)\n",
|
||||
"top_entities = [n['node'] for n in centrality.get(\"rankings\", [])[:3]]\n",
|
||||
"\n",
|
||||
"# 2. Visualization\n",
|
||||
"viz = KGVisualizer()\n",
|
||||
"viz.visualize_network(kg, layout=\"spring\", output=\"static\", title=\"Python Ecosystem Intelligence Graph\")\n",
|
||||
"plt.show()\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 5 Complete. Top Entities: {top_entities}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📦 Phase 6: Orchestration & Export\n",
|
||||
"\n",
|
||||
"Wrapping everything into a repeatable pipeline and exporting the results."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.pipeline import PipelineBuilder\n",
|
||||
"from semantica.export import GraphExporter\n",
|
||||
"\n",
|
||||
"# 1. Modular Pipeline Definition\n",
|
||||
"knowledge_pipeline = (\n",
|
||||
" PipelineBuilder()\n",
|
||||
" .add_step(\"ingest\", \"web_loader\")\n",
|
||||
" .add_step(\"normalize\", \"cleaner\")\n",
|
||||
" .add_step(\"enrich\", \"kg_builder\")\n",
|
||||
" .build()\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# 2. Export\n",
|
||||
"GraphExporter().export_to_json(kg, \"final_ecosystem_graph.json\")\n",
|
||||
"\n",
|
||||
"print(\"✅ Pipeline Orchestration & Export Complete. Project Ready for Deployment.\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 🧪 GraphRAG: Skincare Intelligence System (Powered by Groq)\n",
|
||||
"\n",
|
||||
"## 📖 Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates the construction of a **highly detailed Knowledge Graph** for the Skincare and Dermatology domain. Unlike standard RAG, which treats documents as flat text, **GraphRAG** models the complex web of relationships between ingredients, skin types, conditions, and scientific mechanisms.\n",
|
||||
"\n",
|
||||
"### 🏗️ Pipeline Architecture\n",
|
||||
"\n",
|
||||
"We implement a professional 7-phase pipeline:\n",
|
||||
"\n",
|
||||
"1. **Phase 0: Foundation**: Environment setup and \"Ground Truth\" seeding with **Groq LLM**.\n",
|
||||
"2. **Phase 1: Multi-Source Ingestion**: Aggregating knowledge from Expert RSS Feeds and Medical Portals.\n",
|
||||
"3. **Phase 1.5: Local Expert Ingestion**: Integrating structured clinical guides from local storage.\n",
|
||||
"4. **Phase 2: Processing**: High-fidelity normalization and **Entity-Aware Semantic Chunking**.\n",
|
||||
"5. **Phase 3: Semantic Extraction**: Deep extraction using `semantica.semantic_extract` (NER, Relations) via **Llama 3.1 8B**.\n",
|
||||
"6. **Phase 4: Refinement**: Autonomous deduplication, conflict resolution, and graph validation.\n",
|
||||
"7. **Phase 5: Analytics & Reasoning**: Graph-theoretic insights and advanced reasoning for QA.\n",
|
||||
"\n",
|
||||
"---"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🛠️ Phase 0: Environment & Foundation\n",
|
||||
"\n",
|
||||
"Establishing a reliable baseline is critical. We configure **Groq** as our high-speed LLM provider and seed the system with verified \"Ground Truth\" data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import json\n",
|
||||
"import pandas as pd\n",
|
||||
"from semantica.core import Semantica, ConfigManager\n",
|
||||
"from semantica.seed import SeedDataManager\n",
|
||||
"\n",
|
||||
"# 1. Groq & Advanced Configuration\n",
|
||||
"os.environ[\"GROQ_API_KEY\"] = \"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\"\n",
|
||||
"\n",
|
||||
"config_dict = {\n",
|
||||
" \"project_name\": \"Skincare_Graph_IQ\",\n",
|
||||
" \"embedding\": {\"provider\": \"openai\", \"model\": \"text-embedding-3-small\"}, \n",
|
||||
" \"extraction\": {\n",
|
||||
" \"provider\": \"groq\", \n",
|
||||
" \"model\": \"llama-3.1-8b-instant\", \n",
|
||||
" \"temperature\": 0.0\n",
|
||||
" },\n",
|
||||
" \"vector_store\": {\"provider\": \"faiss\", \"dimension\": 1536},\n",
|
||||
" \"knowledge_graph\": {\"backend\": \"networkx\", \"merge_entities\": True, \"resolution_strategy\": \"fuzzy\"}\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"config = ConfigManager().load_from_dict(config_dict)\n",
|
||||
"core = Semantica(config=config)\n",
|
||||
"\n",
|
||||
"# 2. Seeding Ground Truth (The \"Anchor\" for our Graph)\n",
|
||||
"foundation_data = {\n",
|
||||
" \"entities\": [\n",
|
||||
" {\"id\": \"hyaluronic_acid\", \"name\": \"Hyaluronic Acid\", \"type\": \"Ingredient\", \"properties\": {\"role\": \"Humectant\"}},\n",
|
||||
" {\"id\": \"retinol\", \"name\": \"Retinol\", \"type\": \"Ingredient\", \"properties\": {\"role\": \"Anti-aging actives\"}},\n",
|
||||
" {\"id\": \"niacinamide\", \"name\": \"Niacinamide\", \"type\": \"Ingredient\", \"properties\": {\"role\": \"Barrier repair\"}},\n",
|
||||
" {\"id\": \"collagen\", \"name\": \"Collagen\", \"type\": \"Protein\", \"properties\": {\"location\": \"Dermal Matrix\"}}\n",
|
||||
" ],\n",
|
||||
" \"relationships\": [\n",
|
||||
" {\"source\": \"retinol\", \"target\": \"collagen\", \"type\": \"STIMULATES\", \"properties\": {\"level\": \"High\"}},\n",
|
||||
" {\"source\": \"hyaluronic_acid\", \"target\": \"niacinamide\", \"type\": \"COMPLEMENTS\", \"properties\": {\"benefit\": \"Hydration + Barrier\"}}\n",
|
||||
" ]\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"with open(\"skincare_base.json\", \"w\") as f: json.dump(foundation_data, f)\n",
|
||||
"\n",
|
||||
"seed_manager = SeedDataManager()\n",
|
||||
"seed_manager.register_source(\"core_ontology\", \"json\", \"skincare_base.json\")\n",
|
||||
"foundation_graph = seed_manager.create_foundation_graph()\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 0 Complete. Seeded {len(foundation_data['entities'])} primary nodes with Groq backend.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📥 Phase 1: Multi-Source Web Ingestion\n",
|
||||
"\n",
|
||||
"We pull real-world knowledge from reliable, high-stability RSS feeds and medical portals. Using verified paths ensures we bypass restricted medical endpoints."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import ingest_web, ingest_feed\n",
|
||||
"\n",
|
||||
"sources = []\n",
|
||||
"\n",
|
||||
"# 1. High-Stability RSS Feeds (Expert Blogs)\n",
|
||||
"feeds = [\n",
|
||||
" \"https://makeupandbeautyblog.com/feed\",\n",
|
||||
" \"https://www.thebeautylookbook.com/feed\",\n",
|
||||
" \"https://stylecaster.com/c/beauty/skin-care/feed/\"\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for feed_url in feeds:\n",
|
||||
" try:\n",
|
||||
" feed_data = ingest_feed(feed_url, method=\"rss\")\n",
|
||||
" sources.extend([item.content or item.description for item in feed_data.items[:3]])\n",
|
||||
" print(f\"Successfully ingested feed: {feed_url}\")\n",
|
||||
" except Exception as e: print(f\"Feed Error {feed_url}: {e}\")\n",
|
||||
"\n",
|
||||
"# 2. Targeted Web Ingestion (Clinical Summary Pages)\n",
|
||||
"web_urls = [\n",
|
||||
" \"https://www.niams.nih.gov/health-topics/all-health-topics\", \n",
|
||||
" \"https://dermnetnz.org/topics/emollients-and-moisturisers\"\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for url in web_urls:\n",
|
||||
" try:\n",
|
||||
" content = ingest_web(url, method=\"url\")\n",
|
||||
" sources.append(content.text)\n",
|
||||
" print(f\"Successfully ingested web: {url}\")\n",
|
||||
" except Exception as e: print(f\"Web Error {url}: {e}\")\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 1 Complete. Ingested {len(sources)} total web records.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📂 Phase 1.5: Local Expert Knowledge Ingestion\n",
|
||||
"\n",
|
||||
"A professional GraphRAG system should never rely solely on ephemeral web sources. Here we demonstrate ingesting structured local expertise (e.g., Clinical Guidelines or Ingredient Whitepapers)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import ingest_file\n",
|
||||
"\n",
|
||||
"# Creating a mock expert document for demonstration\n",
|
||||
"expert_content = \"\"\"\n",
|
||||
"RETINOL CLINICAL GUIDE v2.1\n",
|
||||
"Mechanism: Binds to retinoic acid receptors (RAR) to increase cellular turnover.\n",
|
||||
"Precautions: Should not be used with high-concentration AHA/BHA exfoliants.\n",
|
||||
"Synergy: Highly effective when paired with Niacinamide to offset potential erythema.\n",
|
||||
"Target: Stratum corneum thickening and dermal collagen synthesis.\n",
|
||||
"\"\"\"\n",
|
||||
"with open(\"expert_skincare_guide.txt\", \"w\") as f: f.write(expert_content)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" local_data = ingest_file(\"expert_skincare_guide.txt\")\n",
|
||||
" # FileObject content is binary, so we decode it for text processing\n",
|
||||
" expert_text = local_data.content.decode('utf-8') if local_data.content else \"\"\n",
|
||||
" sources.append(expert_text)\n",
|
||||
" print(\"✅ Local expert document ingested successfully.\")\n",
|
||||
"except Exception as e: print(f\"Local Ingest Error: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🔧 Phase 2: High-Fidelity Processing\n",
|
||||
"\n",
|
||||
"Before extraction, we clean the data and perform **Entity-Aware Chunking** to preserve complex ingredient descriptions."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.normalize import TextNormalizer, DataCleaner\n",
|
||||
"from semantica.split import EntityAwareChunker\n",
|
||||
"\n",
|
||||
"# 1. Normalization & Cleaning\n",
|
||||
"normalizer = TextNormalizer()\n",
|
||||
"cleaner = DataCleaner()\n",
|
||||
"\n",
|
||||
"cleaned_docs = []\n",
|
||||
"for text in sources:\n",
|
||||
" if not text: continue\n",
|
||||
" norm_text = normalizer.normalize(text)\n",
|
||||
" cleaned_docs.append({\"text\": norm_text})\n",
|
||||
"\n",
|
||||
"final_dataset = cleaner.clean_data(cleaned_docs, remove_duplicates=True)\n",
|
||||
"\n",
|
||||
"# 2. Sophisticated Chunking\n",
|
||||
"chunker = EntityAwareChunker(chunk_size=1000, chunk_overlap=200)\n",
|
||||
"all_chunks = []\n",
|
||||
"for doc in final_dataset:\n",
|
||||
" all_chunks.extend(chunker.chunk(doc['text']))\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 2 Complete. Generated {len(all_chunks)} semantic chunks.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🧠 Phase 3: Detailed Semantic Extraction (Powered by Llama 3.1 via Groq)\n",
|
||||
"\n",
|
||||
"We use **Groq's Llama 3.1 8B** for high-speed, high-precision semantic extraction."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"\n",
|
||||
"# 1. Named Entity Recognition via Groq\n",
|
||||
"ner = NERExtractor(method=\"llm\", provider=\"groq\", model=\"llama-3.1-8b-instant\")\n",
|
||||
"\n",
|
||||
"# 2. Relation Extraction via Groq\n",
|
||||
"rel_ext = RelationExtractor(method=\"llm\", provider=\"groq\", model=\"llama-3.1-8b-instant\")\n",
|
||||
"\n",
|
||||
"combined_results = {\"entities\": [], \"relationships\": []}\n",
|
||||
"\n",
|
||||
"# Process a subset for demonstration\n",
|
||||
"sample_chunks = all_chunks[:5]\n",
|
||||
"print(\"Extracting nodes and edges using Groq (Llama 3.1 8B)...\")\n",
|
||||
"\n",
|
||||
"for chunk in sample_chunks:\n",
|
||||
" txt = str(chunk.text)\n",
|
||||
" # Extract Entities\n",
|
||||
" entities = ner.extract(txt)\n",
|
||||
" combined_results[\"entities\"].extend([{\"name\": e.text, \"type\": e.label, \"id\": e.text.lower().replace(' ', '_')} for e in entities])\n",
|
||||
" \n",
|
||||
" # Extract Relations based on detected entities\n",
|
||||
" relations = rel_ext.extract(txt, entities=entities)\n",
|
||||
" combined_results[\"relationships\"].extend([{\"source\": r.subject.text, \"target\": r.object.text, \"type\": r.predicate} for r in relations])\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 3 Complete. Extracted {len(combined_results['entities'])} entities using Groq.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## ✨ Phase 4: Graph Refinement & Resolution\n",
|
||||
"\n",
|
||||
"Merging fragments and resolving conflicts using `semantica.kg` and `semantica.conflicts`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import GraphBuilder, GraphValidator\n",
|
||||
"from semantica.deduplication import DuplicateDetector, EntityMerger\n",
|
||||
"from semantica.conflicts import ConflictDetector, ConflictResolver\n",
|
||||
"\n",
|
||||
"# 1. Unified Graph Construction\n",
|
||||
"gb = GraphBuilder(merge_entities=True, entity_resolution_strategy=\"fuzzy\")\n",
|
||||
"kg = gb.build([combined_results])\n",
|
||||
"\n",
|
||||
"# 2. Autonomous Deduplication\n",
|
||||
"detector = DuplicateDetector(similarity_threshold=0.85)\n",
|
||||
"duplicates = detector.detect_duplicates(kg['entities'])\n",
|
||||
"if duplicates:\n",
|
||||
" kg = EntityMerger().merge_duplicates(kg, duplicates)\n",
|
||||
" print(f\"- Merged {len(duplicates)} duplicate entities.\")\n",
|
||||
"\n",
|
||||
"# 3. Conflict Resolution\n",
|
||||
"conflicts = ConflictDetector().detect_conflicts(kg)\n",
|
||||
"if conflicts:\n",
|
||||
" kg = ConflictResolver().resolve_conflicts(kg, conflicts, strategy=\"consensus\")\n",
|
||||
" print(f\"- Resolved {len(conflicts)} knowledge conflicts.\")\n",
|
||||
"\n",
|
||||
"# 4. Quality Validation\n",
|
||||
"validation = GraphValidator().validate(kg)\n",
|
||||
"print(f\"✅ Phase 4 Complete. Graph Integrity: {'Passed' if validation.is_valid else 'Issues Addressed'}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📊 Phase 5: Analytics, Reasoning & Visualization\n",
|
||||
"\n",
|
||||
"Applying graph theory and **Groq-powered reasoning** to the skincare knowledge base."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import CentralityCalculator, CommunityDetector\n",
|
||||
"from semantica.reasoning import GraphReasoner\n",
|
||||
"from semantica.visualization import KGVisualizer\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"# 1. Key Node Analysis\n",
|
||||
"centrality = CentralityCalculator().calculate_degree_centrality(kg)\n",
|
||||
"rankings = centrality.get(\"rankings\", [])[:3]\n",
|
||||
"\n",
|
||||
"# 2. Component Analysis\n",
|
||||
"communities = CommunityDetector().detect_communities(kg, algorithm=\"louvain\")\n",
|
||||
"num_communities = len(communities.get(\"communities\", []))\n",
|
||||
"\n",
|
||||
"# 3. Advanced Reasoning using Groq\n",
|
||||
"reasoner = GraphReasoner(core=core, provider=\"groq\", model=\"llama-3.1-8b-instant\")\n",
|
||||
"query = \"What ingredients should be avoided with Retinol based on the graph?\"\n",
|
||||
"answer = reasoner.reason(kg, query)\n",
|
||||
"\n",
|
||||
"# 4. Visualization\n",
|
||||
"viz = KGVisualizer()\n",
|
||||
"viz.visualize_network(kg, layout=\"spring\", title=\"Skincare Ingredient Intelligence Graph (Groq Enhanced)\")\n",
|
||||
"plt.show()\n",
|
||||
"\n",
|
||||
"print(f\"✅ Phase 5 Complete.\")\n",
|
||||
"print(f\"Top Ingredients: {[r['node'] for r in rankings]}\")\n",
|
||||
"print(f\"Reasoning Output: {answer}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📦 Phase 6: Orchestration & Export\n",
|
||||
"\n",
|
||||
"Serializing our intelligence for production."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.export import GraphExporter\n",
|
||||
"\n",
|
||||
"# 1. Exporting the structured knowledge\n",
|
||||
"exporter = GraphExporter()\n",
|
||||
"exporter.export_to_json(kg, \"skincare_intelligence_graph.json\")\n",
|
||||
"\n",
|
||||
"print(\"🚀 Mission Complete: Skincare Intelligence Graph is ready for deployment.\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
RETINOL CLINICAL GUIDE v2.1
|
||||
Mechanism: Binds to retinoic acid receptors (RAR) to increase cellular turnover.
|
||||
Precautions: Should not be used with high-concentration AHA/BHA exfoliants.
|
||||
Synergy: Highly effective when paired with Niacinamide to offset potential erythema.
|
||||
Target: Stratum corneum thickening and dermal collagen synthesis.
|
||||
@@ -0,0 +1 @@
|
||||
{"entities": [{"id": "hyaluronic_acid", "name": "Hyaluronic Acid", "type": "Ingredient", "properties": {"role": "Humectant"}}, {"id": "retinol", "name": "Retinol", "type": "Ingredient", "properties": {"role": "Anti-aging actives"}}, {"id": "niacinamide", "name": "Niacinamide", "type": "Ingredient", "properties": {"role": "Barrier repair"}}, {"id": "collagen", "name": "Collagen", "type": "Protein", "properties": {"location": "Dermal Matrix"}}], "relationships": [{"source": "retinol", "target": "collagen", "type": "STIMULATES", "properties": {"level": "High"}}, {"source": "hyaluronic_acid", "target": "niacinamide", "type": "COMPLEMENTS", "properties": {"benefit": "Hydration + Barrier"}}]}
|
||||
@@ -84,6 +84,7 @@ dependencies = [
|
||||
"google-cloud-storage>=2.5.0",
|
||||
"pydantic>=2.0.0",
|
||||
"fastmcp>=0.1.0",
|
||||
"groq>=0.4.0",
|
||||
"click>=8.1.0",
|
||||
"rich>=12.5.0",
|
||||
"tqdm>=4.64.0",
|
||||
|
||||
@@ -136,8 +136,8 @@ class GraphBuilder:
|
||||
# It's likely a Relation object
|
||||
subj = item.subject
|
||||
obj = item.object
|
||||
subj_id = getattr(subj, "text", subj) if not isinstance(subj, str) else subj
|
||||
obj_id = getattr(obj, "text", obj) if not isinstance(obj, str) else obj
|
||||
subj_id = getattr(subj, "id", getattr(subj, "text", str(subj))) if not isinstance(subj, str) else subj
|
||||
obj_id = getattr(obj, "id", getattr(obj, "text", str(obj))) if not isinstance(obj, str) else obj
|
||||
rel_dict = {
|
||||
"source": subj_id,
|
||||
"target": obj_id,
|
||||
@@ -147,6 +147,14 @@ class GraphBuilder:
|
||||
}
|
||||
all_relationships.append(rel_dict)
|
||||
elif isinstance(item, dict):
|
||||
# Detect and normalize Entity objects inside dict
|
||||
if "source" in item and not isinstance(item["source"], str):
|
||||
src = item["source"]
|
||||
item["source"] = getattr(src, "id", getattr(src, "text", str(src)))
|
||||
if "target" in item and not isinstance(item["target"], str):
|
||||
tgt = item["target"]
|
||||
item["target"] = getattr(tgt, "id", getattr(tgt, "text", str(tgt)))
|
||||
|
||||
processed = False
|
||||
found_something = False
|
||||
|
||||
|
||||
@@ -201,23 +201,32 @@ class GraphValidator:
|
||||
tgt = rel.get("target")
|
||||
|
||||
# Check Dangling Edges
|
||||
if src not in entity_ids:
|
||||
def is_valid_id(node_id):
|
||||
if node_id is None:
|
||||
return False
|
||||
try:
|
||||
return node_id in entity_ids
|
||||
except TypeError:
|
||||
# Not hashable, so it can't be in the set of string IDs
|
||||
return False
|
||||
|
||||
if not is_valid_id(src):
|
||||
issues.append(ValidationIssue(
|
||||
code="DANGLING_EDGE",
|
||||
message=f"Source entity ID not found: {src}",
|
||||
message=f"Source entity ID not found or invalid: {src}",
|
||||
severity=ValidationSeverity.ERROR,
|
||||
element_id=f"{src}->{tgt}",
|
||||
element_type="relationship",
|
||||
details={"source_id": src}
|
||||
details={"source_id": str(src)}
|
||||
))
|
||||
if tgt not in entity_ids:
|
||||
if not is_valid_id(tgt):
|
||||
issues.append(ValidationIssue(
|
||||
code="DANGLING_EDGE",
|
||||
message=f"Target entity ID not found: {tgt}",
|
||||
message=f"Target entity ID not found or invalid: {tgt}",
|
||||
severity=ValidationSeverity.ERROR,
|
||||
element_id=f"{src}->{tgt}",
|
||||
element_type="relationship",
|
||||
details={"target_id": tgt}
|
||||
details={"target_id": str(tgt)}
|
||||
))
|
||||
|
||||
# Check Self-Loops (Warning)
|
||||
|
||||
@@ -101,6 +101,7 @@ from .abductive_reasoner import Explanation as AbductiveExplanation
|
||||
from .abductive_reasoner import Hypothesis, HypothesisRanking, Observation
|
||||
from .deductive_reasoner import Argument, Conclusion, DeductiveReasoner, Premise, Proof
|
||||
from .reasoner import Reasoner
|
||||
from .graph_reasoner import GraphReasoner
|
||||
from .explanation_generator import (
|
||||
Explanation,
|
||||
ExplanationGenerator,
|
||||
@@ -151,6 +152,7 @@ __all__ = [
|
||||
"Argument",
|
||||
# Reasoner facade
|
||||
"Reasoner",
|
||||
"GraphReasoner",
|
||||
# Rule management
|
||||
"RuleManager",
|
||||
"Rule",
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""
|
||||
Graph Reasoner Module
|
||||
|
||||
This module provides a high-level GraphReasoner class that leverages LLMs
|
||||
to perform natural language reasoning over knowledge graphs.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from ..semantic_extract.providers import create_provider
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
class GraphReasoner:
|
||||
"""
|
||||
High-level Reasoner for Knowledge Graphs using LLMs.
|
||||
"""
|
||||
|
||||
def __init__(self, core=None, config=None, **kwargs):
|
||||
"""
|
||||
Initialize the GraphReasoner.
|
||||
|
||||
Args:
|
||||
core: Semantica core instance.
|
||||
config: Configuration dictionary.
|
||||
**kwargs: Additional configuration parameters.
|
||||
"""
|
||||
self.logger = get_logger("graph_reasoner")
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.core = core
|
||||
# Priority: config arg -> core.config -> kwargs
|
||||
if config:
|
||||
self.config = config
|
||||
elif core and hasattr(core, "config"):
|
||||
self.config = core.config
|
||||
else:
|
||||
self.config = kwargs
|
||||
|
||||
# Try to find provider and model info in various places
|
||||
# 1. From kwargs directly
|
||||
self.provider_name = kwargs.get("provider")
|
||||
self.model = kwargs.get("model")
|
||||
|
||||
# 2. From config object/dict
|
||||
if not self.provider_name:
|
||||
if isinstance(self.config, dict):
|
||||
extraction_config = self.config.get("extraction") or self.config.get("llm_provider") or {}
|
||||
self.provider_name = extraction_config.get("provider")
|
||||
self.model = extraction_config.get("model")
|
||||
elif hasattr(self.config, "get"):
|
||||
extraction_config = self.config.get("extraction") or self.config.get("llm_provider") or {}
|
||||
if isinstance(extraction_config, dict):
|
||||
self.provider_name = extraction_config.get("provider")
|
||||
self.model = extraction_config.get("model")
|
||||
else:
|
||||
# It might be an object
|
||||
self.provider_name = getattr(extraction_config, "provider", None)
|
||||
self.model = getattr(extraction_config, "model", None)
|
||||
|
||||
# 3. Fallbacks
|
||||
self.provider_name = self.provider_name or "openai"
|
||||
|
||||
# Initialize provider
|
||||
provider_config = {}
|
||||
if isinstance(self.config, dict):
|
||||
provider_config = self.config.get("extraction") or self.config.get("llm_provider") or {}
|
||||
elif hasattr(self.config, "get"):
|
||||
provider_config = self.config.get("extraction") or self.config.get("llm_provider") or {}
|
||||
|
||||
# Merge with kwargs for overrides
|
||||
if isinstance(provider_config, dict):
|
||||
provider_config.update(kwargs)
|
||||
|
||||
try:
|
||||
self.logger.info(f"Initializing GraphReasoner with provider: {self.provider_name}")
|
||||
self.provider = create_provider(self.provider_name, **provider_config)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to initialize LLM provider for GraphReasoner: {e}")
|
||||
self.provider = None
|
||||
|
||||
def reason(self, graph: Dict[str, Any], query: str, **options) -> str:
|
||||
"""
|
||||
Reason over the provided knowledge graph to answer a query.
|
||||
|
||||
Args:
|
||||
graph: The knowledge graph dictionary (entities, relationships).
|
||||
query: The natural language query.
|
||||
**options: Additional reasoning options.
|
||||
|
||||
Returns:
|
||||
str: The reasoning result/answer.
|
||||
"""
|
||||
if not self.provider:
|
||||
return "Error: LLM provider not initialized for GraphReasoner. Check your configuration."
|
||||
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="reasoning",
|
||||
submodule="GraphReasoner",
|
||||
message=f"Reasoning over graph for query: {query[:50]}..."
|
||||
)
|
||||
|
||||
try:
|
||||
# 1. Prepare graph context
|
||||
context = self._prepare_graph_context(graph)
|
||||
|
||||
# 2. Build prompt
|
||||
prompt = self._build_reasoning_prompt(context, query)
|
||||
|
||||
# 3. Generate response
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Calling LLM for reasoning...")
|
||||
response = self.provider.generate(prompt, **options)
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Reasoning failed: {e}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
return f"Error during reasoning: {str(e)}"
|
||||
|
||||
def _prepare_graph_context(self, graph: Dict[str, Any]) -> str:
|
||||
"""Convert graph to a text representation for the LLM."""
|
||||
entities = graph.get("entities", [])
|
||||
relationships = graph.get("relationships", [])
|
||||
|
||||
context_lines = ["Knowledge Graph Context:"]
|
||||
|
||||
if entities:
|
||||
context_lines.append("\nEntities:")
|
||||
for ent in entities:
|
||||
name = ent.get("name", ent.get("id", "Unknown"))
|
||||
etype = ent.get("type", "Entity")
|
||||
props = ent.get("properties", {})
|
||||
props_str = f" ({props})" if props else ""
|
||||
context_lines.append(f"- {name} [{etype}]{props_str}")
|
||||
|
||||
if relationships:
|
||||
context_lines.append("\nRelationships:")
|
||||
for rel in relationships:
|
||||
src = rel.get("source", rel.get("source_id", "Unknown"))
|
||||
tgt = rel.get("target", rel.get("target_id", "Unknown"))
|
||||
rtype = rel.get("type", "Relationship")
|
||||
props = rel.get("properties", {})
|
||||
props_str = f" ({props})" if props else ""
|
||||
context_lines.append(f"- {src} --[{rtype}]--> {tgt}{props_str}")
|
||||
|
||||
return "\n".join(context_lines)
|
||||
|
||||
def _build_reasoning_prompt(self, context: str, query: str) -> str:
|
||||
"""Build the expert reasoning prompt."""
|
||||
return f"""You are an advanced Knowledge Graph Reasoning Assistant.
|
||||
Use the following graph data to answer the user's question accurately.
|
||||
If the information is not in the graph, state that clearly but try to provide the best possible reasoning based on what's available.
|
||||
|
||||
{context}
|
||||
|
||||
Question: {query}
|
||||
|
||||
Answer strictly based on the provided graph context. Provide a concise yet thorough explanation."""
|
||||
@@ -71,7 +71,7 @@ License: MIT
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -97,10 +97,49 @@ class BaseProvider:
|
||||
"""Generate text - must be implemented."""
|
||||
raise NotImplementedError
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> dict:
|
||||
def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]:
|
||||
"""Generate structured output - must be implemented."""
|
||||
raise NotImplementedError
|
||||
|
||||
def _parse_json(self, text: str) -> Union[dict, list]:
|
||||
"""Extract and parse JSON from text, supporting objects and lists."""
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
# Try to find JSON boundaries
|
||||
# Look for the first occurrence of { or [
|
||||
start_obj = text.find("{")
|
||||
start_list = text.find("[")
|
||||
|
||||
# Determine which one starts first
|
||||
start = -1
|
||||
end = -1
|
||||
|
||||
if start_obj >= 0 and (start_list < 0 or start_obj < start_list):
|
||||
start = start_obj
|
||||
end = text.rfind("}") + 1
|
||||
elif start_list >= 0:
|
||||
start = start_list
|
||||
end = text.rfind("]") + 1
|
||||
|
||||
if start >= 0 and end > start:
|
||||
try:
|
||||
return json.loads(text[start:end])
|
||||
except json.JSONDecodeError as e:
|
||||
# If that failed, maybe it's a list that ends with ] but we picked } earlier?
|
||||
# Or vice versa. Let's try the other boundary if they exist.
|
||||
if start == start_obj and start_list >= 0:
|
||||
start = start_list
|
||||
end = text.rfind("]") + 1
|
||||
if start >= 0 and end > start:
|
||||
try:
|
||||
return json.loads(text[start:end])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
raise ProcessingError(f"Failed to parse extracted JSON: {e}")
|
||||
raise ProcessingError("No JSON structure found in response")
|
||||
|
||||
|
||||
class OpenAIProvider(BaseProvider):
|
||||
"""OpenAI provider implementation."""
|
||||
@@ -157,7 +196,10 @@ class OpenAIProvider(BaseProvider):
|
||||
response_format={"type": "json_object"},
|
||||
temperature=kwargs.get("temperature", 0.3),
|
||||
)
|
||||
return json.loads(response.choices[0].message.content)
|
||||
try:
|
||||
return self._parse_json(response.choices[0].message.content)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from OpenAI response: {e}")
|
||||
|
||||
|
||||
class GeminiProvider(BaseProvider):
|
||||
@@ -212,15 +254,9 @@ class GeminiProvider(BaseProvider):
|
||||
json_prompt = f"{prompt}\n\nReturn the response as valid JSON only."
|
||||
response = self.client.generate_content(json_prompt)
|
||||
try:
|
||||
return json.loads(response.text)
|
||||
except json.JSONDecodeError:
|
||||
# Try to extract JSON from response
|
||||
text = response.text
|
||||
start = text.find("{")
|
||||
end = text.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
return json.loads(text[start:end])
|
||||
raise ProcessingError("Failed to parse JSON from Gemini response")
|
||||
return self._parse_json(response.text)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from Gemini response: {e}")
|
||||
|
||||
|
||||
class GroqProvider(BaseProvider):
|
||||
@@ -279,14 +315,9 @@ class GroqProvider(BaseProvider):
|
||||
temperature=kwargs.get("temperature", 0.3),
|
||||
)
|
||||
try:
|
||||
return json.loads(response.choices[0].message.content)
|
||||
except json.JSONDecodeError:
|
||||
text = response.choices[0].message.content
|
||||
start = text.find("{")
|
||||
end = text.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
return json.loads(text[start:end])
|
||||
raise ProcessingError("Failed to parse JSON from Groq response")
|
||||
return self._parse_json(response.choices[0].message.content)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from Groq response: {e}")
|
||||
|
||||
|
||||
class AnthropicProvider(BaseProvider):
|
||||
@@ -348,14 +379,9 @@ class AnthropicProvider(BaseProvider):
|
||||
messages=[{"role": "user", "content": json_prompt}],
|
||||
)
|
||||
try:
|
||||
return json.loads(response.content[0].text)
|
||||
except json.JSONDecodeError:
|
||||
text = response.content[0].text
|
||||
start = text.find("{")
|
||||
end = text.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
return json.loads(text[start:end])
|
||||
raise ProcessingError("Failed to parse JSON from Anthropic response")
|
||||
return self._parse_json(response.content[0].text)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from Anthropic response: {e}")
|
||||
|
||||
|
||||
class OllamaProvider(BaseProvider):
|
||||
@@ -421,14 +447,9 @@ class OllamaProvider(BaseProvider):
|
||||
options={"temperature": kwargs.get("temperature", 0.3)},
|
||||
)
|
||||
try:
|
||||
return json.loads(response.get("response", "{}"))
|
||||
except json.JSONDecodeError:
|
||||
text = response.get("response", "")
|
||||
start = text.find("{")
|
||||
end = text.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
return json.loads(text[start:end])
|
||||
raise ProcessingError("Failed to parse JSON from Ollama response")
|
||||
return self._parse_json(response.get("response", "{}"))
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from Ollama response: {e}")
|
||||
|
||||
|
||||
class DeepSeekProvider(BaseProvider):
|
||||
@@ -463,8 +484,8 @@ class DeepSeekProvider(BaseProvider):
|
||||
temperature=kwargs.get("temperature", 0.3),
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> dict:
|
||||
def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]:
|
||||
"""Generate structured output."""
|
||||
if not self.client:
|
||||
raise ProcessingError("DeepSeek client not initialized.")
|
||||
response = self.client.chat.completions.create(
|
||||
@@ -472,15 +493,10 @@ class DeepSeekProvider(BaseProvider):
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=kwargs.get("temperature", 0.3),
|
||||
)
|
||||
text = response.choices[0].message.content
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
start = text.find("{")
|
||||
end = text.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
return json.loads(text[start:end])
|
||||
raise ProcessingError("Failed to parse JSON from DeepSeek response")
|
||||
return self._parse_json(response.choices[0].message.content)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from DeepSeek response: {e}")
|
||||
|
||||
class HuggingFaceLLMProvider(BaseProvider):
|
||||
"""HuggingFace transformers for LLM tasks."""
|
||||
|
||||
Reference in New Issue
Block a user