From 47809f2ef986aa8928349ec26e18c2b714da5299 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sat, 3 Jan 2026 20:39:37 +0530 Subject: [PATCH] Fix: Make PyTorch import lazy to avoid DLL errors on Windows - Remove top-level torch import from providers.py - Add lazy imports in HuggingFaceLLMProvider and HuggingFaceModelLoader - Remove hardcoded API key from notebook - PyTorch now only loads when HuggingFace providers are instantiated Fixes #129 --- .../finance/03_Earnings_Call_Analysis.ipynb | 1993 +++++++++-------- semantica/semantic_extract/providers.py | 24 +- 2 files changed, 1046 insertions(+), 971 deletions(-) diff --git a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb index 7ad03075..078db698 100644 --- a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb +++ b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb @@ -1,972 +1,1027 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb)\n", - "\n", - "# Earnings Call Transcript Analysis with Docling and Semantica\n", - "\n", - "## 📄 Earnings Call Transcript PDF\n", - "\n", - "**Example:** Download a single earnings call transcript PDF from [SEC EDGAR](https://www.sec.gov/edgar/searchedgar/companysearch.html) or company investor relations pages. Use one PDF file for this analysis.\n", - "\n", - "---\n", - "\n", - "## Overview\n", - "\n", - "Extract insights from earnings call transcripts by building a knowledge graph with entity extraction, relationship mapping, and GraphRAG-powered Q&A.\n", - "\n", - "**Workflow:** `PDF → Parse → Extract Entities/Relations → Build KG → GraphRAG → Answers`\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Installation\n", - "\n", - "```bash\n", - "pip install semantica\n", - "```\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize Groq LLM provider\n", - "from semantica.llms import Groq\n", - "import os\n", - "\n", - "groq_llm = Groq(\n", - " model=\"llama-3.1-8b-instant\",\n", - " api_key=os.getenv(\"GROQ_API_KEY\")\n", - ")\n", - "\n", - "print(f\"✓ Groq LLM initialized: {groq_llm.model}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Parse PDF with Docling\n", - "\n", - "Parse earnings call PDF and extract financial tables using DoclingParser.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 1: Parse PDF with Docling\n", - "from semantica.parse import DoclingParser\n", - "\n", - "parser = DoclingParser(\n", - " export_format=\"markdown\",\n", - " enable_ocr=False,\n", - " table_extraction_mode=\"auto\"\n", - ")\n", - "\n", - "# Example parsed document (replace with actual PDF parsing)\n", - "parsed_doc = {\n", - " \"full_text\": \"\"\"Q1 2024 Earnings Call Transcript\n", - "\n", - "Company: TechCorp Inc.\n", - "Date: January 25, 2024\n", - "\n", - "Prepared Remarks:\n", - "Our revenue for Q1 2024 was $2.5 billion, representing 15% year-over-year growth. \n", - "EPS was $1.25 per share. We expect Q2 revenue to be between $2.6 and $2.8 billion.\n", - "\n", - "Q&A Session:\n", - "Analyst: What's your guidance for the full year?\n", - "CEO: We're maintaining our full-year guidance of $10.5 to $11 billion in revenue.\"\"\",\n", - " \"tables\": [],\n", - " \"metadata\": {\"title\": \"Q1 2024 Earnings Call\"}\n", - "}\n", - "\n", - "print(f\"✓ Document parsed: {parsed_doc['metadata'].get('title', 'Unknown')}\")\n", - "print(f\" Text length: {len(parsed_doc['full_text'])} characters\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Normalize Text\n", - "\n", - "Normalize extracted text using TextNormalizer for consistent processing.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 2: Normalize text with Semantica\n", - "from semantica.normalize import TextNormalizer\n", - "\n", - "text_normalizer = TextNormalizer()\n", - "normalized_text = text_normalizer.normalize(\n", - " parsed_doc[\"full_text\"],\n", - " case=\"lower\",\n", - " remove_extra_whitespace=True\n", - ")\n", - "\n", - "print(f\"✓ Text normalized: {len(normalized_text)} characters\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Extract Entities\n", - "\n", - "Extract entities (organizations, people, financial terms) using NERExtractor with Groq LLM.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 3: Extract entities using NERExtractor with Groq\n", - "from semantica.semantic_extract import NERExtractor\n", - "\n", - "ner = NERExtractor(\n", - " method=\"llm\",\n", - " provider=\"groq\",\n", - " llm_model=\"llama-3.1-8b-instant\",\n", - " min_confidence=0.7\n", - ")\n", - "\n", - "entities = ner.extract_entities(\n", - " normalized_text,\n", - " entity_types=[\"ORG\", \"PERSON\", \"MONEY\", \"DATE\", \"PERCENT\"]\n", - ")\n", - "\n", - "print(f\"✓ Entities extracted: {len(entities)}\")\n", - "if entities:\n", - " print(f\" Sample: {entities[0].text} ({entities[0].label})\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Extract Financial Metrics\n", - "\n", - "Extract financial metrics (money, percentages, dates) from text and tables.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 4: Extract financial metrics using NERExtractor\n", - "financial_entities = ner.extract_entities(\n", - " normalized_text,\n", - " entity_types=[\"MONEY\", \"PERCENT\", \"DATE\"]\n", - ")\n", - "\n", - "financial_metrics = {}\n", - "for entity in financial_entities:\n", - " if entity.label == \"MONEY\":\n", - " financial_metrics[entity.text] = entity.text\n", - "\n", - "print(f\"✓ Financial entities: {len(financial_entities)}\")\n", - "print(f\" Financial metrics: {len(financial_metrics)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Extract Relationships\n", - "\n", - "Extract relationships between entities using RelationExtractor with Groq LLM.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 5: Extract relationships using RelationExtractor with Groq LLM\n", - "from semantica.semantic_extract import RelationExtractor\n", - "\n", - "relation_extractor = RelationExtractor(\n", - " method=\"llm\",\n", - " confidence_threshold=0.6,\n", - " relation_types=[\"HAS_REVENUE\", \"HAS_EPS\", \"STATES\", \"PROVIDES_GUIDANCE\", \"OPERATES_IN\"]\n", - ")\n", - "\n", - "relationships = relation_extractor.extract_relations(\n", - " normalized_text,\n", - " entities=entities,\n", - " provider=\"groq\",\n", - " llm_model=\"llama-3.1-8b-instant\"\n", - ")\n", - "\n", - "print(f\"✓ Relationships extracted: {len(relationships)}\")\n", - "if relationships:\n", - " rel = relationships[0]\n", - " print(f\" Sample: {rel.subject.text} → {rel.predicate} → {rel.object.text}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 6: Extract RDF Triplets\n", - "\n", - "Extract RDF triplets (subject-predicate-object) using TripletExtractor with Groq LLM.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 6: Extract RDF triplets using TripletExtractor with Groq LLM\n", - "from semantica.semantic_extract import TripletExtractor\n", - "\n", - "triplet_extractor = TripletExtractor(\n", - " method=\"llm\",\n", - " include_temporal=True,\n", - " include_provenance=True\n", - ")\n", - "\n", - "triplets = triplet_extractor.extract_triplets(\n", - " normalized_text,\n", - " entities=entities,\n", - " relations=relationships,\n", - " provider=\"groq\",\n", - " llm_model=\"llama-3.1-8b-instant\"\n", - ")\n", - "\n", - "validated_triplets = triplet_extractor.validate_triplets(triplets)\n", - "\n", - "print(f\"✓ RDF triplets extracted: {len(triplets)}\")\n", - "if triplets:\n", - " t = triplets[0]\n", - " print(f\" Sample: {t.subject} → {t.predicate} → {t.object}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 7: Detect Conflicts\n", - "\n", - "Detect conflicts in extracted entities and relationships using ConflictDetector.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 7: Detect conflicts in extracted entities and relationships\n", - "from semantica.conflicts import ConflictDetector, SourceTracker, SourceReference\n", - "\n", - "source_tracker = SourceTracker()\n", - "conflict_detector = ConflictDetector(\n", - " source_tracker=source_tracker,\n", - " confidence_threshold=0.7\n", - ")\n", - "\n", - "# Track sources for entities\n", - "for entity in entities:\n", - " entity_id = getattr(entity, 'id', None) or getattr(entity, 'text', '')\n", - " entity_name = getattr(entity, 'text', '')\n", - " source_tracker.track_property_source(\n", - " entity_id,\n", - " 'name',\n", - " entity_name,\n", - " source=SourceReference(\n", - " source='earnings_call',\n", - " timestamp='2024-Q1',\n", - " metadata={'entity_type': getattr(entity, 'label', 'UNKNOWN')}\n", - " )\n", - " )\n", - "\n", - "# Detect value conflicts\n", - "value_conflicts = conflict_detector.detect_value_conflicts(\n", - " [{'id': getattr(e, 'id', ''), 'name': getattr(e, 'text', '')} for e in entities],\n", - " property_name='name'\n", - ")\n", - "\n", - "# Detect relationship conflicts\n", - "relationship_conflicts = conflict_detector.detect_relationship_conflicts(relationships)\n", - "\n", - "print(f\"✓ Conflicts detected\")\n", - "print(f\" Value conflicts: {len(value_conflicts)}\")\n", - "print(f\" Relationship conflicts: {len(relationship_conflicts)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 8: Resolve Conflicts\n", - "\n", - "Resolve detected conflicts using ConflictResolver with voting strategy.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 8: Resolve conflicts using ConflictResolver\n", - "from semantica.conflicts import ConflictResolver\n", - "\n", - "conflict_resolver = ConflictResolver(\n", - " default_strategy='voting',\n", - " source_tracker=source_tracker\n", - ")\n", - "\n", - "# Resolve value conflicts\n", - "resolved_entities = list(entities)\n", - "resolved_conflicts = []\n", - "for conflict in value_conflicts:\n", - " resolution = conflict_resolver.resolve_conflict(conflict, strategy='voting')\n", - " resolved_conflicts.append(resolution)\n", - "\n", - "# Resolve relationship conflicts\n", - "resolved_relationships = list(relationships)\n", - "for conflict in relationship_conflicts:\n", - " resolution = conflict_resolver.resolve_conflict(conflict, strategy='voting')\n", - " resolved_conflicts.append(resolution)\n", - "\n", - "print(f\"✓ Conflicts resolved: {len(resolved_conflicts)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 9: Deduplicate Entities\n", - "\n", - "Detect and merge duplicate entities using DuplicateDetector and EntityMerger.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 9: Deduplicate entities using DuplicateDetector and EntityMerger\n", - "from semantica.deduplication import DuplicateDetector, EntityMerger\n", - "\n", - "duplicate_detector = DuplicateDetector(\n", - " similarity_threshold=0.8,\n", - " confidence_threshold=0.7\n", - ")\n", - "\n", - "# Convert entities to dict format\n", - "entity_dicts = []\n", - "for entity in resolved_entities:\n", - " entity_dicts.append({\n", - " 'id': getattr(entity, 'id', ''),\n", - " 'name': getattr(entity, 'text', ''),\n", - " 'type': getattr(entity, 'label', 'UNKNOWN'),\n", - " 'confidence': getattr(entity, 'confidence', 1.0),\n", - " 'metadata': getattr(entity, 'metadata', {})\n", - " })\n", - "\n", - "# Detect duplicates\n", - "duplicates = duplicate_detector.detect_duplicates(entity_dicts)\n", - "\n", - "# Merge duplicates\n", - "entity_merger = EntityMerger(preserve_provenance=True)\n", - "merge_operations = entity_merger.merge_duplicates(\n", - " entity_dicts,\n", - " strategy='keep_most_complete'\n", - ")\n", - "\n", - "merged_entities = [op.merged_entity for op in merge_operations]\n", - "\n", - "print(f\"✓ Deduplication complete\")\n", - "print(f\" Original entities: {len(entity_dicts)}\")\n", - "print(f\" Merged entities: {len(merged_entities)}\")\n", - "print(f\" Duplicates removed: {len(entity_dicts) - len(merged_entities)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 10: Build Knowledge Graph\n", - "\n", - "Build knowledge graph from cleaned entities, relationships, and triplets using GraphBuilder.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 10: Build knowledge graph from cleaned entities, resolved relationships, and triplets\n", - "from semantica.kg import GraphBuilder\n", - "\n", - "graph_builder = GraphBuilder(\n", - " merge_entities=True,\n", - " entity_resolution_strategy=\"fuzzy\"\n", - ")\n", - "\n", - "# Convert triplets to relationships format\n", - "triplet_relationships = []\n", - "for triplet in triplets:\n", - " triplet_relationships.append({\n", - " \"source\": triplet.subject,\n", - " \"predicate\": triplet.predicate,\n", - " \"target\": triplet.object,\n", - " \"confidence\": triplet.confidence,\n", - " \"metadata\": triplet.metadata\n", - " })\n", - "\n", - "all_relationships = resolved_relationships + triplet_relationships\n", - "\n", - "kg_data = {\n", - " \"entities\": merged_entities,\n", - " \"relationships\": all_relationships,\n", - " \"triplets\": triplets,\n", - " \"metadata\": {\n", - " \"source\": \"earnings_call_transcript\",\n", - " \"financial_metrics\": financial_metrics,\n", - " \"extraction_method\": \"Groq LLM\"\n", - " }\n", - "}\n", - "\n", - "knowledge_graph = graph_builder.build(\n", - " sources=[kg_data],\n", - " merge_entities=True\n", - ")\n", - "\n", - "print(f\"✓ Knowledge graph built\")\n", - "print(f\" Entities: {len(knowledge_graph.get('entities', []))}\")\n", - "print(f\" Relationships: {len(knowledge_graph.get('relationships', []))}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 11: Analyze Knowledge Graph\n", - "\n", - "Analyze graph structure using GraphAnalyzer (centrality, communities, connectivity).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 11: Analyze knowledge graph using GraphAnalyzer\n", - "from semantica.kg import GraphAnalyzer\n", - "\n", - "graph_analyzer = GraphAnalyzer()\n", - "analysis = graph_analyzer.analyze_graph(knowledge_graph)\n", - "centrality = graph_analyzer.calculate_centrality(knowledge_graph, 'degree')\n", - "communities = graph_analyzer.detect_communities(knowledge_graph, algorithm='louvain')\n", - "connectivity = graph_analyzer.analyze_connectivity(knowledge_graph)\n", - "metrics = graph_analyzer.compute_metrics(knowledge_graph)\n", - "\n", - "top_entities = []\n", - "if centrality and 'rankings' in centrality:\n", - " top_entities = centrality['rankings'][:5]\n", - "\n", - "num_communities = len(communities.get('communities', [])) if isinstance(communities, dict) else 0\n", - "\n", - "print(f\"✓ Graph analysis complete\")\n", - "print(f\" Communities: {num_communities}\")\n", - "print(f\" Top entities: {len(top_entities)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 12: Build Context Graph\n", - "\n", - "Build ContextGraph from knowledge graph for enhanced retrieval and GraphRAG.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 12: Build context graph for enhanced retrieval\n", - "from semantica.context import ContextGraph\n", - "\n", - "context_graph = ContextGraph(\n", - " extract_entities=True,\n", - " extract_relationships=True\n", - ")\n", - "\n", - "# Convert knowledge graph to context graph format\n", - "nodes = []\n", - "for entity in knowledge_graph.get('entities', []):\n", - " nodes.append({\n", - " \"id\": entity.get('id', entity.get('name', '')),\n", - " \"type\": entity.get('type', 'entity'),\n", - " \"properties\": {\n", - " \"content\": entity.get('name', ''),\n", - " \"confidence\": entity.get('confidence', 1.0),\n", - " **entity.get('metadata', {})\n", - " }\n", - " })\n", - "\n", - "edges = []\n", - "for rel in knowledge_graph.get('relationships', []):\n", - " edges.append({\n", - " \"source_id\": rel.get('source', ''),\n", - " \"target_id\": rel.get('target', ''),\n", - " \"type\": rel.get('predicate', 'related_to'),\n", - " \"weight\": rel.get('confidence', 1.0)\n", - " })\n", - "\n", - "node_count = context_graph.add_nodes(nodes)\n", - "edge_count = context_graph.add_edges(edges)\n", - "\n", - "print(f\"✓ Context graph built\")\n", - "print(f\" Nodes: {node_count}\")\n", - "print(f\" Edges: {edge_count}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 13: Context Retrieval\n", - "\n", - "Set up hybrid retrieval (vector + graph) using ContextRetriever for GraphRAG queries.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 13: Set up hybrid context retrieval and demonstrate GraphRAG\n", - "from semantica.vector_store import VectorStore\n", - "from semantica.context import ContextRetriever\n", - "\n", - "# Initialize VectorStore\n", - "vector_store = VectorStore(backend=\"faiss\")\n", - "vector_store.add(\n", - " texts=[parsed_doc[\"full_text\"]],\n", - " metadata=[{\"source\": \"earnings_call\", \"type\": \"transcript\"}]\n", - ")\n", - "\n", - "# Initialize ContextRetriever\n", - "context_retriever = ContextRetriever(\n", - " knowledge_graph=context_graph,\n", - " vector_store=vector_store,\n", - " hybrid_alpha=0.6,\n", - " use_graph_expansion=True,\n", - " max_expansion_hops=2\n", - ")\n", - "\n", - "# Retrieve context for financial queries\n", - "financial_queries = [\n", - " \"What was the company's revenue guidance?\",\n", - " \"What were the key financial metrics discussed?\"\n", - "]\n", - "\n", - "retrieved_contexts = []\n", - "for query in financial_queries:\n", - " results = context_retriever.retrieve(\n", - " query=query,\n", - " max_results=3,\n", - " min_relevance_score=0.2\n", - " )\n", - " retrieved_contexts.append({\n", - " \"query\": query,\n", - " \"results\": results,\n", - " \"count\": len(results)\n", - " })\n", - "\n", - "print(f\"✓ Hybrid retrieval configured\")\n", - "print(f\" Queries processed: {len(retrieved_contexts)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 14: Entity Linking\n", - "\n", - "Link entities across sources and assign URIs using EntityLinker.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 14: Link entities using EntityLinker\n", - "from semantica.context import EntityLinker\n", - "\n", - "entity_linker = EntityLinker(knowledge_graph=knowledge_graph)\n", - "\n", - "# Assign URIs to key entities\n", - "linked_entities = []\n", - "for entity in merged_entities[:10]:\n", - " entity_id = entity.get('id', entity.get('name', ''))\n", - " entity_name = entity.get('name', '')\n", - " entity_type = entity.get('type', 'UNKNOWN')\n", - " \n", - " uri = entity_linker.assign_uri(\n", - " entity_id=entity_id,\n", - " text=entity_name,\n", - " entity_type=entity_type\n", - " )\n", - " linked_entities.append({\n", - " \"entity_id\": entity_id,\n", - " \"name\": entity_name,\n", - " \"uri\": uri,\n", - " \"type\": entity_type\n", - " })\n", - "\n", - "# Build entity web\n", - "entity_web = entity_linker.build_entity_web()\n", - "\n", - "print(f\"✓ Entity linking complete\")\n", - "print(f\" Entities linked: {len(linked_entities)}\")\n", - "print(f\" Entity web nodes: {len(entity_web.get('nodes', []))}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 15: Agent Memory\n", - "\n", - "Store and retrieve memories using AgentMemory with RAG integration.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 15: Store and retrieve memories using AgentMemory\n", - "from semantica.context import AgentMemory\n", - "\n", - "agent_memory = AgentMemory(\n", - " vector_store=vector_store,\n", - " knowledge_graph=knowledge_graph,\n", - " retention_days=30\n", - ")\n", - "\n", - "# Store earnings call memories\n", - "memory_ids = []\n", - "memory_contents = [\n", - " f\"Earnings call transcript: {parsed_doc['metadata'].get('title', 'Q1 2024')}\",\n", - " f\"Financial metrics extracted: {len(financial_metrics)} metrics\",\n", - " f\"Key entities identified: {len(merged_entities)} entities\"\n", - "]\n", - "\n", - "for content in memory_contents:\n", - " memory_id = agent_memory.store(\n", - " content=content,\n", - " metadata={\"source\": \"earnings_call\", \"type\": \"transcript_analysis\"},\n", - " extract_entities=True,\n", - " extract_relationships=True\n", - " )\n", - " memory_ids.append(memory_id)\n", - "\n", - "# Retrieve memories\n", - "financial_memories = agent_memory.retrieve(\n", - " query=\"financial metrics and earnings\",\n", - " max_results=5\n", - ")\n", - "\n", - "memory_stats = agent_memory.get_statistics()\n", - "\n", - "print(f\"✓ Agent memory configured\")\n", - "print(f\" Memories stored: {len(memory_ids)}\")\n", - "print(f\" Total memories: {memory_stats.get('total_memories', 0)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 16: Agent Context\n", - "\n", - "Unified context management with AgentContext (auto-detects RAG vs GraphRAG).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 16: High-level context management with AgentContext\n", - "from semantica.context import AgentContext\n", - "\n", - "agent_context = AgentContext(\n", - " vector_store=vector_store,\n", - " knowledge_graph=context_graph,\n", - " use_graph_expansion=True,\n", - " max_expansion_hops=2,\n", - " hybrid_alpha=0.6,\n", - " retention_days=30\n", - ")\n", - "\n", - "# Store content with auto-extraction\n", - "memory_id = agent_context.store(\n", - " content=parsed_doc[\"full_text\"][:1000],\n", - " metadata={\"source\": \"earnings_call\", \"date\": \"2024-Q1\"},\n", - " extract_entities=True,\n", - " extract_relationships=True,\n", - " link_entities=True\n", - ")\n", - "\n", - "# Retrieve with auto-detected GraphRAG\n", - "graphrag_results = agent_context.retrieve(\n", - " query=\"What was discussed about revenue growth?\",\n", - " max_results=5,\n", - " expand_graph=True,\n", - " include_entities=True\n", - ")\n", - "\n", - "context_stats = agent_context.stats()\n", - "\n", - "print(f\"✓ AgentContext configured\")\n", - "print(f\" Memory stored: {memory_id}\")\n", - "print(f\" GraphRAG results: {len(graphrag_results)}\")\n", - "print(f\" Total memories: {context_stats.get('total_memories', 0)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 17: Answer Generation\n", - "\n", - "Generate answers to financial questions using Groq LLM with retrieved context and knowledge graph.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 18: Export Results\n", - "\n", - "Export knowledge graph and analysis results to JSON and RDF formats.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 17: Generate answers using Groq LLM from semantica.llms module\n", - "financial_questions = [\n", - " \"What were the key financial metrics discussed in the earnings call?\",\n", - " \"What guidance did management provide for future quarters?\"\n", - "]\n", - "\n", - "generated_answers = []\n", - "for question in financial_questions:\n", - " # Retrieve relevant context\n", - " context_results = context_retriever.retrieve(\n", - " query=question,\n", - " max_results=3,\n", - " min_relevance_score=0.2\n", - " )\n", - " \n", - " # Build context from retrieved results\n", - " context_text = \"\\n\\n\".join([\n", - " f\"Context {i+1}: {result.get('content', result.get('text', ''))}\"\n", - " for i, result in enumerate(context_results[:3])\n", - " ])\n", - " \n", - " # Extract relevant entities\n", - " relevant_entities = [\n", - " entity.get('name', '') for entity in knowledge_graph.get('entities', [])[:10]\n", - " ]\n", - " entities_text = \", \".join(relevant_entities[:5]) if relevant_entities else \"N/A\"\n", - " \n", - " # Build prompt\n", - " prompt = f\"\"\"Based on the following earnings call transcript context and knowledge graph, answer the question.\n", - "\n", - "Context from transcript:\n", - "{context_text[:1000]}\n", - "\n", - "Key entities identified: {entities_text}\n", - "\n", - "Question: {question}\n", - "\n", - "Provide a comprehensive answer based on the context provided. If information is not available in the context, state that clearly.\n", - "\n", - "Answer:\"\"\"\n", - " \n", - " # Generate answer using Groq LLM\n", - " try:\n", - " answer = groq_llm.generate(\n", - " prompt,\n", - " temperature=0.7,\n", - " max_tokens=500\n", - " )\n", - " generated_answers.append({\n", - " \"question\": question,\n", - " \"answer\": answer,\n", - " \"context_sources\": len(context_results),\n", - " \"model\": groq_llm.model\n", - " })\n", - " except Exception as e:\n", - " generated_answers.append({\n", - " \"question\": question,\n", - " \"answer\": f\"Error generating answer: {str(e)}\",\n", - " \"context_sources\": len(context_results),\n", - " \"model\": groq_llm.model\n", - " })\n", - "\n", - "print(f\"✓ Answer generation complete using Groq LLM\")\n", - "print(f\" LLM Provider: Groq ({groq_llm.model})\")\n", - "print(f\" Questions answered: {len(generated_answers)}\")\n", - "if generated_answers:\n", - " print(f\" Sample question: '{generated_answers[0]['question']}'\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 18: Export structured outputs including triplets\n", - "from semantica.export import JSONExporter, RDFExporter\n", - "\n", - "json_exporter = JSONExporter()\n", - "rdf_exporter = RDFExporter()\n", - "\n", - "# Export knowledge graph to JSON\n", - "kg_json = json_exporter.export(knowledge_graph, format=\"json\")\n", - "\n", - "# Export knowledge graph to RDF (Turtle format)\n", - "rdf_output = rdf_exporter.export_to_rdf(knowledge_graph, format=\"turtle\")\n", - "\n", - "# Create analysis summary\n", - "analysis_summary = {\n", - " \"financial_metrics\": financial_metrics,\n", - " \"extraction_stats\": {\n", - " \"entities\": len(entities),\n", - " \"relationships\": len(relationships),\n", - " \"triplets\": len(triplets),\n", - " \"provider\": f\"Groq LLM (semantica.llms module) - {groq_llm.model}\"\n", - " },\n", - " \"conflict_resolution\": {\n", - " \"conflicts_detected\": len(value_conflicts) + len(relationship_conflicts),\n", - " \"conflicts_resolved\": len(resolved_conflicts),\n", - " \"strategy\": \"voting\"\n", - " },\n", - " \"deduplication\": {\n", - " \"original_entities\": len(entity_dicts),\n", - " \"duplicates_detected\": len(duplicates),\n", - " \"merged_entities\": len(merged_entities),\n", - " \"strategy\": \"keep_most_complete\"\n", - " },\n", - " \"knowledge_graph\": {\n", - " \"entities\": len(knowledge_graph.get('entities', [])),\n", - " \"relationships\": len(knowledge_graph.get('relationships', []))\n", - " },\n", - " \"graph_analytics\": {\n", - " \"metrics\": metrics,\n", - " \"communities\": num_communities,\n", - " \"top_entities\": top_entities[:5] if top_entities else []\n", - " },\n", - " \"context_graph\": {\n", - " \"nodes\": len(context_graph.nodes),\n", - " \"edges\": len(context_graph.edges)\n", - " },\n", - " \"context_retrieval\": {\n", - " \"queries_processed\": len(retrieved_contexts),\n", - " \"total_results\": sum(c[\"count\"] for c in retrieved_contexts)\n", - " },\n", - " \"entity_linking\": {\n", - " \"entities_linked\": len(linked_entities),\n", - " \"entity_web_nodes\": len(entity_web.get('nodes', [])),\n", - " \"entity_web_edges\": len(entity_web.get('edges', []))\n", - " },\n", - " \"agent_memory\": {\n", - " \"memories_stored\": len(memory_ids),\n", - " \"total_memories\": memory_stats.get('total_memories', 0)\n", - " },\n", - " \"agent_context\": {\n", - " \"graphrag_results\": len(graphrag_results),\n", - " \"total_memories\": context_stats.get('total_memories', 0)\n", - " },\n", - " \"answer_generation\": {\n", - " \"questions_answered\": len(generated_answers),\n", - " \"llm_provider\": \"Groq\",\n", - " \"llm_model\": groq_llm.model,\n", - " \"answers\": [\n", - " {\n", - " \"question\": ans[\"question\"],\n", - " \"answer_length\": len(ans[\"answer\"]),\n", - " \"context_sources\": ans[\"context_sources\"]\n", - " }\n", - " for ans in generated_answers\n", - " ]\n", - " }\n", - "}\n", - "\n", - "print(f\"✓ Export complete\")\n", - "print(f\" Analysis summary: {len(analysis_summary)} sections\")\n", - "print(f\" Knowledge graph (JSON): {len(kg_json) if isinstance(kg_json, dict) else 0} items\")\n", - "print(f\" RDF (Turtle): {len(rdf_output)} characters\")\n", - "print(f\" LLM answers generated: {len(generated_answers)}\")\n", - "print(f\" LLM provider: Groq ({groq_llm.model})\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb)\n", + "\n", + "# Earnings Call Transcript Analysis with Docling and Semantica\n", + "\n", + "## 📄 Earnings Call Transcript PDF\n", + "\n", + "**Example:** Download a single earnings call transcript PDF from [SEC EDGAR](https://www.sec.gov/edgar/searchedgar/companysearch.html) or company investor relations pages. Use one PDF file for this analysis.\n", + "\n", + "---\n", + "\n", + "## Overview\n", + "\n", + "Extract insights from earnings call transcripts by building a knowledge graph with entity extraction, relationship mapping, and GraphRAG-powered Q&A.\n", + "\n", + "**Workflow:** `PDF → Parse → Extract Entities/Relations → Build KG → GraphRAG → Answers`\n" + ] }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n" + ] + } + ], + "source": [ + "!pip install -qU semantica \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "ename": "OSError", + "evalue": "[WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mOSError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[2], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;66;03m# Initialize Groq LLM provider\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msemantica\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mllms\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Groq\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mos\u001b[39;00m\n\u001b[0;32m 5\u001b[0m GROQ_API_KEY \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgsk_7BADli5sBUZ4NFBmeIreWGdyb3FYinqEphDW4FUZ0Lg4jV7WXr85\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\llms\\__init__.py:40\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;124;03mLLM Providers Module\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 37\u001b[0m \u001b[38;5;124;03mLicense: MIT\u001b[39;00m\n\u001b[0;32m 38\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m---> 40\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgroq\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Groq\n\u001b[0;32m 41\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mopenai\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m OpenAI\n\u001b[0;32m 42\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhuggingface\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m HuggingFaceLLM\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\llms\\groq.py:9\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;124;03mGroq LLM Provider\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \n\u001b[0;32m 4\u001b[0m \u001b[38;5;124;03mWrapper for Groq API provider with clean interface.\u001b[39;00m\n\u001b[0;32m 5\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, Optional\n\u001b[1;32m----> 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msemantic_extract\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mproviders\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m GroqProvider\n\u001b[0;32m 10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ProcessingError\n\u001b[0;32m 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlogging\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_logger\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\semantic_extract\\__init__.py:51\u001b[0m\n\u001b[0;32m 48\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, List, Optional, Union\n\u001b[0;32m 50\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Config, config\n\u001b[1;32m---> 51\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcoreference_resolver\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 52\u001b[0m CoreferenceChain,\n\u001b[0;32m 53\u001b[0m CoreferenceChainBuilder,\n\u001b[0;32m 54\u001b[0m CoreferenceResolver,\n\u001b[0;32m 55\u001b[0m EntityCoreferenceDetector,\n\u001b[0;32m 56\u001b[0m Mention,\n\u001b[0;32m 57\u001b[0m PronounResolver,\n\u001b[0;32m 58\u001b[0m )\n\u001b[0;32m 59\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mevent_detector\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 60\u001b[0m Event,\n\u001b[0;32m 61\u001b[0m EventClassifier,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 64\u001b[0m TemporalEventProcessor,\n\u001b[0;32m 65\u001b[0m )\n\u001b[0;32m 66\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mextraction_validator\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ExtractionValidator, ValidationResult\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\semantic_extract\\coreference_resolver.py:67\u001b[0m\n\u001b[0;32m 65\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlogging\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_logger\n\u001b[0;32m 66\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprogress_tracker\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_progress_tracker\n\u001b[1;32m---> 67\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mner_extractor\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Entity\n\u001b[0;32m 70\u001b[0m \u001b[38;5;129m@dataclass\u001b[39m\n\u001b[0;32m 71\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mMention\u001b[39;00m:\n\u001b[0;32m 72\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Mention representation.\"\"\"\u001b[39;00m\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\semantic_extract\\ner_extractor.py:75\u001b[0m\n\u001b[0;32m 72\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprogress_tracker\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_progress_tracker\n\u001b[0;32m 74\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 75\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mspacy\u001b[39;00m\n\u001b[0;32m 77\u001b[0m SPACY_AVAILABLE \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m 78\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\spacy\\__init__.py:6\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, Iterable, Union\n\u001b[0;32m 5\u001b[0m \u001b[38;5;66;03m# set library-specific custom warning handling before doing anything else\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01merrors\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m setup_default_warnings\n\u001b[0;32m 8\u001b[0m setup_default_warnings() \u001b[38;5;66;03m# noqa: E402\u001b[39;00m\n\u001b[0;32m 10\u001b[0m \u001b[38;5;66;03m# These are imported as part of the API\u001b[39;00m\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\spacy\\errors.py:3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mwarnings\u001b[39;00m\n\u001b[1;32m----> 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompat\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Literal\n\u001b[0;32m 6\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mErrorsWithCodes\u001b[39;00m(\u001b[38;5;28mtype\u001b[39m):\n\u001b[0;32m 7\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__getattribute__\u001b[39m(\u001b[38;5;28mself\u001b[39m, code):\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\spacy\\compat.py:5\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;124;03m\"\"\"Helpers for Python and platform compatibility.\"\"\"\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msys\u001b[39;00m\n\u001b[1;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mthinc\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutil\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m copy_array\n\u001b[0;32m 7\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mcPickle\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpickle\u001b[39;00m\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\thinc\\__init__.py:5\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mnumpy\u001b[39;00m\n\u001b[0;32m 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mabout\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m __version__\n\u001b[1;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m registry\n\u001b[0;32m 7\u001b[0m \u001b[38;5;66;03m# fmt: off\u001b[39;00m\n\u001b[0;32m 8\u001b[0m __all__ \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m 9\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mregistry\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 10\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__version__\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 11\u001b[0m ]\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\thinc\\config.py:5\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mconfection\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mconfection\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m VARIABLE_RE, Config, ConfigValidationError, Promise\n\u001b[1;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtypes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Decorator\n\u001b[0;32m 8\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mregistry\u001b[39;00m(confection\u001b[38;5;241m.\u001b[39mregistry):\n\u001b[0;32m 9\u001b[0m \u001b[38;5;66;03m# fmt: off\u001b[39;00m\n\u001b[0;32m 10\u001b[0m optimizers: Decorator \u001b[38;5;241m=\u001b[39m catalogue\u001b[38;5;241m.\u001b[39mcreate(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthinc\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moptimizers\u001b[39m\u001b[38;5;124m\"\u001b[39m, entry_points\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\thinc\\types.py:27\u001b[0m\n\u001b[0;32m 24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpydantic\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m GetCoreSchemaHandler\n\u001b[0;32m 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpydantic_core\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m core_schema\n\u001b[1;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompat\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m cupy, has_cupy\n\u001b[0;32m 29\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_cupy:\n\u001b[0;32m 30\u001b[0m get_array_module \u001b[38;5;241m=\u001b[39m cupy\u001b[38;5;241m.\u001b[39mget_array_module\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\thinc\\compat.py:35\u001b[0m\n\u001b[0;32m 31\u001b[0m has_cupy_gpu \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m 34\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m---> 35\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtorch\u001b[39;00m\n\u001b[0;32m 36\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdlpack\u001b[39;00m\n\u001b[0;32m 38\u001b[0m has_torch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\__init__.py:281\u001b[0m\n\u001b[0;32m 277\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[0;32m 279\u001b[0m kernel32\u001b[38;5;241m.\u001b[39mSetErrorMode(prev_error_mode)\n\u001b[1;32m--> 281\u001b[0m \u001b[43m_load_dll_libraries\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 282\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m _load_dll_libraries\n\u001b[0;32m 285\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_get_cuda_dep_paths\u001b[39m(path: \u001b[38;5;28mstr\u001b[39m, lib_folder: \u001b[38;5;28mstr\u001b[39m, lib_name: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m 286\u001b[0m \u001b[38;5;66;03m# Libraries can either be in\u001b[39;00m\n\u001b[0;32m 287\u001b[0m \u001b[38;5;66;03m# path/nvidia/lib_folder/lib or\u001b[39;00m\n\u001b[0;32m 288\u001b[0m \u001b[38;5;66;03m# path/nvidia/cuXX/lib (since CUDA 13.0) or\u001b[39;00m\n\u001b[0;32m 289\u001b[0m \u001b[38;5;66;03m# path/lib_folder/lib\u001b[39;00m\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\__init__.py:264\u001b[0m, in \u001b[0;36m_load_dll_libraries\u001b[1;34m()\u001b[0m\n\u001b[0;32m 260\u001b[0m err \u001b[38;5;241m=\u001b[39m ctypes\u001b[38;5;241m.\u001b[39mWinError(last_error)\n\u001b[0;32m 261\u001b[0m err\u001b[38;5;241m.\u001b[39mstrerror \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 262\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m Error loading \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdll\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m or one of its dependencies.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m 263\u001b[0m )\n\u001b[1;32m--> 264\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[0;32m 265\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m res \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m 266\u001b[0m is_loaded \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "\u001b[1;31mOSError\u001b[0m: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies." + ] + } + ], + "source": [ + "# Initialize Groq LLM provider\n", + "from semantica.llms import Groq\n", + "import os\n", + "\n", + "GROQ_API_KEY = \"\"\n", + " \n", + "groq_llm = Groq(\n", + " model=\"llama-3.1-8b-instant\",\n", + " api_key=os.getenv(\"GROQ_API_KEY\", GROQ_API_KEY))\n", + "\n", + "print(f\"✓ Groq LLM initialized: {groq_llm.model}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Parse PDF with Docling\n", + "\n", + "Parse earnings call PDF and extract financial tables using DoclingParser.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 1: Parse PDF with Docling\n", + "from semantica.parse import DoclingParser\n", + "\n", + "parser = DoclingParser(\n", + " export_format=\"markdown\",\n", + " enable_ocr=False,\n", + " table_extraction_mode=\"auto\"\n", + ")\n", + "\n", + "# Example parsed document (replace with actual PDF parsing)\n", + "parsed_doc = {\n", + " \"full_text\": \"\"\"Q1 2024 Earnings Call Transcript\n", + "\n", + "Company: TechCorp Inc.\n", + "Date: January 25, 2024\n", + "\n", + "Prepared Remarks:\n", + "Our revenue for Q1 2024 was $2.5 billion, representing 15% year-over-year growth. \n", + "EPS was $1.25 per share. We expect Q2 revenue to be between $2.6 and $2.8 billion.\n", + "\n", + "Q&A Session:\n", + "Analyst: What's your guidance for the full year?\n", + "CEO: We're maintaining our full-year guidance of $10.5 to $11 billion in revenue.\"\"\",\n", + " \"tables\": [],\n", + " \"metadata\": {\"title\": \"Q1 2024 Earnings Call\"}\n", + "}\n", + "\n", + "print(f\"✓ Document parsed: {parsed_doc['metadata'].get('title', 'Unknown')}\")\n", + "print(f\" Text length: {len(parsed_doc['full_text'])} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Normalize Text\n", + "\n", + "Normalize extracted text using TextNormalizer for consistent processing.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 2: Normalize text with Semantica\n", + "from semantica.normalize import TextNormalizer\n", + "\n", + "text_normalizer = TextNormalizer()\n", + "normalized_text = text_normalizer.normalize(\n", + " parsed_doc[\"full_text\"],\n", + " case=\"lower\",\n", + " remove_extra_whitespace=True\n", + ")\n", + "\n", + "print(f\"✓ Text normalized: {len(normalized_text)} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Extract Entities\n", + "\n", + "Extract entities (organizations, people, financial terms) using NERExtractor with Groq LLM.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 3: Extract entities using NERExtractor with Groq\n", + "from semantica.semantic_extract import NERExtractor\n", + "\n", + "ner = NERExtractor(\n", + " method=\"llm\",\n", + " provider=\"groq\",\n", + " llm_model=\"llama-3.1-8b-instant\",\n", + " min_confidence=0.7\n", + ")\n", + "\n", + "entities = ner.extract_entities(\n", + " normalized_text,\n", + " entity_types=[\"ORG\", \"PERSON\", \"MONEY\", \"DATE\", \"PERCENT\"]\n", + ")\n", + "\n", + "print(f\"✓ Entities extracted: {len(entities)}\")\n", + "if entities:\n", + " print(f\" Sample: {entities[0].text} ({entities[0].label})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Financial Metrics\n", + "\n", + "Extract financial metrics (money, percentages, dates) from text and tables.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 4: Extract financial metrics using NERExtractor\n", + "financial_entities = ner.extract_entities(\n", + " normalized_text,\n", + " entity_types=[\"MONEY\", \"PERCENT\", \"DATE\"]\n", + ")\n", + "\n", + "financial_metrics = {}\n", + "for entity in financial_entities:\n", + " if entity.label == \"MONEY\":\n", + " financial_metrics[entity.text] = entity.text\n", + "\n", + "print(f\"✓ Financial entities: {len(financial_entities)}\")\n", + "print(f\" Financial metrics: {len(financial_metrics)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Extract Relationships\n", + "\n", + "Extract relationships between entities using RelationExtractor with Groq LLM.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 5: Extract relationships using RelationExtractor with Groq LLM\n", + "from semantica.semantic_extract import RelationExtractor\n", + "\n", + "relation_extractor = RelationExtractor(\n", + " method=\"llm\",\n", + " confidence_threshold=0.6,\n", + " relation_types=[\"HAS_REVENUE\", \"HAS_EPS\", \"STATES\", \"PROVIDES_GUIDANCE\", \"OPERATES_IN\"]\n", + ")\n", + "\n", + "relationships = relation_extractor.extract_relations(\n", + " normalized_text,\n", + " entities=entities,\n", + " provider=\"groq\",\n", + " llm_model=\"llama-3.1-8b-instant\"\n", + ")\n", + "\n", + "print(f\"✓ Relationships extracted: {len(relationships)}\")\n", + "if relationships:\n", + " rel = relationships[0]\n", + " print(f\" Sample: {rel.subject.text} → {rel.predicate} → {rel.object.text}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Extract RDF Triplets\n", + "\n", + "Extract RDF triplets (subject-predicate-object) using TripletExtractor with Groq LLM.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 6: Extract RDF triplets using TripletExtractor with Groq LLM\n", + "from semantica.semantic_extract import TripletExtractor\n", + "\n", + "triplet_extractor = TripletExtractor(\n", + " method=\"llm\",\n", + " include_temporal=True,\n", + " include_provenance=True\n", + ")\n", + "\n", + "triplets = triplet_extractor.extract_triplets(\n", + " normalized_text,\n", + " entities=entities,\n", + " relations=relationships,\n", + " provider=\"groq\",\n", + " llm_model=\"llama-3.1-8b-instant\"\n", + ")\n", + "\n", + "validated_triplets = triplet_extractor.validate_triplets(triplets)\n", + "\n", + "print(f\"✓ RDF triplets extracted: {len(triplets)}\")\n", + "if triplets:\n", + " t = triplets[0]\n", + " print(f\" Sample: {t.subject} → {t.predicate} → {t.object}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Detect Conflicts\n", + "\n", + "Detect conflicts in extracted entities and relationships using ConflictDetector.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 7: Detect conflicts in extracted entities and relationships\n", + "from semantica.conflicts import ConflictDetector, SourceTracker, SourceReference\n", + "\n", + "source_tracker = SourceTracker()\n", + "conflict_detector = ConflictDetector(\n", + " source_tracker=source_tracker,\n", + " confidence_threshold=0.7\n", + ")\n", + "\n", + "# Track sources for entities\n", + "for entity in entities:\n", + " entity_id = getattr(entity, 'id', None) or getattr(entity, 'text', '')\n", + " entity_name = getattr(entity, 'text', '')\n", + " source_tracker.track_property_source(\n", + " entity_id,\n", + " 'name',\n", + " entity_name,\n", + " source=SourceReference(\n", + " source='earnings_call',\n", + " timestamp='2024-Q1',\n", + " metadata={'entity_type': getattr(entity, 'label', 'UNKNOWN')}\n", + " )\n", + " )\n", + "\n", + "# Detect value conflicts\n", + "value_conflicts = conflict_detector.detect_value_conflicts(\n", + " [{'id': getattr(e, 'id', ''), 'name': getattr(e, 'text', '')} for e in entities],\n", + " property_name='name'\n", + ")\n", + "\n", + "# Detect relationship conflicts\n", + "relationship_conflicts = conflict_detector.detect_relationship_conflicts(relationships)\n", + "\n", + "print(f\"✓ Conflicts detected\")\n", + "print(f\" Value conflicts: {len(value_conflicts)}\")\n", + "print(f\" Relationship conflicts: {len(relationship_conflicts)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Resolve Conflicts\n", + "\n", + "Resolve detected conflicts using ConflictResolver with voting strategy.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 8: Resolve conflicts using ConflictResolver\n", + "from semantica.conflicts import ConflictResolver\n", + "\n", + "conflict_resolver = ConflictResolver(\n", + " default_strategy='voting',\n", + " source_tracker=source_tracker\n", + ")\n", + "\n", + "# Resolve value conflicts\n", + "resolved_entities = list(entities)\n", + "resolved_conflicts = []\n", + "for conflict in value_conflicts:\n", + " resolution = conflict_resolver.resolve_conflict(conflict, strategy='voting')\n", + " resolved_conflicts.append(resolution)\n", + "\n", + "# Resolve relationship conflicts\n", + "resolved_relationships = list(relationships)\n", + "for conflict in relationship_conflicts:\n", + " resolution = conflict_resolver.resolve_conflict(conflict, strategy='voting')\n", + " resolved_conflicts.append(resolution)\n", + "\n", + "print(f\"✓ Conflicts resolved: {len(resolved_conflicts)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Deduplicate Entities\n", + "\n", + "Detect and merge duplicate entities using DuplicateDetector and EntityMerger.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 9: Deduplicate entities using DuplicateDetector and EntityMerger\n", + "from semantica.deduplication import DuplicateDetector, EntityMerger\n", + "\n", + "duplicate_detector = DuplicateDetector(\n", + " similarity_threshold=0.8,\n", + " confidence_threshold=0.7\n", + ")\n", + "\n", + "# Convert entities to dict format\n", + "entity_dicts = []\n", + "for entity in resolved_entities:\n", + " entity_dicts.append({\n", + " 'id': getattr(entity, 'id', ''),\n", + " 'name': getattr(entity, 'text', ''),\n", + " 'type': getattr(entity, 'label', 'UNKNOWN'),\n", + " 'confidence': getattr(entity, 'confidence', 1.0),\n", + " 'metadata': getattr(entity, 'metadata', {})\n", + " })\n", + "\n", + "# Detect duplicates\n", + "duplicates = duplicate_detector.detect_duplicates(entity_dicts)\n", + "\n", + "# Merge duplicates\n", + "entity_merger = EntityMerger(preserve_provenance=True)\n", + "merge_operations = entity_merger.merge_duplicates(\n", + " entity_dicts,\n", + " strategy='keep_most_complete'\n", + ")\n", + "\n", + "merged_entities = [op.merged_entity for op in merge_operations]\n", + "\n", + "print(f\"✓ Deduplication complete\")\n", + "print(f\" Original entities: {len(entity_dicts)}\")\n", + "print(f\" Merged entities: {len(merged_entities)}\")\n", + "print(f\" Duplicates removed: {len(entity_dicts) - len(merged_entities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: Build Knowledge Graph\n", + "\n", + "Build knowledge graph from cleaned entities, relationships, and triplets using GraphBuilder.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 10: Build knowledge graph from cleaned entities, resolved relationships, and triplets\n", + "from semantica.kg import GraphBuilder\n", + "\n", + "graph_builder = GraphBuilder(\n", + " merge_entities=True,\n", + " entity_resolution_strategy=\"fuzzy\"\n", + ")\n", + "\n", + "# Convert triplets to relationships format\n", + "triplet_relationships = []\n", + "for triplet in triplets:\n", + " triplet_relationships.append({\n", + " \"source\": triplet.subject,\n", + " \"predicate\": triplet.predicate,\n", + " \"target\": triplet.object,\n", + " \"confidence\": triplet.confidence,\n", + " \"metadata\": triplet.metadata\n", + " })\n", + "\n", + "all_relationships = resolved_relationships + triplet_relationships\n", + "\n", + "kg_data = {\n", + " \"entities\": merged_entities,\n", + " \"relationships\": all_relationships,\n", + " \"triplets\": triplets,\n", + " \"metadata\": {\n", + " \"source\": \"earnings_call_transcript\",\n", + " \"financial_metrics\": financial_metrics,\n", + " \"extraction_method\": \"Groq LLM\"\n", + " }\n", + "}\n", + "\n", + "knowledge_graph = graph_builder.build(\n", + " sources=[kg_data],\n", + " merge_entities=True\n", + ")\n", + "\n", + "print(f\"✓ Knowledge graph built\")\n", + "print(f\" Entities: {len(knowledge_graph.get('entities', []))}\")\n", + "print(f\" Relationships: {len(knowledge_graph.get('relationships', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 11: Analyze Knowledge Graph\n", + "\n", + "Analyze graph structure using GraphAnalyzer (centrality, communities, connectivity).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 11: Analyze knowledge graph using GraphAnalyzer\n", + "from semantica.kg import GraphAnalyzer\n", + "\n", + "graph_analyzer = GraphAnalyzer()\n", + "analysis = graph_analyzer.analyze_graph(knowledge_graph)\n", + "centrality = graph_analyzer.calculate_centrality(knowledge_graph, 'degree')\n", + "communities = graph_analyzer.detect_communities(knowledge_graph, algorithm='louvain')\n", + "connectivity = graph_analyzer.analyze_connectivity(knowledge_graph)\n", + "metrics = graph_analyzer.compute_metrics(knowledge_graph)\n", + "\n", + "top_entities = []\n", + "if centrality and 'rankings' in centrality:\n", + " top_entities = centrality['rankings'][:5]\n", + "\n", + "num_communities = len(communities.get('communities', [])) if isinstance(communities, dict) else 0\n", + "\n", + "print(f\"✓ Graph analysis complete\")\n", + "print(f\" Communities: {num_communities}\")\n", + "print(f\" Top entities: {len(top_entities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 12: Build Context Graph\n", + "\n", + "Build ContextGraph from knowledge graph for enhanced retrieval and GraphRAG.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 12: Build context graph for enhanced retrieval\n", + "from semantica.context import ContextGraph\n", + "\n", + "context_graph = ContextGraph(\n", + " extract_entities=True,\n", + " extract_relationships=True\n", + ")\n", + "\n", + "# Convert knowledge graph to context graph format\n", + "nodes = []\n", + "for entity in knowledge_graph.get('entities', []):\n", + " nodes.append({\n", + " \"id\": entity.get('id', entity.get('name', '')),\n", + " \"type\": entity.get('type', 'entity'),\n", + " \"properties\": {\n", + " \"content\": entity.get('name', ''),\n", + " \"confidence\": entity.get('confidence', 1.0),\n", + " **entity.get('metadata', {})\n", + " }\n", + " })\n", + "\n", + "edges = []\n", + "for rel in knowledge_graph.get('relationships', []):\n", + " edges.append({\n", + " \"source_id\": rel.get('source', ''),\n", + " \"target_id\": rel.get('target', ''),\n", + " \"type\": rel.get('predicate', 'related_to'),\n", + " \"weight\": rel.get('confidence', 1.0)\n", + " })\n", + "\n", + "node_count = context_graph.add_nodes(nodes)\n", + "edge_count = context_graph.add_edges(edges)\n", + "\n", + "print(f\"✓ Context graph built\")\n", + "print(f\" Nodes: {node_count}\")\n", + "print(f\" Edges: {edge_count}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 13: Context Retrieval\n", + "\n", + "Set up hybrid retrieval (vector + graph) using ContextRetriever for GraphRAG queries.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 13: Set up hybrid context retrieval and demonstrate GraphRAG\n", + "from semantica.vector_store import VectorStore\n", + "from semantica.context import ContextRetriever\n", + "\n", + "# Initialize VectorStore\n", + "vector_store = VectorStore(backend=\"faiss\")\n", + "vector_store.add(\n", + " texts=[parsed_doc[\"full_text\"]],\n", + " metadata=[{\"source\": \"earnings_call\", \"type\": \"transcript\"}]\n", + ")\n", + "\n", + "# Initialize ContextRetriever\n", + "context_retriever = ContextRetriever(\n", + " knowledge_graph=context_graph,\n", + " vector_store=vector_store,\n", + " hybrid_alpha=0.6,\n", + " use_graph_expansion=True,\n", + " max_expansion_hops=2\n", + ")\n", + "\n", + "# Retrieve context for financial queries\n", + "financial_queries = [\n", + " \"What was the company's revenue guidance?\",\n", + " \"What were the key financial metrics discussed?\"\n", + "]\n", + "\n", + "retrieved_contexts = []\n", + "for query in financial_queries:\n", + " results = context_retriever.retrieve(\n", + " query=query,\n", + " max_results=3,\n", + " min_relevance_score=0.2\n", + " )\n", + " retrieved_contexts.append({\n", + " \"query\": query,\n", + " \"results\": results,\n", + " \"count\": len(results)\n", + " })\n", + "\n", + "print(f\"✓ Hybrid retrieval configured\")\n", + "print(f\" Queries processed: {len(retrieved_contexts)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 14: Entity Linking\n", + "\n", + "Link entities across sources and assign URIs using EntityLinker.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 14: Link entities using EntityLinker\n", + "from semantica.context import EntityLinker\n", + "\n", + "entity_linker = EntityLinker(knowledge_graph=knowledge_graph)\n", + "\n", + "# Assign URIs to key entities\n", + "linked_entities = []\n", + "for entity in merged_entities[:10]:\n", + " entity_id = entity.get('id', entity.get('name', ''))\n", + " entity_name = entity.get('name', '')\n", + " entity_type = entity.get('type', 'UNKNOWN')\n", + " \n", + " uri = entity_linker.assign_uri(\n", + " entity_id=entity_id,\n", + " text=entity_name,\n", + " entity_type=entity_type\n", + " )\n", + " linked_entities.append({\n", + " \"entity_id\": entity_id,\n", + " \"name\": entity_name,\n", + " \"uri\": uri,\n", + " \"type\": entity_type\n", + " })\n", + "\n", + "# Build entity web\n", + "entity_web = entity_linker.build_entity_web()\n", + "\n", + "print(f\"✓ Entity linking complete\")\n", + "print(f\" Entities linked: {len(linked_entities)}\")\n", + "print(f\" Entity web nodes: {len(entity_web.get('nodes', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 15: Agent Memory\n", + "\n", + "Store and retrieve memories using AgentMemory with RAG integration.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 15: Store and retrieve memories using AgentMemory\n", + "from semantica.context import AgentMemory\n", + "\n", + "agent_memory = AgentMemory(\n", + " vector_store=vector_store,\n", + " knowledge_graph=knowledge_graph,\n", + " retention_days=30\n", + ")\n", + "\n", + "# Store earnings call memories\n", + "memory_ids = []\n", + "memory_contents = [\n", + " f\"Earnings call transcript: {parsed_doc['metadata'].get('title', 'Q1 2024')}\",\n", + " f\"Financial metrics extracted: {len(financial_metrics)} metrics\",\n", + " f\"Key entities identified: {len(merged_entities)} entities\"\n", + "]\n", + "\n", + "for content in memory_contents:\n", + " memory_id = agent_memory.store(\n", + " content=content,\n", + " metadata={\"source\": \"earnings_call\", \"type\": \"transcript_analysis\"},\n", + " extract_entities=True,\n", + " extract_relationships=True\n", + " )\n", + " memory_ids.append(memory_id)\n", + "\n", + "# Retrieve memories\n", + "financial_memories = agent_memory.retrieve(\n", + " query=\"financial metrics and earnings\",\n", + " max_results=5\n", + ")\n", + "\n", + "memory_stats = agent_memory.get_statistics()\n", + "\n", + "print(f\"✓ Agent memory configured\")\n", + "print(f\" Memories stored: {len(memory_ids)}\")\n", + "print(f\" Total memories: {memory_stats.get('total_memories', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 16: Agent Context\n", + "\n", + "Unified context management with AgentContext (auto-detects RAG vs GraphRAG).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 16: High-level context management with AgentContext\n", + "from semantica.context import AgentContext\n", + "\n", + "agent_context = AgentContext(\n", + " vector_store=vector_store,\n", + " knowledge_graph=context_graph,\n", + " use_graph_expansion=True,\n", + " max_expansion_hops=2,\n", + " hybrid_alpha=0.6,\n", + " retention_days=30\n", + ")\n", + "\n", + "# Store content with auto-extraction\n", + "memory_id = agent_context.store(\n", + " content=parsed_doc[\"full_text\"][:1000],\n", + " metadata={\"source\": \"earnings_call\", \"date\": \"2024-Q1\"},\n", + " extract_entities=True,\n", + " extract_relationships=True,\n", + " link_entities=True\n", + ")\n", + "\n", + "# Retrieve with auto-detected GraphRAG\n", + "graphrag_results = agent_context.retrieve(\n", + " query=\"What was discussed about revenue growth?\",\n", + " max_results=5,\n", + " expand_graph=True,\n", + " include_entities=True\n", + ")\n", + "\n", + "context_stats = agent_context.stats()\n", + "\n", + "print(f\"✓ AgentContext configured\")\n", + "print(f\" Memory stored: {memory_id}\")\n", + "print(f\" GraphRAG results: {len(graphrag_results)}\")\n", + "print(f\" Total memories: {context_stats.get('total_memories', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 17: Answer Generation\n", + "\n", + "Generate answers to financial questions using Groq LLM with retrieved context and knowledge graph.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 18: Export Results\n", + "\n", + "Export knowledge graph and analysis results to JSON and RDF formats.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 17: Generate answers using Groq LLM from semantica.llms module\n", + "financial_questions = [\n", + " \"What were the key financial metrics discussed in the earnings call?\",\n", + " \"What guidance did management provide for future quarters?\"\n", + "]\n", + "\n", + "generated_answers = []\n", + "for question in financial_questions:\n", + " # Retrieve relevant context\n", + " context_results = context_retriever.retrieve(\n", + " query=question,\n", + " max_results=3,\n", + " min_relevance_score=0.2\n", + " )\n", + " \n", + " # Build context from retrieved results\n", + " context_text = \"\\n\\n\".join([\n", + " f\"Context {i+1}: {result.get('content', result.get('text', ''))}\"\n", + " for i, result in enumerate(context_results[:3])\n", + " ])\n", + " \n", + " # Extract relevant entities\n", + " relevant_entities = [\n", + " entity.get('name', '') for entity in knowledge_graph.get('entities', [])[:10]\n", + " ]\n", + " entities_text = \", \".join(relevant_entities[:5]) if relevant_entities else \"N/A\"\n", + " \n", + " # Build prompt\n", + " prompt = f\"\"\"Based on the following earnings call transcript context and knowledge graph, answer the question.\n", + "\n", + "Context from transcript:\n", + "{context_text[:1000]}\n", + "\n", + "Key entities identified: {entities_text}\n", + "\n", + "Question: {question}\n", + "\n", + "Provide a comprehensive answer based on the context provided. If information is not available in the context, state that clearly.\n", + "\n", + "Answer:\"\"\"\n", + " \n", + " # Generate answer using Groq LLM\n", + " try:\n", + " answer = groq_llm.generate(\n", + " prompt,\n", + " temperature=0.7,\n", + " max_tokens=500\n", + " )\n", + " generated_answers.append({\n", + " \"question\": question,\n", + " \"answer\": answer,\n", + " \"context_sources\": len(context_results),\n", + " \"model\": groq_llm.model\n", + " })\n", + " except Exception as e:\n", + " generated_answers.append({\n", + " \"question\": question,\n", + " \"answer\": f\"Error generating answer: {str(e)}\",\n", + " \"context_sources\": len(context_results),\n", + " \"model\": groq_llm.model\n", + " })\n", + "\n", + "print(f\"✓ Answer generation complete using Groq LLM\")\n", + "print(f\" LLM Provider: Groq ({groq_llm.model})\")\n", + "print(f\" Questions answered: {len(generated_answers)}\")\n", + "if generated_answers:\n", + " print(f\" Sample question: '{generated_answers[0]['question']}'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 18: Export structured outputs including triplets\n", + "from semantica.export import JSONExporter, RDFExporter\n", + "\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "\n", + "# Export knowledge graph to JSON\n", + "kg_json = json_exporter.export(knowledge_graph, format=\"json\")\n", + "\n", + "# Export knowledge graph to RDF (Turtle format)\n", + "rdf_output = rdf_exporter.export_to_rdf(knowledge_graph, format=\"turtle\")\n", + "\n", + "# Create analysis summary\n", + "analysis_summary = {\n", + " \"financial_metrics\": financial_metrics,\n", + " \"extraction_stats\": {\n", + " \"entities\": len(entities),\n", + " \"relationships\": len(relationships),\n", + " \"triplets\": len(triplets),\n", + " \"provider\": f\"Groq LLM (semantica.llms module) - {groq_llm.model}\"\n", + " },\n", + " \"conflict_resolution\": {\n", + " \"conflicts_detected\": len(value_conflicts) + len(relationship_conflicts),\n", + " \"conflicts_resolved\": len(resolved_conflicts),\n", + " \"strategy\": \"voting\"\n", + " },\n", + " \"deduplication\": {\n", + " \"original_entities\": len(entity_dicts),\n", + " \"duplicates_detected\": len(duplicates),\n", + " \"merged_entities\": len(merged_entities),\n", + " \"strategy\": \"keep_most_complete\"\n", + " },\n", + " \"knowledge_graph\": {\n", + " \"entities\": len(knowledge_graph.get('entities', [])),\n", + " \"relationships\": len(knowledge_graph.get('relationships', []))\n", + " },\n", + " \"graph_analytics\": {\n", + " \"metrics\": metrics,\n", + " \"communities\": num_communities,\n", + " \"top_entities\": top_entities[:5] if top_entities else []\n", + " },\n", + " \"context_graph\": {\n", + " \"nodes\": len(context_graph.nodes),\n", + " \"edges\": len(context_graph.edges)\n", + " },\n", + " \"context_retrieval\": {\n", + " \"queries_processed\": len(retrieved_contexts),\n", + " \"total_results\": sum(c[\"count\"] for c in retrieved_contexts)\n", + " },\n", + " \"entity_linking\": {\n", + " \"entities_linked\": len(linked_entities),\n", + " \"entity_web_nodes\": len(entity_web.get('nodes', [])),\n", + " \"entity_web_edges\": len(entity_web.get('edges', []))\n", + " },\n", + " \"agent_memory\": {\n", + " \"memories_stored\": len(memory_ids),\n", + " \"total_memories\": memory_stats.get('total_memories', 0)\n", + " },\n", + " \"agent_context\": {\n", + " \"graphrag_results\": len(graphrag_results),\n", + " \"total_memories\": context_stats.get('total_memories', 0)\n", + " },\n", + " \"answer_generation\": {\n", + " \"questions_answered\": len(generated_answers),\n", + " \"llm_provider\": \"Groq\",\n", + " \"llm_model\": groq_llm.model,\n", + " \"answers\": [\n", + " {\n", + " \"question\": ans[\"question\"],\n", + " \"answer_length\": len(ans[\"answer\"]),\n", + " \"context_sources\": ans[\"context_sources\"]\n", + " }\n", + " for ans in generated_answers\n", + " ]\n", + " }\n", + "}\n", + "\n", + "print(f\"✓ Export complete\")\n", + "print(f\" Analysis summary: {len(analysis_summary)} sections\")\n", + "print(f\" Knowledge graph (JSON): {len(kg_json) if isinstance(kg_json, dict) else 0} items\")\n", + "print(f\" RDF (Turtle): {len(rdf_output)} characters\")\n", + "print(f\" LLM answers generated: {len(generated_answers)}\")\n", + "print(f\" LLM provider: Groq ({groq_llm.model})\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/semantica/semantic_extract/providers.py b/semantica/semantic_extract/providers.py index 90a447a3..2f973e25 100644 --- a/semantica/semantic_extract/providers.py +++ b/semantica/semantic_extract/providers.py @@ -73,8 +73,6 @@ import json import os from typing import Any, Dict, List, Optional, Union -import torch - from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger from .config import config @@ -544,6 +542,14 @@ class HuggingFaceLLMProvider(BaseProvider): ): """Initialize HuggingFace LLM provider.""" super().__init__(**kwargs) + # Lazy import torch only when needed + try: + import torch + except ImportError: + raise ImportError( + "torch is required for HuggingFaceLLMProvider. Install with: pip install torch" + ) + self.model_name = model_name self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") self.model = None @@ -607,12 +613,23 @@ class HuggingFaceModelLoader: def __init__(self, device: Optional[str] = None): """Initialize HuggingFace model loader.""" + # Lazy import torch only when needed + try: + import torch + except ImportError: + raise ImportError( + "torch is required for HuggingFaceModelLoader. Install with: pip install torch" + ) + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") self._cache: Dict[str, Any] = {} self.logger = get_logger("huggingface_loader") def load_ner_model(self, model_name: str, **kwargs): """Load NER model.""" + # Import torch at method level to ensure it's available + import torch + cache_key = f"{model_name}_ner" if cache_key in self._cache: return self._cache[cache_key] @@ -638,6 +655,9 @@ class HuggingFaceModelLoader: def load_relation_model(self, model_name: str, **kwargs): """Load relation extraction model.""" + # Import torch at method level to ensure it's available + import torch + cache_key = f"{model_name}_relation" if cache_key in self._cache: return self._cache[cache_key]