diff --git a/cookbook/introduction/03_Document_Parsing.ipynb b/cookbook/introduction/03_Document_Parsing.ipynb index eb95b085..6badada6 100644 --- a/cookbook/introduction/03_Document_Parsing.ipynb +++ b/cookbook/introduction/03_Document_Parsing.ipynb @@ -266,4 +266,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb new file mode 100644 index 00000000..7ad03075 --- /dev/null +++ b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb @@ -0,0 +1,972 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb)\n", + "\n", + "# Earnings Call Transcript Analysis with Docling and Semantica\n", + "\n", + "## 📄 Earnings Call Transcript PDF\n", + "\n", + "**Example:** Download a single earnings call transcript PDF from [SEC EDGAR](https://www.sec.gov/edgar/searchedgar/companysearch.html) or company investor relations pages. Use one PDF file for this analysis.\n", + "\n", + "---\n", + "\n", + "## Overview\n", + "\n", + "Extract insights from earnings call transcripts by building a knowledge graph with entity extraction, relationship mapping, and GraphRAG-powered Q&A.\n", + "\n", + "**Workflow:** `PDF → Parse → Extract Entities/Relations → Build KG → GraphRAG → Answers`\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Installation\n", + "\n", + "```bash\n", + "pip install semantica\n", + "```\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Groq LLM provider\n", + "from semantica.llms import Groq\n", + "import os\n", + "\n", + "groq_llm = Groq(\n", + " model=\"llama-3.1-8b-instant\",\n", + " api_key=os.getenv(\"GROQ_API_KEY\")\n", + ")\n", + "\n", + "print(f\"✓ Groq LLM initialized: {groq_llm.model}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Parse PDF with Docling\n", + "\n", + "Parse earnings call PDF and extract financial tables using DoclingParser.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 1: Parse PDF with Docling\n", + "from semantica.parse import DoclingParser\n", + "\n", + "parser = DoclingParser(\n", + " export_format=\"markdown\",\n", + " enable_ocr=False,\n", + " table_extraction_mode=\"auto\"\n", + ")\n", + "\n", + "# Example parsed document (replace with actual PDF parsing)\n", + "parsed_doc = {\n", + " \"full_text\": \"\"\"Q1 2024 Earnings Call Transcript\n", + "\n", + "Company: TechCorp Inc.\n", + "Date: January 25, 2024\n", + "\n", + "Prepared Remarks:\n", + "Our revenue for Q1 2024 was $2.5 billion, representing 15% year-over-year growth. \n", + "EPS was $1.25 per share. We expect Q2 revenue to be between $2.6 and $2.8 billion.\n", + "\n", + "Q&A Session:\n", + "Analyst: What's your guidance for the full year?\n", + "CEO: We're maintaining our full-year guidance of $10.5 to $11 billion in revenue.\"\"\",\n", + " \"tables\": [],\n", + " \"metadata\": {\"title\": \"Q1 2024 Earnings Call\"}\n", + "}\n", + "\n", + "print(f\"✓ Document parsed: {parsed_doc['metadata'].get('title', 'Unknown')}\")\n", + "print(f\" Text length: {len(parsed_doc['full_text'])} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Normalize Text\n", + "\n", + "Normalize extracted text using TextNormalizer for consistent processing.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 2: Normalize text with Semantica\n", + "from semantica.normalize import TextNormalizer\n", + "\n", + "text_normalizer = TextNormalizer()\n", + "normalized_text = text_normalizer.normalize(\n", + " parsed_doc[\"full_text\"],\n", + " case=\"lower\",\n", + " remove_extra_whitespace=True\n", + ")\n", + "\n", + "print(f\"✓ Text normalized: {len(normalized_text)} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Extract Entities\n", + "\n", + "Extract entities (organizations, people, financial terms) using NERExtractor with Groq LLM.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 3: Extract entities using NERExtractor with Groq\n", + "from semantica.semantic_extract import NERExtractor\n", + "\n", + "ner = NERExtractor(\n", + " method=\"llm\",\n", + " provider=\"groq\",\n", + " llm_model=\"llama-3.1-8b-instant\",\n", + " min_confidence=0.7\n", + ")\n", + "\n", + "entities = ner.extract_entities(\n", + " normalized_text,\n", + " entity_types=[\"ORG\", \"PERSON\", \"MONEY\", \"DATE\", \"PERCENT\"]\n", + ")\n", + "\n", + "print(f\"✓ Entities extracted: {len(entities)}\")\n", + "if entities:\n", + " print(f\" Sample: {entities[0].text} ({entities[0].label})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Financial Metrics\n", + "\n", + "Extract financial metrics (money, percentages, dates) from text and tables.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 4: Extract financial metrics using NERExtractor\n", + "financial_entities = ner.extract_entities(\n", + " normalized_text,\n", + " entity_types=[\"MONEY\", \"PERCENT\", \"DATE\"]\n", + ")\n", + "\n", + "financial_metrics = {}\n", + "for entity in financial_entities:\n", + " if entity.label == \"MONEY\":\n", + " financial_metrics[entity.text] = entity.text\n", + "\n", + "print(f\"✓ Financial entities: {len(financial_entities)}\")\n", + "print(f\" Financial metrics: {len(financial_metrics)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Extract Relationships\n", + "\n", + "Extract relationships between entities using RelationExtractor with Groq LLM.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 5: Extract relationships using RelationExtractor with Groq LLM\n", + "from semantica.semantic_extract import RelationExtractor\n", + "\n", + "relation_extractor = RelationExtractor(\n", + " method=\"llm\",\n", + " confidence_threshold=0.6,\n", + " relation_types=[\"HAS_REVENUE\", \"HAS_EPS\", \"STATES\", \"PROVIDES_GUIDANCE\", \"OPERATES_IN\"]\n", + ")\n", + "\n", + "relationships = relation_extractor.extract_relations(\n", + " normalized_text,\n", + " entities=entities,\n", + " provider=\"groq\",\n", + " llm_model=\"llama-3.1-8b-instant\"\n", + ")\n", + "\n", + "print(f\"✓ Relationships extracted: {len(relationships)}\")\n", + "if relationships:\n", + " rel = relationships[0]\n", + " print(f\" Sample: {rel.subject.text} → {rel.predicate} → {rel.object.text}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Extract RDF Triplets\n", + "\n", + "Extract RDF triplets (subject-predicate-object) using TripletExtractor with Groq LLM.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 6: Extract RDF triplets using TripletExtractor with Groq LLM\n", + "from semantica.semantic_extract import TripletExtractor\n", + "\n", + "triplet_extractor = TripletExtractor(\n", + " method=\"llm\",\n", + " include_temporal=True,\n", + " include_provenance=True\n", + ")\n", + "\n", + "triplets = triplet_extractor.extract_triplets(\n", + " normalized_text,\n", + " entities=entities,\n", + " relations=relationships,\n", + " provider=\"groq\",\n", + " llm_model=\"llama-3.1-8b-instant\"\n", + ")\n", + "\n", + "validated_triplets = triplet_extractor.validate_triplets(triplets)\n", + "\n", + "print(f\"✓ RDF triplets extracted: {len(triplets)}\")\n", + "if triplets:\n", + " t = triplets[0]\n", + " print(f\" Sample: {t.subject} → {t.predicate} → {t.object}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Detect Conflicts\n", + "\n", + "Detect conflicts in extracted entities and relationships using ConflictDetector.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 7: Detect conflicts in extracted entities and relationships\n", + "from semantica.conflicts import ConflictDetector, SourceTracker, SourceReference\n", + "\n", + "source_tracker = SourceTracker()\n", + "conflict_detector = ConflictDetector(\n", + " source_tracker=source_tracker,\n", + " confidence_threshold=0.7\n", + ")\n", + "\n", + "# Track sources for entities\n", + "for entity in entities:\n", + " entity_id = getattr(entity, 'id', None) or getattr(entity, 'text', '')\n", + " entity_name = getattr(entity, 'text', '')\n", + " source_tracker.track_property_source(\n", + " entity_id,\n", + " 'name',\n", + " entity_name,\n", + " source=SourceReference(\n", + " source='earnings_call',\n", + " timestamp='2024-Q1',\n", + " metadata={'entity_type': getattr(entity, 'label', 'UNKNOWN')}\n", + " )\n", + " )\n", + "\n", + "# Detect value conflicts\n", + "value_conflicts = conflict_detector.detect_value_conflicts(\n", + " [{'id': getattr(e, 'id', ''), 'name': getattr(e, 'text', '')} for e in entities],\n", + " property_name='name'\n", + ")\n", + "\n", + "# Detect relationship conflicts\n", + "relationship_conflicts = conflict_detector.detect_relationship_conflicts(relationships)\n", + "\n", + "print(f\"✓ Conflicts detected\")\n", + "print(f\" Value conflicts: {len(value_conflicts)}\")\n", + "print(f\" Relationship conflicts: {len(relationship_conflicts)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Resolve Conflicts\n", + "\n", + "Resolve detected conflicts using ConflictResolver with voting strategy.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 8: Resolve conflicts using ConflictResolver\n", + "from semantica.conflicts import ConflictResolver\n", + "\n", + "conflict_resolver = ConflictResolver(\n", + " default_strategy='voting',\n", + " source_tracker=source_tracker\n", + ")\n", + "\n", + "# Resolve value conflicts\n", + "resolved_entities = list(entities)\n", + "resolved_conflicts = []\n", + "for conflict in value_conflicts:\n", + " resolution = conflict_resolver.resolve_conflict(conflict, strategy='voting')\n", + " resolved_conflicts.append(resolution)\n", + "\n", + "# Resolve relationship conflicts\n", + "resolved_relationships = list(relationships)\n", + "for conflict in relationship_conflicts:\n", + " resolution = conflict_resolver.resolve_conflict(conflict, strategy='voting')\n", + " resolved_conflicts.append(resolution)\n", + "\n", + "print(f\"✓ Conflicts resolved: {len(resolved_conflicts)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Deduplicate Entities\n", + "\n", + "Detect and merge duplicate entities using DuplicateDetector and EntityMerger.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 9: Deduplicate entities using DuplicateDetector and EntityMerger\n", + "from semantica.deduplication import DuplicateDetector, EntityMerger\n", + "\n", + "duplicate_detector = DuplicateDetector(\n", + " similarity_threshold=0.8,\n", + " confidence_threshold=0.7\n", + ")\n", + "\n", + "# Convert entities to dict format\n", + "entity_dicts = []\n", + "for entity in resolved_entities:\n", + " entity_dicts.append({\n", + " 'id': getattr(entity, 'id', ''),\n", + " 'name': getattr(entity, 'text', ''),\n", + " 'type': getattr(entity, 'label', 'UNKNOWN'),\n", + " 'confidence': getattr(entity, 'confidence', 1.0),\n", + " 'metadata': getattr(entity, 'metadata', {})\n", + " })\n", + "\n", + "# Detect duplicates\n", + "duplicates = duplicate_detector.detect_duplicates(entity_dicts)\n", + "\n", + "# Merge duplicates\n", + "entity_merger = EntityMerger(preserve_provenance=True)\n", + "merge_operations = entity_merger.merge_duplicates(\n", + " entity_dicts,\n", + " strategy='keep_most_complete'\n", + ")\n", + "\n", + "merged_entities = [op.merged_entity for op in merge_operations]\n", + "\n", + "print(f\"✓ Deduplication complete\")\n", + "print(f\" Original entities: {len(entity_dicts)}\")\n", + "print(f\" Merged entities: {len(merged_entities)}\")\n", + "print(f\" Duplicates removed: {len(entity_dicts) - len(merged_entities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: Build Knowledge Graph\n", + "\n", + "Build knowledge graph from cleaned entities, relationships, and triplets using GraphBuilder.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 10: Build knowledge graph from cleaned entities, resolved relationships, and triplets\n", + "from semantica.kg import GraphBuilder\n", + "\n", + "graph_builder = GraphBuilder(\n", + " merge_entities=True,\n", + " entity_resolution_strategy=\"fuzzy\"\n", + ")\n", + "\n", + "# Convert triplets to relationships format\n", + "triplet_relationships = []\n", + "for triplet in triplets:\n", + " triplet_relationships.append({\n", + " \"source\": triplet.subject,\n", + " \"predicate\": triplet.predicate,\n", + " \"target\": triplet.object,\n", + " \"confidence\": triplet.confidence,\n", + " \"metadata\": triplet.metadata\n", + " })\n", + "\n", + "all_relationships = resolved_relationships + triplet_relationships\n", + "\n", + "kg_data = {\n", + " \"entities\": merged_entities,\n", + " \"relationships\": all_relationships,\n", + " \"triplets\": triplets,\n", + " \"metadata\": {\n", + " \"source\": \"earnings_call_transcript\",\n", + " \"financial_metrics\": financial_metrics,\n", + " \"extraction_method\": \"Groq LLM\"\n", + " }\n", + "}\n", + "\n", + "knowledge_graph = graph_builder.build(\n", + " sources=[kg_data],\n", + " merge_entities=True\n", + ")\n", + "\n", + "print(f\"✓ Knowledge graph built\")\n", + "print(f\" Entities: {len(knowledge_graph.get('entities', []))}\")\n", + "print(f\" Relationships: {len(knowledge_graph.get('relationships', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 11: Analyze Knowledge Graph\n", + "\n", + "Analyze graph structure using GraphAnalyzer (centrality, communities, connectivity).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 11: Analyze knowledge graph using GraphAnalyzer\n", + "from semantica.kg import GraphAnalyzer\n", + "\n", + "graph_analyzer = GraphAnalyzer()\n", + "analysis = graph_analyzer.analyze_graph(knowledge_graph)\n", + "centrality = graph_analyzer.calculate_centrality(knowledge_graph, 'degree')\n", + "communities = graph_analyzer.detect_communities(knowledge_graph, algorithm='louvain')\n", + "connectivity = graph_analyzer.analyze_connectivity(knowledge_graph)\n", + "metrics = graph_analyzer.compute_metrics(knowledge_graph)\n", + "\n", + "top_entities = []\n", + "if centrality and 'rankings' in centrality:\n", + " top_entities = centrality['rankings'][:5]\n", + "\n", + "num_communities = len(communities.get('communities', [])) if isinstance(communities, dict) else 0\n", + "\n", + "print(f\"✓ Graph analysis complete\")\n", + "print(f\" Communities: {num_communities}\")\n", + "print(f\" Top entities: {len(top_entities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 12: Build Context Graph\n", + "\n", + "Build ContextGraph from knowledge graph for enhanced retrieval and GraphRAG.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 12: Build context graph for enhanced retrieval\n", + "from semantica.context import ContextGraph\n", + "\n", + "context_graph = ContextGraph(\n", + " extract_entities=True,\n", + " extract_relationships=True\n", + ")\n", + "\n", + "# Convert knowledge graph to context graph format\n", + "nodes = []\n", + "for entity in knowledge_graph.get('entities', []):\n", + " nodes.append({\n", + " \"id\": entity.get('id', entity.get('name', '')),\n", + " \"type\": entity.get('type', 'entity'),\n", + " \"properties\": {\n", + " \"content\": entity.get('name', ''),\n", + " \"confidence\": entity.get('confidence', 1.0),\n", + " **entity.get('metadata', {})\n", + " }\n", + " })\n", + "\n", + "edges = []\n", + "for rel in knowledge_graph.get('relationships', []):\n", + " edges.append({\n", + " \"source_id\": rel.get('source', ''),\n", + " \"target_id\": rel.get('target', ''),\n", + " \"type\": rel.get('predicate', 'related_to'),\n", + " \"weight\": rel.get('confidence', 1.0)\n", + " })\n", + "\n", + "node_count = context_graph.add_nodes(nodes)\n", + "edge_count = context_graph.add_edges(edges)\n", + "\n", + "print(f\"✓ Context graph built\")\n", + "print(f\" Nodes: {node_count}\")\n", + "print(f\" Edges: {edge_count}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 13: Context Retrieval\n", + "\n", + "Set up hybrid retrieval (vector + graph) using ContextRetriever for GraphRAG queries.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 13: Set up hybrid context retrieval and demonstrate GraphRAG\n", + "from semantica.vector_store import VectorStore\n", + "from semantica.context import ContextRetriever\n", + "\n", + "# Initialize VectorStore\n", + "vector_store = VectorStore(backend=\"faiss\")\n", + "vector_store.add(\n", + " texts=[parsed_doc[\"full_text\"]],\n", + " metadata=[{\"source\": \"earnings_call\", \"type\": \"transcript\"}]\n", + ")\n", + "\n", + "# Initialize ContextRetriever\n", + "context_retriever = ContextRetriever(\n", + " knowledge_graph=context_graph,\n", + " vector_store=vector_store,\n", + " hybrid_alpha=0.6,\n", + " use_graph_expansion=True,\n", + " max_expansion_hops=2\n", + ")\n", + "\n", + "# Retrieve context for financial queries\n", + "financial_queries = [\n", + " \"What was the company's revenue guidance?\",\n", + " \"What were the key financial metrics discussed?\"\n", + "]\n", + "\n", + "retrieved_contexts = []\n", + "for query in financial_queries:\n", + " results = context_retriever.retrieve(\n", + " query=query,\n", + " max_results=3,\n", + " min_relevance_score=0.2\n", + " )\n", + " retrieved_contexts.append({\n", + " \"query\": query,\n", + " \"results\": results,\n", + " \"count\": len(results)\n", + " })\n", + "\n", + "print(f\"✓ Hybrid retrieval configured\")\n", + "print(f\" Queries processed: {len(retrieved_contexts)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 14: Entity Linking\n", + "\n", + "Link entities across sources and assign URIs using EntityLinker.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 14: Link entities using EntityLinker\n", + "from semantica.context import EntityLinker\n", + "\n", + "entity_linker = EntityLinker(knowledge_graph=knowledge_graph)\n", + "\n", + "# Assign URIs to key entities\n", + "linked_entities = []\n", + "for entity in merged_entities[:10]:\n", + " entity_id = entity.get('id', entity.get('name', ''))\n", + " entity_name = entity.get('name', '')\n", + " entity_type = entity.get('type', 'UNKNOWN')\n", + " \n", + " uri = entity_linker.assign_uri(\n", + " entity_id=entity_id,\n", + " text=entity_name,\n", + " entity_type=entity_type\n", + " )\n", + " linked_entities.append({\n", + " \"entity_id\": entity_id,\n", + " \"name\": entity_name,\n", + " \"uri\": uri,\n", + " \"type\": entity_type\n", + " })\n", + "\n", + "# Build entity web\n", + "entity_web = entity_linker.build_entity_web()\n", + "\n", + "print(f\"✓ Entity linking complete\")\n", + "print(f\" Entities linked: {len(linked_entities)}\")\n", + "print(f\" Entity web nodes: {len(entity_web.get('nodes', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 15: Agent Memory\n", + "\n", + "Store and retrieve memories using AgentMemory with RAG integration.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 15: Store and retrieve memories using AgentMemory\n", + "from semantica.context import AgentMemory\n", + "\n", + "agent_memory = AgentMemory(\n", + " vector_store=vector_store,\n", + " knowledge_graph=knowledge_graph,\n", + " retention_days=30\n", + ")\n", + "\n", + "# Store earnings call memories\n", + "memory_ids = []\n", + "memory_contents = [\n", + " f\"Earnings call transcript: {parsed_doc['metadata'].get('title', 'Q1 2024')}\",\n", + " f\"Financial metrics extracted: {len(financial_metrics)} metrics\",\n", + " f\"Key entities identified: {len(merged_entities)} entities\"\n", + "]\n", + "\n", + "for content in memory_contents:\n", + " memory_id = agent_memory.store(\n", + " content=content,\n", + " metadata={\"source\": \"earnings_call\", \"type\": \"transcript_analysis\"},\n", + " extract_entities=True,\n", + " extract_relationships=True\n", + " )\n", + " memory_ids.append(memory_id)\n", + "\n", + "# Retrieve memories\n", + "financial_memories = agent_memory.retrieve(\n", + " query=\"financial metrics and earnings\",\n", + " max_results=5\n", + ")\n", + "\n", + "memory_stats = agent_memory.get_statistics()\n", + "\n", + "print(f\"✓ Agent memory configured\")\n", + "print(f\" Memories stored: {len(memory_ids)}\")\n", + "print(f\" Total memories: {memory_stats.get('total_memories', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 16: Agent Context\n", + "\n", + "Unified context management with AgentContext (auto-detects RAG vs GraphRAG).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 16: High-level context management with AgentContext\n", + "from semantica.context import AgentContext\n", + "\n", + "agent_context = AgentContext(\n", + " vector_store=vector_store,\n", + " knowledge_graph=context_graph,\n", + " use_graph_expansion=True,\n", + " max_expansion_hops=2,\n", + " hybrid_alpha=0.6,\n", + " retention_days=30\n", + ")\n", + "\n", + "# Store content with auto-extraction\n", + "memory_id = agent_context.store(\n", + " content=parsed_doc[\"full_text\"][:1000],\n", + " metadata={\"source\": \"earnings_call\", \"date\": \"2024-Q1\"},\n", + " extract_entities=True,\n", + " extract_relationships=True,\n", + " link_entities=True\n", + ")\n", + "\n", + "# Retrieve with auto-detected GraphRAG\n", + "graphrag_results = agent_context.retrieve(\n", + " query=\"What was discussed about revenue growth?\",\n", + " max_results=5,\n", + " expand_graph=True,\n", + " include_entities=True\n", + ")\n", + "\n", + "context_stats = agent_context.stats()\n", + "\n", + "print(f\"✓ AgentContext configured\")\n", + "print(f\" Memory stored: {memory_id}\")\n", + "print(f\" GraphRAG results: {len(graphrag_results)}\")\n", + "print(f\" Total memories: {context_stats.get('total_memories', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 17: Answer Generation\n", + "\n", + "Generate answers to financial questions using Groq LLM with retrieved context and knowledge graph.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 18: Export Results\n", + "\n", + "Export knowledge graph and analysis results to JSON and RDF formats.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 17: Generate answers using Groq LLM from semantica.llms module\n", + "financial_questions = [\n", + " \"What were the key financial metrics discussed in the earnings call?\",\n", + " \"What guidance did management provide for future quarters?\"\n", + "]\n", + "\n", + "generated_answers = []\n", + "for question in financial_questions:\n", + " # Retrieve relevant context\n", + " context_results = context_retriever.retrieve(\n", + " query=question,\n", + " max_results=3,\n", + " min_relevance_score=0.2\n", + " )\n", + " \n", + " # Build context from retrieved results\n", + " context_text = \"\\n\\n\".join([\n", + " f\"Context {i+1}: {result.get('content', result.get('text', ''))}\"\n", + " for i, result in enumerate(context_results[:3])\n", + " ])\n", + " \n", + " # Extract relevant entities\n", + " relevant_entities = [\n", + " entity.get('name', '') for entity in knowledge_graph.get('entities', [])[:10]\n", + " ]\n", + " entities_text = \", \".join(relevant_entities[:5]) if relevant_entities else \"N/A\"\n", + " \n", + " # Build prompt\n", + " prompt = f\"\"\"Based on the following earnings call transcript context and knowledge graph, answer the question.\n", + "\n", + "Context from transcript:\n", + "{context_text[:1000]}\n", + "\n", + "Key entities identified: {entities_text}\n", + "\n", + "Question: {question}\n", + "\n", + "Provide a comprehensive answer based on the context provided. If information is not available in the context, state that clearly.\n", + "\n", + "Answer:\"\"\"\n", + " \n", + " # Generate answer using Groq LLM\n", + " try:\n", + " answer = groq_llm.generate(\n", + " prompt,\n", + " temperature=0.7,\n", + " max_tokens=500\n", + " )\n", + " generated_answers.append({\n", + " \"question\": question,\n", + " \"answer\": answer,\n", + " \"context_sources\": len(context_results),\n", + " \"model\": groq_llm.model\n", + " })\n", + " except Exception as e:\n", + " generated_answers.append({\n", + " \"question\": question,\n", + " \"answer\": f\"Error generating answer: {str(e)}\",\n", + " \"context_sources\": len(context_results),\n", + " \"model\": groq_llm.model\n", + " })\n", + "\n", + "print(f\"✓ Answer generation complete using Groq LLM\")\n", + "print(f\" LLM Provider: Groq ({groq_llm.model})\")\n", + "print(f\" Questions answered: {len(generated_answers)}\")\n", + "if generated_answers:\n", + " print(f\" Sample question: '{generated_answers[0]['question']}'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Step 18: Export structured outputs including triplets\n", + "from semantica.export import JSONExporter, RDFExporter\n", + "\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "\n", + "# Export knowledge graph to JSON\n", + "kg_json = json_exporter.export(knowledge_graph, format=\"json\")\n", + "\n", + "# Export knowledge graph to RDF (Turtle format)\n", + "rdf_output = rdf_exporter.export_to_rdf(knowledge_graph, format=\"turtle\")\n", + "\n", + "# Create analysis summary\n", + "analysis_summary = {\n", + " \"financial_metrics\": financial_metrics,\n", + " \"extraction_stats\": {\n", + " \"entities\": len(entities),\n", + " \"relationships\": len(relationships),\n", + " \"triplets\": len(triplets),\n", + " \"provider\": f\"Groq LLM (semantica.llms module) - {groq_llm.model}\"\n", + " },\n", + " \"conflict_resolution\": {\n", + " \"conflicts_detected\": len(value_conflicts) + len(relationship_conflicts),\n", + " \"conflicts_resolved\": len(resolved_conflicts),\n", + " \"strategy\": \"voting\"\n", + " },\n", + " \"deduplication\": {\n", + " \"original_entities\": len(entity_dicts),\n", + " \"duplicates_detected\": len(duplicates),\n", + " \"merged_entities\": len(merged_entities),\n", + " \"strategy\": \"keep_most_complete\"\n", + " },\n", + " \"knowledge_graph\": {\n", + " \"entities\": len(knowledge_graph.get('entities', [])),\n", + " \"relationships\": len(knowledge_graph.get('relationships', []))\n", + " },\n", + " \"graph_analytics\": {\n", + " \"metrics\": metrics,\n", + " \"communities\": num_communities,\n", + " \"top_entities\": top_entities[:5] if top_entities else []\n", + " },\n", + " \"context_graph\": {\n", + " \"nodes\": len(context_graph.nodes),\n", + " \"edges\": len(context_graph.edges)\n", + " },\n", + " \"context_retrieval\": {\n", + " \"queries_processed\": len(retrieved_contexts),\n", + " \"total_results\": sum(c[\"count\"] for c in retrieved_contexts)\n", + " },\n", + " \"entity_linking\": {\n", + " \"entities_linked\": len(linked_entities),\n", + " \"entity_web_nodes\": len(entity_web.get('nodes', [])),\n", + " \"entity_web_edges\": len(entity_web.get('edges', []))\n", + " },\n", + " \"agent_memory\": {\n", + " \"memories_stored\": len(memory_ids),\n", + " \"total_memories\": memory_stats.get('total_memories', 0)\n", + " },\n", + " \"agent_context\": {\n", + " \"graphrag_results\": len(graphrag_results),\n", + " \"total_memories\": context_stats.get('total_memories', 0)\n", + " },\n", + " \"answer_generation\": {\n", + " \"questions_answered\": len(generated_answers),\n", + " \"llm_provider\": \"Groq\",\n", + " \"llm_model\": groq_llm.model,\n", + " \"answers\": [\n", + " {\n", + " \"question\": ans[\"question\"],\n", + " \"answer_length\": len(ans[\"answer\"]),\n", + " \"context_sources\": ans[\"context_sources\"]\n", + " }\n", + " for ans in generated_answers\n", + " ]\n", + " }\n", + "}\n", + "\n", + "print(f\"✓ Export complete\")\n", + "print(f\" Analysis summary: {len(analysis_summary)} sections\")\n", + "print(f\" Knowledge graph (JSON): {len(kg_json) if isinstance(kg_json, dict) else 0} items\")\n", + "print(f\" RDF (Turtle): {len(rdf_output)} characters\")\n", + "print(f\" LLM answers generated: {len(generated_answers)}\")\n", + "print(f\" LLM provider: Groq ({groq_llm.model})\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml index 3397dbb5..38ea6380 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "lxml>=4.9.0", "pypdf2>=2.10.0", "python-docx>=0.8.11", + "docling>=1.0.0", "openpyxl>=3.0.10", "pillow>=9.2.0", "librosa>=0.9.0", @@ -204,8 +205,11 @@ graph-falkordb = [ graph-all = [ "semantica[graph-neo4j,graph-falkordb]" ] +parse-docling = [ + "docling>=1.0.0" +] all = [ - "semantica[dev,viz,gpu,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all]" + "semantica[dev,viz,gpu,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all,parse-docling]" ] [project.scripts] diff --git a/semantica/parse/__init__.py b/semantica/parse/__init__.py index a4834648..213cb160 100644 --- a/semantica/parse/__init__.py +++ b/semantica/parse/__init__.py @@ -10,6 +10,7 @@ Algorithms Used: Document Parsing: - PDF Parsing: pdfplumber integration (pdfplumber.PDF()) for text extraction, PyPDF2.PdfReader() fallback, table extraction (pdfplumber.extract_tables()), image extraction, metadata extraction (title, author, dates via pdf.metadata), page-level processing (page iteration) - DOCX Parsing: python-docx integration (docx.Document()), paragraph extraction (document.paragraphs), table extraction (docx.table.Table), section/heading detection (paragraph.style), metadata extraction (core_properties), formatting extraction + - Docling Parsing: Docling integration (DocumentConverter.convert()) for enhanced table extraction and document structure understanding, supports PDF, DOCX, PPTX, XLSX, HTML, images, markdown/HTML/JSON export formats, OCR support (optional dependency) - PPTX Parsing: python-pptx integration (pptx.Presentation()), slide extraction (presentation.slides), shape extraction, notes extraction, metadata extraction - Excel Parsing: openpyxl integration (openpyxl.load_workbook()), pandas integration (pandas.read_excel()), sheet iteration, cell value extraction, formula extraction, metadata extraction - HTML Parsing: BeautifulSoup integration (BeautifulSoup(html, 'html.parser')), text extraction (soup.get_text()), link extraction (find_all('a')), metadata extraction (meta tags), structure analysis @@ -46,6 +47,7 @@ Media Parsing: Format-Specific Parsers: - PDFParser: pdfplumber.PDF() for text/tables, PyPDF2.PdfReader() fallback, page iteration (pdf.pages), metadata extraction - DOCXParser: docx.Document() for document loading, paragraph iteration, table extraction, core_properties access + - DoclingParser: DocumentConverter.convert() for multi-format parsing with enhanced table extraction, supports PDF/DOCX/PPTX/XLSX/HTML/images, markdown/HTML/JSON export (optional dependency) - PPTXParser: pptx.Presentation() for presentation loading, slide iteration, shape extraction - ExcelParser: openpyxl.load_workbook() for workbook loading, pandas.read_excel() for data extraction, sheet iteration - HTMLParser: BeautifulSoup() for HTML parsing, element traversal, metadata extraction @@ -77,6 +79,7 @@ Main Classes: - PDFParser: PDF document parser with text, table, and image extraction - DOCXParser: Word document parser with structure and metadata extraction - PPTXParser: PowerPoint parser with slide and notes extraction + - DoclingParser: Docling-based parser for enhanced table extraction (optional, requires docling package) - ExcelParser: Excel spreadsheet parser with multi-sheet support - HTMLParser: HTML document parser with metadata and link extraction - JSONParser: JSON data parser with nested structure handling @@ -164,6 +167,15 @@ from .methods import ( ) from .pdf_parser import PDFMetadata, PDFPage, PDFParser from .pptx_parser import PPTXData, PPTXParser, SlideContent + +# Try to import DoclingParser (optional dependency) +try: + from .docling_parser import DoclingParser, DoclingMetadata + DOCLING_AVAILABLE = True +except ImportError: + DOCLING_AVAILABLE = False + DoclingParser = None + DoclingMetadata = None from .registry import MethodRegistry, method_registry from .structured_data_parser import StructuredDataParser from .web_parser import HTMLContentParser, JavaScriptRenderer, WebParser @@ -238,3 +250,7 @@ __all__ = [ "get_parse_method", "list_available_methods", ] + +# Conditionally add DoclingParser to exports if available +if DOCLING_AVAILABLE: + __all__.extend(["DoclingParser", "DoclingMetadata"]) diff --git a/semantica/parse/docling_parser.py b/semantica/parse/docling_parser.py new file mode 100644 index 00000000..b281c196 --- /dev/null +++ b/semantica/parse/docling_parser.py @@ -0,0 +1,487 @@ +""" +Docling Document Parser Module + +This module handles document parsing using Docling for enhanced table extraction +and better document structure understanding across multiple formats (PDF, DOCX, PPTX, XLSX, HTML, images). + +Key Features: + - Multi-format document parsing (PDF, DOCX, PPTX, XLSX, HTML, images) + - Superior table extraction accuracy + - Enhanced document structure understanding + - Markdown, HTML, and JSON export formats + - Local execution support + - OCR support for scanned documents + +Main Classes: + - DoclingParser: Docling-based document parser + +Example Usage: + >>> from semantica.parse import DoclingParser + >>> parser = DoclingParser() + >>> result = parser.parse("document.pdf") + >>> text = parser.extract_text("document.pdf") + >>> tables = parser.extract_tables("document.pdf") + +Author: Semantica Contributors +License: MIT +""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from ..utils.exceptions import ProcessingError, ValidationError +from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker + +# Try to import docling, handle gracefully if not available +try: + from docling.document_converter import DocumentConverter + from docling.datamodel.base_models import InputFormat + from docling.datamodel.pipeline_options import PdfPipelineOptions + DOCLING_AVAILABLE = True +except ImportError: + DOCLING_AVAILABLE = False + DocumentConverter = None + InputFormat = None + PdfPipelineOptions = None + + +@dataclass +class DoclingMetadata: + """Document metadata representation from Docling.""" + + title: Optional[str] = None + author: Optional[str] = None + subject: Optional[str] = None + creator: Optional[str] = None + producer: Optional[str] = None + creation_date: Optional[str] = None + modification_date: Optional[str] = None + page_count: int = 0 + format: Optional[str] = None + + +class DoclingParser: + """Docling-based document parser for enhanced table extraction.""" + + def __init__(self, **config): + """ + Initialize Docling parser. + + Args: + **config: Parser configuration: + - export_format: Export format ("markdown", "html", "json") (default: "markdown") + - enable_ocr: Enable OCR for scanned documents (default: False) + - table_extraction_mode: Table extraction mode (default: "auto") + """ + if not DOCLING_AVAILABLE: + raise ImportError( + "Docling is not installed. Install it with: pip install docling" + ) + + self.logger = get_logger("docling_parser") + self.config = config + self.progress_tracker = get_progress_tracker() + # Ensure progress tracker is enabled + if not self.progress_tracker.enabled: + self.progress_tracker.enabled = True + + # Initialize DocumentConverter + export_format = config.get("export_format", "markdown") + enable_ocr = config.get("enable_ocr", False) + table_extraction_mode = config.get("table_extraction_mode", "auto") + + # Configure pipeline options + pipeline_options = PdfPipelineOptions() + if enable_ocr: + pipeline_options.do_ocr = True + + self.converter = DocumentConverter( + format_options={ + "markdown": {"table_format": table_extraction_mode}, + }, + pipeline_options=pipeline_options, + ) + self.export_format = export_format + + def parse(self, file_path: Union[str, Path], **options) -> Dict[str, Any]: + """ + Parse document using Docling. + + Args: + file_path: Path to document file (PDF, DOCX, PPTX, XLSX, HTML, images) + **options: Parsing options: + - extract_text: Whether to extract text (default: True) + - extract_tables: Whether to extract tables (default: True) + - extract_images: Whether to extract images (default: False) + - export_format: Export format ("markdown", "html", "json") (default: from config) + - pages: Specific page numbers to parse (None = all pages) - PDF only + + Returns: + dict: Parsed document data matching Semantica format + """ + file_path = Path(file_path) + + # Track document parsing + tracking_id = self.progress_tracker.start_tracking( + file=str(file_path), + module="parse", + submodule="DoclingParser", + message=f"Docling: {file_path.name}", + ) + + try: + if not file_path.exists(): + raise ValidationError(f"Document file not found: {file_path}") + + # Check if docling is available + if not DOCLING_AVAILABLE: + raise ImportError( + "Docling is not installed. Install it with: pip install docling" + ) + + # Determine export format + export_format = options.get("export_format", self.export_format) + + self.progress_tracker.update_tracking( + tracking_id, message=f"Converting document with Docling..." + ) + + # Convert document using Docling + result = self.converter.convert(str(file_path)) + + # Extract content based on export format + extract_text = options.get("extract_text", True) + extract_tables = options.get("extract_tables", True) + extract_images = options.get("extract_images", False) + + # Get document content + if export_format == "markdown": + full_text = result.document.export_to_markdown() + elif export_format == "html": + full_text = result.document.export_to_html() + elif export_format == "json": + # JSON export returns structured data + doc_dict = result.document.export_to_dict() + full_text = self._extract_text_from_dict(doc_dict) + else: + full_text = result.document.export_to_markdown() + + # Extract metadata + metadata = self._extract_metadata(result, file_path) + + # Extract tables + tables = [] + if extract_tables: + tables = self._extract_tables(result, export_format) + + # Extract pages (for PDF-like structure) + pages = self._extract_pages(result, options) + + # Extract images if requested + images = [] + if extract_images: + images = self._extract_images(result) + + self.progress_tracker.stop_tracking( + tracking_id, + status="completed", + message=f"Parsed document: {len(tables)} tables extracted", + ) + + return { + "metadata": metadata.__dict__, + "pages": pages, + "full_text": full_text if extract_text else "", + "tables": tables, + "images": images, + "total_pages": metadata.page_count, + "export_format": export_format, + } + + except ImportError: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message="Docling not installed" + ) + raise + except Exception as e: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message=str(e) + ) + self.logger.error(f"Failed to parse document with Docling {file_path}: {e}") + raise ProcessingError(f"Failed to parse document with Docling: {e}") + + def extract_text( + self, file_path: Union[str, Path], export_format: str = "markdown" + ) -> str: + """ + Extract text from document. + + Args: + file_path: Path to document file + export_format: Export format ("markdown", "html", "json") + + Returns: + str: Extracted text + """ + result = self.parse( + file_path, + extract_tables=False, + extract_images=False, + export_format=export_format, + ) + return result["full_text"] + + def extract_tables( + self, file_path: Union[str, Path], export_format: str = "markdown" + ) -> List[Dict[str, Any]]: + """ + Extract tables from document. + + Args: + file_path: Path to document file + export_format: Export format for table extraction + + Returns: + list: Extracted tables + """ + result = self.parse( + file_path, + extract_text=False, + extract_images=False, + export_format=export_format, + ) + return result["tables"] + + def _extract_text_from_dict(self, doc_dict: Dict[str, Any]) -> str: + """Extract text content from Docling dict structure.""" + text_parts = [] + + def extract_from_item(item: Dict[str, Any]): + if isinstance(item, dict): + if item.get("type") == "text": + text_parts.append(item.get("text", "")) + elif item.get("type") == "paragraph": + if "content" in item: + for content_item in item["content"]: + extract_from_item(content_item) + elif "content" in item: + for content_item in item["content"]: + extract_from_item(content_item) + + if isinstance(doc_dict, dict) and "content" in doc_dict: + for item in doc_dict["content"]: + extract_from_item(item) + + return "\n\n".join(text_parts) + + def _extract_tables( + self, result: Any, export_format: str + ) -> List[Dict[str, Any]]: + """Extract tables from Docling result.""" + tables = [] + + try: + # Try to get tables from document structure + doc_dict = result.document.export_to_dict() + + def find_tables(item: Dict[str, Any], page_num: int = 1): + if isinstance(item, dict): + if item.get("type") == "table": + table_data = self._convert_table_to_dict(item) + table_data["page_number"] = page_num + tables.append(table_data) + elif "content" in item: + # Check if this is a page + if item.get("type") == "page": + page_num = item.get("page", page_num) + for content_item in item.get("content", []): + find_tables(content_item, page_num) + + if isinstance(doc_dict, dict) and "content" in doc_dict: + for item in doc_dict["content"]: + find_tables(item) + + except Exception as e: + self.logger.warning(f"Could not extract tables from Docling result: {e}") + + return tables + + def _convert_table_to_dict(self, table_item: Dict[str, Any]) -> Dict[str, Any]: + """Convert Docling table structure to Semantica format.""" + table_data = { + "rows": [], + "row_count": 0, + "col_count": 0, + "data": [], + } + + try: + # Extract table rows + if "content" in table_item: + for row_item in table_item["content"]: + if row_item.get("type") == "table-row": + row_data = [] + if "content" in row_item: + for cell_item in row_item["content"]: + if cell_item.get("type") == "table-cell": + cell_text = "" + if "content" in cell_item: + for cell_content in cell_item["content"]: + if isinstance(cell_content, dict): + cell_text += cell_content.get("text", "") + elif isinstance(cell_content, str): + cell_text += cell_content + row_data.append(cell_text.strip()) + if row_data: + table_data["rows"].append(row_data) + table_data["data"].append(row_data) + + if table_data["rows"]: + table_data["row_count"] = len(table_data["rows"]) + table_data["col_count"] = ( + max(len(row) for row in table_data["rows"]) if table_data["rows"] else 0 + ) + + except Exception as e: + self.logger.warning(f"Error converting table: {e}") + + return table_data + + def _extract_pages(self, result: Any, options: Dict[str, Any]) -> List[Dict[str, Any]]: + """Extract pages from Docling result.""" + pages = [] + + try: + doc_dict = result.document.export_to_dict() + + def extract_page(page_item: Dict[str, Any], page_num: int): + page_text = "" + page_tables = [] + + if "content" in page_item: + for content_item in page_item["content"]: + if content_item.get("type") == "text": + page_text += content_item.get("text", "") + "\n" + elif content_item.get("type") == "table": + table_data = self._convert_table_to_dict(content_item) + table_data["page_number"] = page_num + page_tables.append(table_data) + + pages.append({ + "page_number": page_num, + "text": page_text.strip(), + "width": page_item.get("width", 0), + "height": page_item.get("height", 0), + "tables": [t for t in page_tables], + "images": [], + }) + + # Find pages in document structure + if isinstance(doc_dict, dict) and "content" in doc_dict: + page_num = 1 + for item in doc_dict["content"]: + if item.get("type") == "page": + extract_page(item, page_num) + page_num += 1 + elif "pages" in item: + # Handle paginated content + for page_item in item.get("pages", []): + extract_page(page_item, page_num) + page_num += 1 + + # If no pages found, create a single page from full content + if not pages: + full_text = result.document.export_to_markdown() + pages.append({ + "page_number": 1, + "text": full_text, + "width": 0, + "height": 0, + "tables": [], + "images": [], + }) + + except Exception as e: + self.logger.warning(f"Could not extract pages from Docling result: {e}") + # Fallback: create single page + try: + full_text = result.document.export_to_markdown() + pages.append({ + "page_number": 1, + "text": full_text, + "width": 0, + "height": 0, + "tables": [], + "images": [], + }) + except: + pass + + return pages + + def _extract_images(self, result: Any) -> List[Dict[str, Any]]: + """Extract images from Docling result.""" + images = [] + + try: + doc_dict = result.document.export_to_dict() + + def find_images(item: Dict[str, Any], page_num: int = 1): + if isinstance(item, dict): + if item.get("type") == "image": + img_data = { + "page_number": page_num, + "x0": item.get("bbox", {}).get("x0", 0) if "bbox" in item else 0, + "y0": item.get("bbox", {}).get("y0", 0) if "bbox" in item else 0, + "x1": item.get("bbox", {}).get("x1", 0) if "bbox" in item else 0, + "y1": item.get("bbox", {}).get("y1", 0) if "bbox" in item else 0, + "width": item.get("width", 0), + "height": item.get("height", 0), + } + images.append(img_data) + elif "content" in item: + if item.get("type") == "page": + page_num = item.get("page", page_num) + for content_item in item.get("content", []): + find_images(content_item, page_num) + + if isinstance(doc_dict, dict) and "content" in doc_dict: + for item in doc_dict["content"]: + find_images(item) + + except Exception as e: + self.logger.warning(f"Could not extract images from Docling result: {e}") + + return images + + def _extract_metadata(self, result: Any, file_path: Path) -> DoclingMetadata: + """Extract metadata from Docling result.""" + metadata = DoclingMetadata() + + try: + # Try to get metadata from document + doc_dict = result.document.export_to_dict() + + # Extract format + metadata.format = file_path.suffix.lower() + + # Try to extract page count + if isinstance(doc_dict, dict) and "content" in doc_dict: + page_count = 0 + for item in doc_dict["content"]: + if item.get("type") == "page": + page_count += 1 + metadata.page_count = page_count if page_count > 0 else 1 + + # Docling may not provide all metadata fields directly + # These would need to be extracted from the original document if available + + except Exception as e: + self.logger.warning(f"Could not extract metadata from Docling result: {e}") + metadata.page_count = 1 + metadata.format = file_path.suffix.lower() + + return metadata + diff --git a/semantica/parse/document_parser.py b/semantica/parse/document_parser.py index ec52d9ce..b8705ed8 100644 --- a/semantica/parse/document_parser.py +++ b/semantica/parse/document_parser.py @@ -38,6 +38,14 @@ from .docx_parser import DOCXParser from .html_parser import HTMLParser from .pdf_parser import PDFParser +# Try to import DoclingParser (optional dependency) +try: + from .docling_parser import DoclingParser + DOCLING_AVAILABLE = True +except ImportError: + DOCLING_AVAILABLE = False + DoclingParser = None + class DocumentParser: """ @@ -82,6 +90,14 @@ class DocumentParser: self.pdf_parser = PDFParser(**self.config.get("pdf", {})) self.docx_parser = DOCXParser(**self.config.get("docx", {})) self.html_parser = HTMLParser(**self.config.get("html", {})) + + # Initialize Docling parser if available (optional) + self.docling_parser = None + if DOCLING_AVAILABLE: + try: + self.docling_parser = DoclingParser(**self.config.get("docling", {})) + except Exception as e: + self.logger.warning(f"Could not initialize DoclingParser: {e}") # Supported formats self.supported_formats = { @@ -177,9 +193,16 @@ class DocumentParser: tracking_id, message=f"Parsing {file_type} document" ) + # Check if docling method is requested + method = options.get("method", "default") + use_docling = method == "docling" and self.docling_parser is not None + # Route to appropriate parser try: - if file_type == "pdf": + if use_docling: + # Use Docling for parsing (supports multiple formats) + result = self.docling_parser.parse(file_path, **options) + elif file_type == "pdf": result = self.pdf_parser.parse(file_path, **options) elif file_type == "docx": result = self.docx_parser.parse(file_path, **options) diff --git a/semantica/parse/methods.py b/semantica/parse/methods.py index af3d79a4..ef01caa5 100644 --- a/semantica/parse/methods.py +++ b/semantica/parse/methods.py @@ -13,6 +13,7 @@ Document Parsing: - "pdf": PDF-focused parsing - "docx": DOCX-focused parsing - "html": HTML-focused parsing + - "docling": Docling-based parsing for enhanced table extraction (requires docling package) Web Content Parsing: - "default": Default web parsing using WebParser @@ -161,6 +162,7 @@ def parse_document( - "pdf": PDF-focused parsing - "docx": DOCX-focused parsing - "html": HTML-focused parsing + - "docling": Docling-based parsing for enhanced table extraction (requires docling package) **kwargs: Additional options passed to DocumentParser - extract_text: Whether to extract text (default: True) - extract_tables: Whether to extract tables (default: True) @@ -195,6 +197,63 @@ def parse_document( raise +def parse_document_docling( + file_path: Union[str, Path], + file_type: Optional[str] = None, + **kwargs, +) -> Dict[str, Any]: + """ + Parse document using Docling (convenience function). + + This function uses Docling for enhanced table extraction and better document + structure understanding. Docling must be installed separately. + + Args: + file_path: Path to document file (PDF, DOCX, PPTX, XLSX, HTML, images) + file_type: Document type (auto-detected if None) + **kwargs: Additional options passed to DoclingParser: + - extract_text: Whether to extract text (default: True) + - extract_tables: Whether to extract tables (default: True) + - extract_images: Whether to extract images (default: False) + - export_format: Export format ("markdown", "html", "json") (default: "markdown") + - enable_ocr: Enable OCR for scanned documents (default: False) + + Returns: + dict: Parsed document data + + Examples: + >>> from semantica.parse.methods import parse_document_docling + >>> doc = parse_document_docling("document.pdf") + >>> tables = parse_document_docling("document.pdf", extract_tables=True) + """ + try: + from .docling_parser import DoclingParser + except ImportError: + raise ImportError( + "Docling is not installed. Install it with: pip install docling" + ) + + try: + config = parse_config.get_method_config("document") + config.update(kwargs) + + parser = DoclingParser(**config) + return parser.parse(file_path, **kwargs) + + except Exception as e: + logger.error(f"Failed to parse document with Docling: {e}") + raise + + +# Register Docling method +try: + from .docling_parser import DoclingParser + method_registry.register("document", "docling", parse_document_docling) +except ImportError: + # Docling not available, skip registration + pass + + def parse_web_content( content: Union[str, Path], content_type: str = "html",