diff --git a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb index 8ae47061..9e1464a9 100644 --- a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb +++ b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb @@ -96,35 +96,16 @@ "execution_count": 3, "metadata": {}, "outputs": [ - { - "data": { - "text/html": [ - "

🧠 Semantica - 📊 Current Progress

StatusActionModuleSubmoduleProgressETARateTime
❌Semantica is parsing🔍 parseDoclingParser---0.01s
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🧠 Semantica is parsing: Docling: mda_space_q3_2025_press_release.pdf 🔄🔍 (0.0s) " - ] - }, { "ename": "ImportError", - "evalue": "[WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.", + "evalue": "DoclingParser requires the 'docling' package to be installed.\n\nImport error: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.\n\nInstall it with: pip install docling", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mImportError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[3], line 22\u001b[0m\n\u001b[0;32m 19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m transcript_pdf\u001b[38;5;241m.\u001b[39mexists():\n\u001b[0;32m 20\u001b[0m transcript_pdf\u001b[38;5;241m.\u001b[39mwrite_bytes(requests\u001b[38;5;241m.\u001b[39mget(transcript_url)\u001b[38;5;241m.\u001b[39mcontent)\n\u001b[1;32m---> 22\u001b[0m press_release \u001b[38;5;241m=\u001b[39m \u001b[43mparser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpress_release_pdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 23\u001b[0m transcript \u001b[38;5;241m=\u001b[39m parser\u001b[38;5;241m.\u001b[39mparse(transcript_pdf)\n\u001b[0;32m 25\u001b[0m parsed_doc \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m 26\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# Press Release\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mpress_release[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m# Transcript\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mtranscript[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 27\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m\"\u001b[39m: press_release[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m+\u001b[39m transcript[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 33\u001b[0m }\n\u001b[0;32m 34\u001b[0m }\n", - "File \u001b[1;32m~\\semantica\\semantica\\parse\\docling_parser.py:137\u001b[0m, in \u001b[0;36mDoclingParser.parse\u001b[1;34m(self, file_path, **options)\u001b[0m\n\u001b[0;32m 135\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m DOCLING_AVAILABLE:\n\u001b[0;32m 136\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m DOCLING_IMPORT_ERROR:\n\u001b[1;32m--> 137\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(DOCLING_IMPORT_ERROR)\n\u001b[0;32m 138\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 139\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDocling is not installed\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "\u001b[1;31mImportError\u001b[0m: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies." + "Cell \u001b[1;32mIn[3], line 7\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msemantica\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mparse\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DoclingParser\n\u001b[0;32m 5\u001b[0m \u001b[38;5;66;03m# Initialize DoclingParser with default settings\u001b[39;00m\n\u001b[0;32m 6\u001b[0m \u001b[38;5;66;03m# Optional: configure with export_format=\"html\" or enable_ocr=True if needed\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m parser \u001b[38;5;241m=\u001b[39m \u001b[43mDoclingParser\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 9\u001b[0m \u001b[38;5;66;03m# PDF URLs for MDA Space Q3 2025 earnings documents\u001b[39;00m\n\u001b[0;32m 10\u001b[0m press_release_url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://filecache.investorroom.com/mr5ircnw_mda/677/MDA_Space_Ltd_Q3_2025_Press_Release_Nov_14_2025_FINAL.pdf\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[1;32m~\\semantica\\semantica\\parse\\__init__.py:198\u001b[0m, in \u001b[0;36mDoclingParser.__init__\u001b[1;34m(self, **config)\u001b[0m\n\u001b[0;32m 196\u001b[0m error_msg \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mImport error: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mimport_error_msg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 197\u001b[0m error_msg \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mInstall it with: pip install docling\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 198\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(error_msg)\n", + "\u001b[1;31mImportError\u001b[0m: DoclingParser requires the 'docling' package to be installed.\n\nImport error: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.\n\nInstall it with: pip install docling" ] } ], @@ -133,26 +114,33 @@ "from pathlib import Path\n", "from semantica.parse import DoclingParser\n", "\n", + "# Initialize DoclingParser with default settings\n", + "# Optional: configure with export_format=\"html\" or enable_ocr=True if needed\n", "parser = DoclingParser()\n", "\n", + "# PDF URLs for MDA Space Q3 2025 earnings documents\n", "press_release_url = \"https://filecache.investorroom.com/mr5ircnw_mda/677/MDA_Space_Ltd_Q3_2025_Press_Release_Nov_14_2025_FINAL.pdf\"\n", "transcript_url = \"https://filecache.investorroom.com/mr5ircnw_mda/681/MDA%20Space%20Ltd.%20Q3%202025%20Earnings%20Conference%20Call%20Transcript%20%28November%2014%202025%29.pdf\"\n", "\n", + "# Setup download directory\n", "download_dir = Path(\"downloads\")\n", "download_dir.mkdir(exist_ok=True)\n", "\n", "press_release_pdf = download_dir / \"mda_space_q3_2025_press_release.pdf\"\n", "transcript_pdf = download_dir / \"mda_space_q3_2025_transcript.pdf\"\n", "\n", + "# Download PDFs if they don't exist\n", "if not press_release_pdf.exists():\n", " press_release_pdf.write_bytes(requests.get(press_release_url).content)\n", "\n", "if not transcript_pdf.exists():\n", " transcript_pdf.write_bytes(requests.get(transcript_url).content)\n", "\n", + "# Parse documents using DoclingParser\n", "press_release = parser.parse(press_release_pdf)\n", "transcript = parser.parse(transcript_pdf)\n", "\n", + "# Combine parsed documents\n", "parsed_doc = {\n", " \"full_text\": f\"# Press Release\\n\\n{press_release['full_text']}\\n\\n# Transcript\\n\\n{transcript['full_text']}\",\n", " \"tables\": press_release['tables'] + transcript['tables'],\n", @@ -162,7 +150,9 @@ " \"quarter\": \"Q3 2025\",\n", " \"date\": \"November 14, 2025\"\n", " }\n", - "}\n" + "}\n", + "\n", + "print(f\"✓ Parsed {len(parsed_doc['tables'])} tables from {len(press_release['tables']) + len(transcript['tables'])} documents\")\n" ] }, { diff --git a/semantica/parse/__init__.py b/semantica/parse/__init__.py index a7a7eb9c..5091f562 100644 --- a/semantica/parse/__init__.py +++ b/semantica/parse/__init__.py @@ -169,12 +169,66 @@ from .pdf_parser import PDFMetadata, PDFPage, PDFParser from .pptx_parser import PPTXData, PPTXParser, SlideContent # Try to import DoclingParser (optional dependency) +# First check if docling_parser module can be imported and get DOCLING_AVAILABLE +DOCLING_AVAILABLE = False +DoclingParser = None +DoclingMetadata = None +DOCLING_IMPORT_ERROR = None + try: - from .docling_parser import DoclingParser, DoclingMetadata, DOCLING_AVAILABLE -except (ImportError, OSError): + # Import the module first to check DOCLING_AVAILABLE + from . import docling_parser + DOCLING_AVAILABLE = getattr(docling_parser, 'DOCLING_AVAILABLE', False) + DOCLING_IMPORT_ERROR = getattr(docling_parser, 'DOCLING_IMPORT_ERROR', None) + + # Only import classes if docling is available + if DOCLING_AVAILABLE: + from .docling_parser import DoclingParser, DoclingMetadata + else: + # Create placeholder classes that raise helpful errors + import_error_msg = DOCLING_IMPORT_ERROR + + class DoclingParser: + """Placeholder for DoclingParser when docling is not available.""" + def __init__(self, **config): + error_msg = "DoclingParser requires the 'docling' package to be installed and working." + if import_error_msg: + error_msg += f"\n\nError: {import_error_msg}" + error_msg += "\n\nInstall it with: pip install docling" + raise ImportError(error_msg) + + class DoclingMetadata: + """Placeholder for DoclingMetadata when docling is not available.""" + def __init__(self, **kwargs): + error_msg = "DoclingMetadata requires the 'docling' package to be installed and working." + if import_error_msg: + error_msg += f"\n\nError: {import_error_msg}" + error_msg += "\n\nInstall it with: pip install docling" + raise ImportError(error_msg) +except (ImportError, OSError, AttributeError) as e: + # If import fails, docling is not available DOCLING_AVAILABLE = False - DoclingParser = None - DoclingMetadata = None + import_error_msg = str(e) + DOCLING_IMPORT_ERROR = import_error_msg + + # Create placeholder classes that raise helpful errors + class DoclingParser: + """Placeholder for DoclingParser when docling is not available.""" + def __init__(self, **config): + error_msg = "DoclingParser requires the 'docling' package to be installed and working." + if import_error_msg: + error_msg += f"\n\nError: {import_error_msg}" + error_msg += "\n\nInstall it with: pip install docling" + raise ImportError(error_msg) + + class DoclingMetadata: + """Placeholder for DoclingMetadata when docling is not available.""" + def __init__(self, **kwargs): + error_msg = "DoclingMetadata requires the 'docling' package to be installed and working." + if import_error_msg: + error_msg += f"\n\nError: {import_error_msg}" + error_msg += "\n\nInstall it with: pip install docling" + raise ImportError(error_msg) from .registry import MethodRegistry, method_registry from .structured_data_parser import StructuredDataParser from .web_parser import HTMLContentParser, JavaScriptRenderer, WebParser @@ -251,5 +305,5 @@ __all__ = [ ] # Conditionally add DoclingParser to exports if available -if DOCLING_AVAILABLE: - __all__.extend(["DoclingParser", "DoclingMetadata"]) +# Always add to __all__ so import doesn't fail, but classes will be None if not available +__all__.extend(["DoclingParser", "DoclingMetadata"]) diff --git a/semantica/parse/docling_parser.py b/semantica/parse/docling_parser.py index 6a456996..7ebb896a 100644 --- a/semantica/parse/docling_parser.py +++ b/semantica/parse/docling_parser.py @@ -48,9 +48,10 @@ DOCLING_IMPORT_ERROR = None DocumentConverter = None InputFormat = None PdfPipelineOptions = None +PdfFormatOption = None try: - from docling.document_converter import DocumentConverter + from docling.document_converter import DocumentConverter, PdfFormatOption from docling.datamodel.base_models import InputFormat from docling.datamodel.pipeline_options import PdfPipelineOptions DOCLING_AVAILABLE = True @@ -86,7 +87,7 @@ class DoclingParser: **config: Parser configuration: - export_format: Export format ("markdown", "html", "json") (default: "markdown") - enable_ocr: Enable OCR for scanned documents (default: False) - - table_extraction_mode: Table extraction mode (default: "auto") + Note: OCR is handled via PdfPipelineOptions if needed """ self.logger = get_logger("docling_parser") self.config = config @@ -98,7 +99,6 @@ class DoclingParser: # Store config for lazy initialization self.export_format = config.get("export_format", "markdown") self.enable_ocr = config.get("enable_ocr", False) - self.table_extraction_mode = config.get("table_extraction_mode", "auto") self._converter = None def parse(self, file_path: Union[str, Path], **options) -> Dict[str, Any]: @@ -152,11 +152,19 @@ class DoclingParser: # Lazy initialization of converter if self._converter is None: - self._converter = DocumentConverter( - format_options={ - "markdown": {"table_format": self.table_extraction_mode}, - }, - ) + # Initialize with proper format_options if OCR is needed + if self.enable_ocr: + # Configure PDF pipeline options for OCR + pipeline_options = PdfPipelineOptions() + # OCR will be automatically used when needed + self._converter = DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options) + } + ) + else: + # Use default converter without special options + self._converter = DocumentConverter() # Determine export format export_format = options.get("export_format", self.export_format) @@ -375,129 +383,181 @@ class DoclingParser: def _extract_tables( self, result: Any, export_format: str ) -> List[Dict[str, Any]]: - """Extract tables from Docling result.""" + """Extract tables from Docling result using direct API access.""" tables = [] try: - # Try to get tables from document structure - doc_dict = result.document.export_to_dict() - - def find_tables(item: Dict[str, Any], page_num: int = 1): - if isinstance(item, dict): - if item.get("type") == "table": - table_data = self._convert_table_to_dict(item) - table_data["page_number"] = page_num - tables.append(table_data) - elif "content" in item: - # Check if this is a page - if item.get("type") == "page": - page_num = item.get("page", page_num) - for content_item in item.get("content", []): - find_tables(content_item, page_num) - - if isinstance(doc_dict, dict) and "content" in doc_dict: - for item in doc_dict["content"]: - find_tables(item) + # Use direct API access to tables + doc = result.document + + # Iterate through tables directly + for table in doc.tables: + try: + # Get table data + table_data_obj = table.data + + # Extract rows and columns + rows = [] + + # Try to get table as markdown to parse rows (most reliable method) + try: + table_md = table.export_to_markdown(doc=doc) + + # Parse markdown table into rows + for line in table_md.strip().split('\n'): + if '|' in line and not line.strip().startswith('|---'): + # Parse markdown table row + cells = [cell.strip() for cell in line.split('|')[1:-1]] + if cells: + rows.append(cells) + except Exception as e: + self.logger.debug(f"Could not extract table via markdown: {e}") + + # If markdown parsing didn't work, try dataframe export (requires pandas) + if not rows: + try: + import pandas as pd + df = table.export_to_dataframe(doc=doc) + rows = df.values.tolist() + # Convert to strings + rows = [[str(cell) for cell in row] for row in rows] + except ImportError: + self.logger.debug("Pandas not available for table extraction") + except Exception as e: + self.logger.debug(f"Could not extract table via dataframe: {e}") + + # If still no rows, create empty structure + if not rows: + rows = [] + + # Determine page number from table provenance + page_number = 1 + if hasattr(table, 'prov') and table.prov: + if hasattr(table.prov, 'page_no'): + page_number = table.prov.page_no + elif isinstance(table.prov, dict) and 'page_no' in table.prov: + page_number = table.prov['page_no'] + + table_data = { + "rows": rows, + "row_count": len(rows), + "col_count": max(len(row) for row in rows) if rows else 0, + "data": rows, + "page_number": page_number, + } + + tables.append(table_data) + + except Exception as e: + self.logger.warning(f"Error extracting table: {e}") + continue except Exception as e: self.logger.warning(f"Could not extract tables from Docling result: {e}") return tables - def _convert_table_to_dict(self, table_item: Dict[str, Any]) -> Dict[str, Any]: - """Convert Docling table structure to Semantica format.""" - table_data = { - "rows": [], - "row_count": 0, - "col_count": 0, - "data": [], - } - - try: - # Extract table rows - if "content" in table_item: - for row_item in table_item["content"]: - if row_item.get("type") == "table-row": - row_data = [] - if "content" in row_item: - for cell_item in row_item["content"]: - if cell_item.get("type") == "table-cell": - cell_text = "" - if "content" in cell_item: - for cell_content in cell_item["content"]: - if isinstance(cell_content, dict): - cell_text += cell_content.get("text", "") - elif isinstance(cell_content, str): - cell_text += cell_content - row_data.append(cell_text.strip()) - if row_data: - table_data["rows"].append(row_data) - table_data["data"].append(row_data) - - if table_data["rows"]: - table_data["row_count"] = len(table_data["rows"]) - table_data["col_count"] = ( - max(len(row) for row in table_data["rows"]) if table_data["rows"] else 0 - ) - - except Exception as e: - self.logger.warning(f"Error converting table: {e}") - - return table_data def _extract_pages(self, result: Any, options: Dict[str, Any]) -> List[Dict[str, Any]]: - """Extract pages from Docling result.""" + """Extract pages from Docling result using direct API access.""" pages = [] try: - doc_dict = result.document.export_to_dict() - - def extract_page(page_item: Dict[str, Any], page_num: int): - page_text = "" - page_tables = [] - - if "content" in page_item: - for content_item in page_item["content"]: - if content_item.get("type") == "text": - page_text += content_item.get("text", "") + "\n" - elif content_item.get("type") == "table": - table_data = self._convert_table_to_dict(content_item) - table_data["page_number"] = page_num - page_tables.append(table_data) - - pages.append({ - "page_number": page_num, - "text": page_text.strip(), - "width": page_item.get("width", 0), - "height": page_item.get("height", 0), - "tables": [t for t in page_tables], - "images": [], - }) - - # Find pages in document structure - if isinstance(doc_dict, dict) and "content" in doc_dict: - page_num = 1 - for item in doc_dict["content"]: - if item.get("type") == "page": - extract_page(item, page_num) - page_num += 1 - elif "pages" in item: - # Handle paginated content - for page_item in item.get("pages", []): - extract_page(page_item, page_num) - page_num += 1 - - # If no pages found, create a single page from full content + doc = result.document + + # Use direct API access to pages + # DoclingDocument.pages is a dict-like object + if hasattr(doc, 'pages') and doc.pages: + for page_no, page in doc.pages.items(): + try: + # Extract text from page - iterate through items on this page + page_text_parts = [] + page_tables = [] + + # Iterate through document items to find those on this page + for item, level in doc.iterate_items(): + # Check if item is on this page + item_page = 1 + if hasattr(item, 'prov') and item.prov: + if hasattr(item.prov, 'page_no'): + item_page = item.prov.page_no + elif isinstance(item.prov, dict) and 'page_no' in item.prov: + item_page = item.prov['page_no'] + + if item_page == page_no: + # Extract text from text items + if hasattr(item, 'text'): + page_text_parts.append(item.text) + elif hasattr(item, 'export_to_markdown'): + try: + page_text_parts.append(item.export_to_markdown(doc=doc)) + except: + pass + + # Extract tables on this page + from docling_core.types.doc import TableItem + if isinstance(item, TableItem): + # Get table data for this page + table_data = self._extract_single_table(item, doc, page_no) + if table_data: + page_tables.append(table_data) + + # Get page dimensions + width = 0 + height = 0 + if hasattr(page, 'size'): + if hasattr(page.size, 'width'): + width = page.size.width + if hasattr(page.size, 'height'): + height = page.size.height + + pages.append({ + "page_number": page_no, + "text": "\n".join(page_text_parts).strip(), + "width": width, + "height": height, + "tables": page_tables, + "images": [], + }) + except Exception as e: + self.logger.warning(f"Error extracting page {page_no}: {e}") + continue + + # Fallback: if no pages found, use ConversionResult.pages or create single page if not pages: - full_text = result.document.export_to_markdown() - pages.append({ - "page_number": 1, - "text": full_text, - "width": 0, - "height": 0, - "tables": [], - "images": [], - }) + # Try to use result.pages (list of Page objects) + if hasattr(result, 'pages') and result.pages: + for page_obj in result.pages: + pages.append({ + "page_number": getattr(page_obj, 'page_no', len(pages) + 1), + "text": "", + "width": getattr(page_obj.size, 'width', 0) if hasattr(page_obj, 'size') else 0, + "height": getattr(page_obj.size, 'height', 0) if hasattr(page_obj, 'size') else 0, + "tables": [], + "images": [], + }) + + # If still no pages, create a single page from full content + if not pages: + try: + full_text = doc.export_to_markdown() + pages.append({ + "page_number": 1, + "text": full_text, + "width": 0, + "height": 0, + "tables": [], + "images": [], + }) + except: + pages.append({ + "page_number": 1, + "text": "", + "width": 0, + "height": 0, + "tables": [], + "images": [], + }) except Exception as e: self.logger.warning(f"Could not extract pages from Docling result: {e}") @@ -513,39 +573,123 @@ class DoclingParser: "images": [], }) except: - pass + pages.append({ + "page_number": 1, + "text": "", + "width": 0, + "height": 0, + "tables": [], + "images": [], + }) return pages + + def _extract_single_table(self, table_item: Any, doc: Any, page_no: int) -> Optional[Dict[str, Any]]: + """Extract a single table item to dict format.""" + try: + # Get table data + table_data_obj = table_item.data + + # Extract rows + rows = [] + try: + # Try markdown export first (most reliable) + table_md = table_item.export_to_markdown(doc=doc) + for line in table_md.strip().split('\n'): + if '|' in line and not line.strip().startswith('|---'): + cells = [cell.strip() for cell in line.split('|')[1:-1]] + if cells: + rows.append(cells) + except Exception as e: + self.logger.debug(f"Could not extract table via markdown: {e}") + # Fallback: try dataframe export (requires pandas) + try: + import pandas as pd + df = table_item.export_to_dataframe(doc=doc) + rows = df.values.tolist() + rows = [[str(cell) for cell in row] for row in rows] + except ImportError: + self.logger.debug("Pandas not available for table extraction") + except Exception as e2: + self.logger.debug(f"Could not extract table via dataframe: {e2}") + + if rows: + return { + "rows": rows, + "row_count": len(rows), + "col_count": max(len(row) for row in rows) if rows else 0, + "data": rows, + "page_number": page_no, + } + except Exception as e: + self.logger.warning(f"Error extracting single table: {e}") + + return None def _extract_images(self, result: Any) -> List[Dict[str, Any]]: - """Extract images from Docling result.""" + """Extract images/pictures from Docling result using direct API access.""" images = [] try: - doc_dict = result.document.export_to_dict() - - def find_images(item: Dict[str, Any], page_num: int = 1): - if isinstance(item, dict): - if item.get("type") == "image": + doc = result.document + + # Use direct API access to pictures + # DoclingDocument.pictures is iterable + if hasattr(doc, 'pictures') and doc.pictures: + for picture in doc.pictures: + try: + # Get page number from provenance + page_number = 1 + if hasattr(picture, 'prov') and picture.prov: + if hasattr(picture.prov, 'page_no'): + page_number = picture.prov.page_no + elif isinstance(picture.prov, dict) and 'page_no' in picture.prov: + page_number = picture.prov['page_no'] + + # Get bounding box + x0, y0, x1, y1 = 0, 0, 0, 0 + width, height = 0, 0 + + if hasattr(picture, 'bbox'): + bbox = picture.bbox + if hasattr(bbox, 'x0'): + x0 = bbox.x0 + if hasattr(bbox, 'y0'): + y0 = bbox.y0 + if hasattr(bbox, 'x1'): + x1 = bbox.x1 + if hasattr(bbox, 'y1'): + y1 = bbox.y1 + elif isinstance(picture.bbox, dict): + x0 = picture.bbox.get('x0', 0) + y0 = picture.bbox.get('y0', 0) + x1 = picture.bbox.get('x1', 0) + y1 = picture.bbox.get('y1', 0) + + # Get dimensions + if hasattr(picture, 'size'): + if hasattr(picture.size, 'width'): + width = picture.size.width + if hasattr(picture.size, 'height'): + height = picture.size.height + elif hasattr(picture, 'width'): + width = picture.width + elif hasattr(picture, 'height'): + height = picture.height + img_data = { - "page_number": page_num, - "x0": item.get("bbox", {}).get("x0", 0) if "bbox" in item else 0, - "y0": item.get("bbox", {}).get("y0", 0) if "bbox" in item else 0, - "x1": item.get("bbox", {}).get("x1", 0) if "bbox" in item else 0, - "y1": item.get("bbox", {}).get("y1", 0) if "bbox" in item else 0, - "width": item.get("width", 0), - "height": item.get("height", 0), + "page_number": page_number, + "x0": x0, + "y0": y0, + "x1": x1, + "y1": y1, + "width": width, + "height": height, } images.append(img_data) - elif "content" in item: - if item.get("type") == "page": - page_num = item.get("page", page_num) - for content_item in item.get("content", []): - find_images(content_item, page_num) - - if isinstance(doc_dict, dict) and "content" in doc_dict: - for item in doc_dict["content"]: - find_images(item) + except Exception as e: + self.logger.warning(f"Error extracting picture: {e}") + continue except Exception as e: self.logger.warning(f"Could not extract images from Docling result: {e}") @@ -553,31 +697,33 @@ class DoclingParser: return images def _extract_metadata(self, result: Any, file_path: Path) -> DoclingMetadata: - """Extract metadata from Docling result.""" + """Extract metadata from Docling result using direct API access.""" metadata = DoclingMetadata() try: - # Try to get metadata from document - doc_dict = result.document.export_to_dict() - + doc = result.document + # Extract format - metadata.format = file_path.suffix.lower() + metadata.format = file_path.suffix.lower().lstrip('.') - # Try to extract page count - if isinstance(doc_dict, dict) and "content" in doc_dict: - page_count = 0 - for item in doc_dict["content"]: - if item.get("type") == "page": - page_count += 1 - metadata.page_count = page_count if page_count > 0 else 1 + # Extract page count using direct API + if hasattr(doc, 'pages') and doc.pages: + metadata.page_count = len(doc.pages) + elif hasattr(result, 'pages') and result.pages: + metadata.page_count = len(result.pages) + else: + metadata.page_count = 1 + # Try to extract other metadata if available # Docling may not provide all metadata fields directly # These would need to be extracted from the original document if available + if hasattr(doc, 'name'): + metadata.title = doc.name except Exception as e: self.logger.warning(f"Could not extract metadata from Docling result: {e}") metadata.page_count = 1 - metadata.format = file_path.suffix.lower() + metadata.format = file_path.suffix.lower().lstrip('.') return metadata