mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Fix DoclingParser integration and align with docling API
- Fix import logic in __init__.py to properly export DoclingParser - Rewrite docling_parser.py to use docling's native API (direct attribute access) - Remove unsupported features (table_extraction_mode, invalid format_options) - Use doc.tables, doc.pictures, doc.pages directly instead of dict parsing - Update notebook with improved code and documentation - Add proper error handling for when docling is not available Fixes #138
This commit is contained in:
KaifAhmad1
@@ -96,35 +96,16 @@
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>Progress</th><th>ETA</th><th>Rate</th><th>Time</th></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DoclingParser</td><td>-</td><td>-</td><td>-</td><td>0.01s</td></tr></table></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🧠 Semantica is parsing: Docling: mda_space_q3_2025_press_release.pdf 🔄🔍 (0.0s) "
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "ImportError",
|
||||
"evalue": "[WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.",
|
||||
"evalue": "DoclingParser requires the 'docling' package to be installed.\n\nImport error: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.\n\nInstall it with: pip install docling",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mImportError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[3], line 22\u001b[0m\n\u001b[0;32m 19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m transcript_pdf\u001b[38;5;241m.\u001b[39mexists():\n\u001b[0;32m 20\u001b[0m transcript_pdf\u001b[38;5;241m.\u001b[39mwrite_bytes(requests\u001b[38;5;241m.\u001b[39mget(transcript_url)\u001b[38;5;241m.\u001b[39mcontent)\n\u001b[1;32m---> 22\u001b[0m press_release \u001b[38;5;241m=\u001b[39m \u001b[43mparser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpress_release_pdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 23\u001b[0m transcript \u001b[38;5;241m=\u001b[39m parser\u001b[38;5;241m.\u001b[39mparse(transcript_pdf)\n\u001b[0;32m 25\u001b[0m parsed_doc \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m 26\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# Press Release\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mpress_release[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m# Transcript\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mtranscript[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 27\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m\"\u001b[39m: press_release[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m+\u001b[39m transcript[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 33\u001b[0m }\n\u001b[0;32m 34\u001b[0m }\n",
|
||||
"File \u001b[1;32m~\\semantica\\semantica\\parse\\docling_parser.py:137\u001b[0m, in \u001b[0;36mDoclingParser.parse\u001b[1;34m(self, file_path, **options)\u001b[0m\n\u001b[0;32m 135\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m DOCLING_AVAILABLE:\n\u001b[0;32m 136\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m DOCLING_IMPORT_ERROR:\n\u001b[1;32m--> 137\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(DOCLING_IMPORT_ERROR)\n\u001b[0;32m 138\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 139\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDocling is not installed\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
|
||||
"\u001b[1;31mImportError\u001b[0m: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies."
|
||||
"Cell \u001b[1;32mIn[3], line 7\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msemantica\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mparse\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DoclingParser\n\u001b[0;32m 5\u001b[0m \u001b[38;5;66;03m# Initialize DoclingParser with default settings\u001b[39;00m\n\u001b[0;32m 6\u001b[0m \u001b[38;5;66;03m# Optional: configure with export_format=\"html\" or enable_ocr=True if needed\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m parser \u001b[38;5;241m=\u001b[39m \u001b[43mDoclingParser\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 9\u001b[0m \u001b[38;5;66;03m# PDF URLs for MDA Space Q3 2025 earnings documents\u001b[39;00m\n\u001b[0;32m 10\u001b[0m press_release_url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://filecache.investorroom.com/mr5ircnw_mda/677/MDA_Space_Ltd_Q3_2025_Press_Release_Nov_14_2025_FINAL.pdf\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
|
||||
"File \u001b[1;32m~\\semantica\\semantica\\parse\\__init__.py:198\u001b[0m, in \u001b[0;36mDoclingParser.__init__\u001b[1;34m(self, **config)\u001b[0m\n\u001b[0;32m 196\u001b[0m error_msg \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mImport error: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mimport_error_msg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 197\u001b[0m error_msg \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mInstall it with: pip install docling\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 198\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(error_msg)\n",
|
||||
"\u001b[1;31mImportError\u001b[0m: DoclingParser requires the 'docling' package to be installed.\n\nImport error: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.\n\nInstall it with: pip install docling"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -133,26 +114,33 @@
|
||||
"from pathlib import Path\n",
|
||||
"from semantica.parse import DoclingParser\n",
|
||||
"\n",
|
||||
"# Initialize DoclingParser with default settings\n",
|
||||
"# Optional: configure with export_format=\"html\" or enable_ocr=True if needed\n",
|
||||
"parser = DoclingParser()\n",
|
||||
"\n",
|
||||
"# PDF URLs for MDA Space Q3 2025 earnings documents\n",
|
||||
"press_release_url = \"https://filecache.investorroom.com/mr5ircnw_mda/677/MDA_Space_Ltd_Q3_2025_Press_Release_Nov_14_2025_FINAL.pdf\"\n",
|
||||
"transcript_url = \"https://filecache.investorroom.com/mr5ircnw_mda/681/MDA%20Space%20Ltd.%20Q3%202025%20Earnings%20Conference%20Call%20Transcript%20%28November%2014%202025%29.pdf\"\n",
|
||||
"\n",
|
||||
"# Setup download directory\n",
|
||||
"download_dir = Path(\"downloads\")\n",
|
||||
"download_dir.mkdir(exist_ok=True)\n",
|
||||
"\n",
|
||||
"press_release_pdf = download_dir / \"mda_space_q3_2025_press_release.pdf\"\n",
|
||||
"transcript_pdf = download_dir / \"mda_space_q3_2025_transcript.pdf\"\n",
|
||||
"\n",
|
||||
"# Download PDFs if they don't exist\n",
|
||||
"if not press_release_pdf.exists():\n",
|
||||
" press_release_pdf.write_bytes(requests.get(press_release_url).content)\n",
|
||||
"\n",
|
||||
"if not transcript_pdf.exists():\n",
|
||||
" transcript_pdf.write_bytes(requests.get(transcript_url).content)\n",
|
||||
"\n",
|
||||
"# Parse documents using DoclingParser\n",
|
||||
"press_release = parser.parse(press_release_pdf)\n",
|
||||
"transcript = parser.parse(transcript_pdf)\n",
|
||||
"\n",
|
||||
"# Combine parsed documents\n",
|
||||
"parsed_doc = {\n",
|
||||
" \"full_text\": f\"# Press Release\\n\\n{press_release['full_text']}\\n\\n# Transcript\\n\\n{transcript['full_text']}\",\n",
|
||||
" \"tables\": press_release['tables'] + transcript['tables'],\n",
|
||||
@@ -162,7 +150,9 @@
|
||||
" \"quarter\": \"Q3 2025\",\n",
|
||||
" \"date\": \"November 14, 2025\"\n",
|
||||
" }\n",
|
||||
"}\n"
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(f\"✓ Parsed {len(parsed_doc['tables'])} tables from {len(press_release['tables']) + len(transcript['tables'])} documents\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -169,12 +169,66 @@ from .pdf_parser import PDFMetadata, PDFPage, PDFParser
|
||||
from .pptx_parser import PPTXData, PPTXParser, SlideContent
|
||||
|
||||
# Try to import DoclingParser (optional dependency)
|
||||
# First check if docling_parser module can be imported and get DOCLING_AVAILABLE
|
||||
DOCLING_AVAILABLE = False
|
||||
DoclingParser = None
|
||||
DoclingMetadata = None
|
||||
DOCLING_IMPORT_ERROR = None
|
||||
|
||||
try:
|
||||
from .docling_parser import DoclingParser, DoclingMetadata, DOCLING_AVAILABLE
|
||||
except (ImportError, OSError):
|
||||
# Import the module first to check DOCLING_AVAILABLE
|
||||
from . import docling_parser
|
||||
DOCLING_AVAILABLE = getattr(docling_parser, 'DOCLING_AVAILABLE', False)
|
||||
DOCLING_IMPORT_ERROR = getattr(docling_parser, 'DOCLING_IMPORT_ERROR', None)
|
||||
|
||||
# Only import classes if docling is available
|
||||
if DOCLING_AVAILABLE:
|
||||
from .docling_parser import DoclingParser, DoclingMetadata
|
||||
else:
|
||||
# Create placeholder classes that raise helpful errors
|
||||
import_error_msg = DOCLING_IMPORT_ERROR
|
||||
|
||||
class DoclingParser:
|
||||
"""Placeholder for DoclingParser when docling is not available."""
|
||||
def __init__(self, **config):
|
||||
error_msg = "DoclingParser requires the 'docling' package to be installed and working."
|
||||
if import_error_msg:
|
||||
error_msg += f"\n\nError: {import_error_msg}"
|
||||
error_msg += "\n\nInstall it with: pip install docling"
|
||||
raise ImportError(error_msg)
|
||||
|
||||
class DoclingMetadata:
|
||||
"""Placeholder for DoclingMetadata when docling is not available."""
|
||||
def __init__(self, **kwargs):
|
||||
error_msg = "DoclingMetadata requires the 'docling' package to be installed and working."
|
||||
if import_error_msg:
|
||||
error_msg += f"\n\nError: {import_error_msg}"
|
||||
error_msg += "\n\nInstall it with: pip install docling"
|
||||
raise ImportError(error_msg)
|
||||
except (ImportError, OSError, AttributeError) as e:
|
||||
# If import fails, docling is not available
|
||||
DOCLING_AVAILABLE = False
|
||||
DoclingParser = None
|
||||
DoclingMetadata = None
|
||||
import_error_msg = str(e)
|
||||
DOCLING_IMPORT_ERROR = import_error_msg
|
||||
|
||||
# Create placeholder classes that raise helpful errors
|
||||
class DoclingParser:
|
||||
"""Placeholder for DoclingParser when docling is not available."""
|
||||
def __init__(self, **config):
|
||||
error_msg = "DoclingParser requires the 'docling' package to be installed and working."
|
||||
if import_error_msg:
|
||||
error_msg += f"\n\nError: {import_error_msg}"
|
||||
error_msg += "\n\nInstall it with: pip install docling"
|
||||
raise ImportError(error_msg)
|
||||
|
||||
class DoclingMetadata:
|
||||
"""Placeholder for DoclingMetadata when docling is not available."""
|
||||
def __init__(self, **kwargs):
|
||||
error_msg = "DoclingMetadata requires the 'docling' package to be installed and working."
|
||||
if import_error_msg:
|
||||
error_msg += f"\n\nError: {import_error_msg}"
|
||||
error_msg += "\n\nInstall it with: pip install docling"
|
||||
raise ImportError(error_msg)
|
||||
from .registry import MethodRegistry, method_registry
|
||||
from .structured_data_parser import StructuredDataParser
|
||||
from .web_parser import HTMLContentParser, JavaScriptRenderer, WebParser
|
||||
@@ -251,5 +305,5 @@ __all__ = [
|
||||
]
|
||||
|
||||
# Conditionally add DoclingParser to exports if available
|
||||
if DOCLING_AVAILABLE:
|
||||
__all__.extend(["DoclingParser", "DoclingMetadata"])
|
||||
# Always add to __all__ so import doesn't fail, but classes will be None if not available
|
||||
__all__.extend(["DoclingParser", "DoclingMetadata"])
|
||||
|
||||
+298
-152
@@ -48,9 +48,10 @@ DOCLING_IMPORT_ERROR = None
|
||||
DocumentConverter = None
|
||||
InputFormat = None
|
||||
PdfPipelineOptions = None
|
||||
PdfFormatOption = None
|
||||
|
||||
try:
|
||||
from docling.document_converter import DocumentConverter
|
||||
from docling.document_converter import DocumentConverter, PdfFormatOption
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.pipeline_options import PdfPipelineOptions
|
||||
DOCLING_AVAILABLE = True
|
||||
@@ -86,7 +87,7 @@ class DoclingParser:
|
||||
**config: Parser configuration:
|
||||
- export_format: Export format ("markdown", "html", "json") (default: "markdown")
|
||||
- enable_ocr: Enable OCR for scanned documents (default: False)
|
||||
- table_extraction_mode: Table extraction mode (default: "auto")
|
||||
Note: OCR is handled via PdfPipelineOptions if needed
|
||||
"""
|
||||
self.logger = get_logger("docling_parser")
|
||||
self.config = config
|
||||
@@ -98,7 +99,6 @@ class DoclingParser:
|
||||
# Store config for lazy initialization
|
||||
self.export_format = config.get("export_format", "markdown")
|
||||
self.enable_ocr = config.get("enable_ocr", False)
|
||||
self.table_extraction_mode = config.get("table_extraction_mode", "auto")
|
||||
self._converter = None
|
||||
|
||||
def parse(self, file_path: Union[str, Path], **options) -> Dict[str, Any]:
|
||||
@@ -152,11 +152,19 @@ class DoclingParser:
|
||||
|
||||
# Lazy initialization of converter
|
||||
if self._converter is None:
|
||||
self._converter = DocumentConverter(
|
||||
format_options={
|
||||
"markdown": {"table_format": self.table_extraction_mode},
|
||||
},
|
||||
)
|
||||
# Initialize with proper format_options if OCR is needed
|
||||
if self.enable_ocr:
|
||||
# Configure PDF pipeline options for OCR
|
||||
pipeline_options = PdfPipelineOptions()
|
||||
# OCR will be automatically used when needed
|
||||
self._converter = DocumentConverter(
|
||||
format_options={
|
||||
InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
|
||||
}
|
||||
)
|
||||
else:
|
||||
# Use default converter without special options
|
||||
self._converter = DocumentConverter()
|
||||
|
||||
# Determine export format
|
||||
export_format = options.get("export_format", self.export_format)
|
||||
@@ -375,129 +383,181 @@ class DoclingParser:
|
||||
def _extract_tables(
|
||||
self, result: Any, export_format: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Extract tables from Docling result."""
|
||||
"""Extract tables from Docling result using direct API access."""
|
||||
tables = []
|
||||
|
||||
try:
|
||||
# Try to get tables from document structure
|
||||
doc_dict = result.document.export_to_dict()
|
||||
|
||||
def find_tables(item: Dict[str, Any], page_num: int = 1):
|
||||
if isinstance(item, dict):
|
||||
if item.get("type") == "table":
|
||||
table_data = self._convert_table_to_dict(item)
|
||||
table_data["page_number"] = page_num
|
||||
tables.append(table_data)
|
||||
elif "content" in item:
|
||||
# Check if this is a page
|
||||
if item.get("type") == "page":
|
||||
page_num = item.get("page", page_num)
|
||||
for content_item in item.get("content", []):
|
||||
find_tables(content_item, page_num)
|
||||
|
||||
if isinstance(doc_dict, dict) and "content" in doc_dict:
|
||||
for item in doc_dict["content"]:
|
||||
find_tables(item)
|
||||
# Use direct API access to tables
|
||||
doc = result.document
|
||||
|
||||
# Iterate through tables directly
|
||||
for table in doc.tables:
|
||||
try:
|
||||
# Get table data
|
||||
table_data_obj = table.data
|
||||
|
||||
# Extract rows and columns
|
||||
rows = []
|
||||
|
||||
# Try to get table as markdown to parse rows (most reliable method)
|
||||
try:
|
||||
table_md = table.export_to_markdown(doc=doc)
|
||||
|
||||
# Parse markdown table into rows
|
||||
for line in table_md.strip().split('\n'):
|
||||
if '|' in line and not line.strip().startswith('|---'):
|
||||
# Parse markdown table row
|
||||
cells = [cell.strip() for cell in line.split('|')[1:-1]]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Could not extract table via markdown: {e}")
|
||||
|
||||
# If markdown parsing didn't work, try dataframe export (requires pandas)
|
||||
if not rows:
|
||||
try:
|
||||
import pandas as pd
|
||||
df = table.export_to_dataframe(doc=doc)
|
||||
rows = df.values.tolist()
|
||||
# Convert to strings
|
||||
rows = [[str(cell) for cell in row] for row in rows]
|
||||
except ImportError:
|
||||
self.logger.debug("Pandas not available for table extraction")
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Could not extract table via dataframe: {e}")
|
||||
|
||||
# If still no rows, create empty structure
|
||||
if not rows:
|
||||
rows = []
|
||||
|
||||
# Determine page number from table provenance
|
||||
page_number = 1
|
||||
if hasattr(table, 'prov') and table.prov:
|
||||
if hasattr(table.prov, 'page_no'):
|
||||
page_number = table.prov.page_no
|
||||
elif isinstance(table.prov, dict) and 'page_no' in table.prov:
|
||||
page_number = table.prov['page_no']
|
||||
|
||||
table_data = {
|
||||
"rows": rows,
|
||||
"row_count": len(rows),
|
||||
"col_count": max(len(row) for row in rows) if rows else 0,
|
||||
"data": rows,
|
||||
"page_number": page_number,
|
||||
}
|
||||
|
||||
tables.append(table_data)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Error extracting table: {e}")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not extract tables from Docling result: {e}")
|
||||
|
||||
return tables
|
||||
|
||||
def _convert_table_to_dict(self, table_item: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Convert Docling table structure to Semantica format."""
|
||||
table_data = {
|
||||
"rows": [],
|
||||
"row_count": 0,
|
||||
"col_count": 0,
|
||||
"data": [],
|
||||
}
|
||||
|
||||
try:
|
||||
# Extract table rows
|
||||
if "content" in table_item:
|
||||
for row_item in table_item["content"]:
|
||||
if row_item.get("type") == "table-row":
|
||||
row_data = []
|
||||
if "content" in row_item:
|
||||
for cell_item in row_item["content"]:
|
||||
if cell_item.get("type") == "table-cell":
|
||||
cell_text = ""
|
||||
if "content" in cell_item:
|
||||
for cell_content in cell_item["content"]:
|
||||
if isinstance(cell_content, dict):
|
||||
cell_text += cell_content.get("text", "")
|
||||
elif isinstance(cell_content, str):
|
||||
cell_text += cell_content
|
||||
row_data.append(cell_text.strip())
|
||||
if row_data:
|
||||
table_data["rows"].append(row_data)
|
||||
table_data["data"].append(row_data)
|
||||
|
||||
if table_data["rows"]:
|
||||
table_data["row_count"] = len(table_data["rows"])
|
||||
table_data["col_count"] = (
|
||||
max(len(row) for row in table_data["rows"]) if table_data["rows"] else 0
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Error converting table: {e}")
|
||||
|
||||
return table_data
|
||||
|
||||
def _extract_pages(self, result: Any, options: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Extract pages from Docling result."""
|
||||
"""Extract pages from Docling result using direct API access."""
|
||||
pages = []
|
||||
|
||||
try:
|
||||
doc_dict = result.document.export_to_dict()
|
||||
|
||||
def extract_page(page_item: Dict[str, Any], page_num: int):
|
||||
page_text = ""
|
||||
page_tables = []
|
||||
|
||||
if "content" in page_item:
|
||||
for content_item in page_item["content"]:
|
||||
if content_item.get("type") == "text":
|
||||
page_text += content_item.get("text", "") + "\n"
|
||||
elif content_item.get("type") == "table":
|
||||
table_data = self._convert_table_to_dict(content_item)
|
||||
table_data["page_number"] = page_num
|
||||
page_tables.append(table_data)
|
||||
|
||||
pages.append({
|
||||
"page_number": page_num,
|
||||
"text": page_text.strip(),
|
||||
"width": page_item.get("width", 0),
|
||||
"height": page_item.get("height", 0),
|
||||
"tables": [t for t in page_tables],
|
||||
"images": [],
|
||||
})
|
||||
|
||||
# Find pages in document structure
|
||||
if isinstance(doc_dict, dict) and "content" in doc_dict:
|
||||
page_num = 1
|
||||
for item in doc_dict["content"]:
|
||||
if item.get("type") == "page":
|
||||
extract_page(item, page_num)
|
||||
page_num += 1
|
||||
elif "pages" in item:
|
||||
# Handle paginated content
|
||||
for page_item in item.get("pages", []):
|
||||
extract_page(page_item, page_num)
|
||||
page_num += 1
|
||||
|
||||
# If no pages found, create a single page from full content
|
||||
doc = result.document
|
||||
|
||||
# Use direct API access to pages
|
||||
# DoclingDocument.pages is a dict-like object
|
||||
if hasattr(doc, 'pages') and doc.pages:
|
||||
for page_no, page in doc.pages.items():
|
||||
try:
|
||||
# Extract text from page - iterate through items on this page
|
||||
page_text_parts = []
|
||||
page_tables = []
|
||||
|
||||
# Iterate through document items to find those on this page
|
||||
for item, level in doc.iterate_items():
|
||||
# Check if item is on this page
|
||||
item_page = 1
|
||||
if hasattr(item, 'prov') and item.prov:
|
||||
if hasattr(item.prov, 'page_no'):
|
||||
item_page = item.prov.page_no
|
||||
elif isinstance(item.prov, dict) and 'page_no' in item.prov:
|
||||
item_page = item.prov['page_no']
|
||||
|
||||
if item_page == page_no:
|
||||
# Extract text from text items
|
||||
if hasattr(item, 'text'):
|
||||
page_text_parts.append(item.text)
|
||||
elif hasattr(item, 'export_to_markdown'):
|
||||
try:
|
||||
page_text_parts.append(item.export_to_markdown(doc=doc))
|
||||
except:
|
||||
pass
|
||||
|
||||
# Extract tables on this page
|
||||
from docling_core.types.doc import TableItem
|
||||
if isinstance(item, TableItem):
|
||||
# Get table data for this page
|
||||
table_data = self._extract_single_table(item, doc, page_no)
|
||||
if table_data:
|
||||
page_tables.append(table_data)
|
||||
|
||||
# Get page dimensions
|
||||
width = 0
|
||||
height = 0
|
||||
if hasattr(page, 'size'):
|
||||
if hasattr(page.size, 'width'):
|
||||
width = page.size.width
|
||||
if hasattr(page.size, 'height'):
|
||||
height = page.size.height
|
||||
|
||||
pages.append({
|
||||
"page_number": page_no,
|
||||
"text": "\n".join(page_text_parts).strip(),
|
||||
"width": width,
|
||||
"height": height,
|
||||
"tables": page_tables,
|
||||
"images": [],
|
||||
})
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Error extracting page {page_no}: {e}")
|
||||
continue
|
||||
|
||||
# Fallback: if no pages found, use ConversionResult.pages or create single page
|
||||
if not pages:
|
||||
full_text = result.document.export_to_markdown()
|
||||
pages.append({
|
||||
"page_number": 1,
|
||||
"text": full_text,
|
||||
"width": 0,
|
||||
"height": 0,
|
||||
"tables": [],
|
||||
"images": [],
|
||||
})
|
||||
# Try to use result.pages (list of Page objects)
|
||||
if hasattr(result, 'pages') and result.pages:
|
||||
for page_obj in result.pages:
|
||||
pages.append({
|
||||
"page_number": getattr(page_obj, 'page_no', len(pages) + 1),
|
||||
"text": "",
|
||||
"width": getattr(page_obj.size, 'width', 0) if hasattr(page_obj, 'size') else 0,
|
||||
"height": getattr(page_obj.size, 'height', 0) if hasattr(page_obj, 'size') else 0,
|
||||
"tables": [],
|
||||
"images": [],
|
||||
})
|
||||
|
||||
# If still no pages, create a single page from full content
|
||||
if not pages:
|
||||
try:
|
||||
full_text = doc.export_to_markdown()
|
||||
pages.append({
|
||||
"page_number": 1,
|
||||
"text": full_text,
|
||||
"width": 0,
|
||||
"height": 0,
|
||||
"tables": [],
|
||||
"images": [],
|
||||
})
|
||||
except:
|
||||
pages.append({
|
||||
"page_number": 1,
|
||||
"text": "",
|
||||
"width": 0,
|
||||
"height": 0,
|
||||
"tables": [],
|
||||
"images": [],
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not extract pages from Docling result: {e}")
|
||||
@@ -513,39 +573,123 @@ class DoclingParser:
|
||||
"images": [],
|
||||
})
|
||||
except:
|
||||
pass
|
||||
pages.append({
|
||||
"page_number": 1,
|
||||
"text": "",
|
||||
"width": 0,
|
||||
"height": 0,
|
||||
"tables": [],
|
||||
"images": [],
|
||||
})
|
||||
|
||||
return pages
|
||||
|
||||
def _extract_single_table(self, table_item: Any, doc: Any, page_no: int) -> Optional[Dict[str, Any]]:
|
||||
"""Extract a single table item to dict format."""
|
||||
try:
|
||||
# Get table data
|
||||
table_data_obj = table_item.data
|
||||
|
||||
# Extract rows
|
||||
rows = []
|
||||
try:
|
||||
# Try markdown export first (most reliable)
|
||||
table_md = table_item.export_to_markdown(doc=doc)
|
||||
for line in table_md.strip().split('\n'):
|
||||
if '|' in line and not line.strip().startswith('|---'):
|
||||
cells = [cell.strip() for cell in line.split('|')[1:-1]]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Could not extract table via markdown: {e}")
|
||||
# Fallback: try dataframe export (requires pandas)
|
||||
try:
|
||||
import pandas as pd
|
||||
df = table_item.export_to_dataframe(doc=doc)
|
||||
rows = df.values.tolist()
|
||||
rows = [[str(cell) for cell in row] for row in rows]
|
||||
except ImportError:
|
||||
self.logger.debug("Pandas not available for table extraction")
|
||||
except Exception as e2:
|
||||
self.logger.debug(f"Could not extract table via dataframe: {e2}")
|
||||
|
||||
if rows:
|
||||
return {
|
||||
"rows": rows,
|
||||
"row_count": len(rows),
|
||||
"col_count": max(len(row) for row in rows) if rows else 0,
|
||||
"data": rows,
|
||||
"page_number": page_no,
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Error extracting single table: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def _extract_images(self, result: Any) -> List[Dict[str, Any]]:
|
||||
"""Extract images from Docling result."""
|
||||
"""Extract images/pictures from Docling result using direct API access."""
|
||||
images = []
|
||||
|
||||
try:
|
||||
doc_dict = result.document.export_to_dict()
|
||||
|
||||
def find_images(item: Dict[str, Any], page_num: int = 1):
|
||||
if isinstance(item, dict):
|
||||
if item.get("type") == "image":
|
||||
doc = result.document
|
||||
|
||||
# Use direct API access to pictures
|
||||
# DoclingDocument.pictures is iterable
|
||||
if hasattr(doc, 'pictures') and doc.pictures:
|
||||
for picture in doc.pictures:
|
||||
try:
|
||||
# Get page number from provenance
|
||||
page_number = 1
|
||||
if hasattr(picture, 'prov') and picture.prov:
|
||||
if hasattr(picture.prov, 'page_no'):
|
||||
page_number = picture.prov.page_no
|
||||
elif isinstance(picture.prov, dict) and 'page_no' in picture.prov:
|
||||
page_number = picture.prov['page_no']
|
||||
|
||||
# Get bounding box
|
||||
x0, y0, x1, y1 = 0, 0, 0, 0
|
||||
width, height = 0, 0
|
||||
|
||||
if hasattr(picture, 'bbox'):
|
||||
bbox = picture.bbox
|
||||
if hasattr(bbox, 'x0'):
|
||||
x0 = bbox.x0
|
||||
if hasattr(bbox, 'y0'):
|
||||
y0 = bbox.y0
|
||||
if hasattr(bbox, 'x1'):
|
||||
x1 = bbox.x1
|
||||
if hasattr(bbox, 'y1'):
|
||||
y1 = bbox.y1
|
||||
elif isinstance(picture.bbox, dict):
|
||||
x0 = picture.bbox.get('x0', 0)
|
||||
y0 = picture.bbox.get('y0', 0)
|
||||
x1 = picture.bbox.get('x1', 0)
|
||||
y1 = picture.bbox.get('y1', 0)
|
||||
|
||||
# Get dimensions
|
||||
if hasattr(picture, 'size'):
|
||||
if hasattr(picture.size, 'width'):
|
||||
width = picture.size.width
|
||||
if hasattr(picture.size, 'height'):
|
||||
height = picture.size.height
|
||||
elif hasattr(picture, 'width'):
|
||||
width = picture.width
|
||||
elif hasattr(picture, 'height'):
|
||||
height = picture.height
|
||||
|
||||
img_data = {
|
||||
"page_number": page_num,
|
||||
"x0": item.get("bbox", {}).get("x0", 0) if "bbox" in item else 0,
|
||||
"y0": item.get("bbox", {}).get("y0", 0) if "bbox" in item else 0,
|
||||
"x1": item.get("bbox", {}).get("x1", 0) if "bbox" in item else 0,
|
||||
"y1": item.get("bbox", {}).get("y1", 0) if "bbox" in item else 0,
|
||||
"width": item.get("width", 0),
|
||||
"height": item.get("height", 0),
|
||||
"page_number": page_number,
|
||||
"x0": x0,
|
||||
"y0": y0,
|
||||
"x1": x1,
|
||||
"y1": y1,
|
||||
"width": width,
|
||||
"height": height,
|
||||
}
|
||||
images.append(img_data)
|
||||
elif "content" in item:
|
||||
if item.get("type") == "page":
|
||||
page_num = item.get("page", page_num)
|
||||
for content_item in item.get("content", []):
|
||||
find_images(content_item, page_num)
|
||||
|
||||
if isinstance(doc_dict, dict) and "content" in doc_dict:
|
||||
for item in doc_dict["content"]:
|
||||
find_images(item)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Error extracting picture: {e}")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not extract images from Docling result: {e}")
|
||||
@@ -553,31 +697,33 @@ class DoclingParser:
|
||||
return images
|
||||
|
||||
def _extract_metadata(self, result: Any, file_path: Path) -> DoclingMetadata:
|
||||
"""Extract metadata from Docling result."""
|
||||
"""Extract metadata from Docling result using direct API access."""
|
||||
metadata = DoclingMetadata()
|
||||
|
||||
try:
|
||||
# Try to get metadata from document
|
||||
doc_dict = result.document.export_to_dict()
|
||||
|
||||
doc = result.document
|
||||
|
||||
# Extract format
|
||||
metadata.format = file_path.suffix.lower()
|
||||
metadata.format = file_path.suffix.lower().lstrip('.')
|
||||
|
||||
# Try to extract page count
|
||||
if isinstance(doc_dict, dict) and "content" in doc_dict:
|
||||
page_count = 0
|
||||
for item in doc_dict["content"]:
|
||||
if item.get("type") == "page":
|
||||
page_count += 1
|
||||
metadata.page_count = page_count if page_count > 0 else 1
|
||||
# Extract page count using direct API
|
||||
if hasattr(doc, 'pages') and doc.pages:
|
||||
metadata.page_count = len(doc.pages)
|
||||
elif hasattr(result, 'pages') and result.pages:
|
||||
metadata.page_count = len(result.pages)
|
||||
else:
|
||||
metadata.page_count = 1
|
||||
|
||||
# Try to extract other metadata if available
|
||||
# Docling may not provide all metadata fields directly
|
||||
# These would need to be extracted from the original document if available
|
||||
if hasattr(doc, 'name'):
|
||||
metadata.title = doc.name
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not extract metadata from Docling result: {e}")
|
||||
metadata.page_count = 1
|
||||
metadata.format = file_path.suffix.lower()
|
||||
metadata.format = file_path.suffix.lower().lstrip('.')
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
Reference in New Issue
Block a user