mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
feat(parse): add progress tracking to DoclingParser and update earnings call notebook
- Add 8-stage progress tracking (0-100%) with ETA to DoclingParser - Update earnings call analysis notebook with MDA Space Q3 2025 example - Simplify notebook code structure - Add real-time progress visibility for PDF parsing Closes #133
This commit is contained in:
KaifAhmad1
@@ -87,44 +87,73 @@
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>Progress</th><th>ETA</th><th>Rate</th><th>Time</th></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DoclingParser</td><td>-</td><td>-</td><td>-</td><td>0.01s</td></tr></table></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✓ Document parsed: Q1 2024 Earnings Call\n",
|
||||
" Text length: 409 characters\n"
|
||||
"🧠 Semantica is parsing: Docling: mda_space_q3_2025_press_release.pdf 🔄🔍 (0.0s) "
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "ImportError",
|
||||
"evalue": "[WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mImportError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[3], line 22\u001b[0m\n\u001b[0;32m 19\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m transcript_pdf\u001b[38;5;241m.\u001b[39mexists():\n\u001b[0;32m 20\u001b[0m transcript_pdf\u001b[38;5;241m.\u001b[39mwrite_bytes(requests\u001b[38;5;241m.\u001b[39mget(transcript_url)\u001b[38;5;241m.\u001b[39mcontent)\n\u001b[1;32m---> 22\u001b[0m press_release \u001b[38;5;241m=\u001b[39m \u001b[43mparser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpress_release_pdf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 23\u001b[0m transcript \u001b[38;5;241m=\u001b[39m parser\u001b[38;5;241m.\u001b[39mparse(transcript_pdf)\n\u001b[0;32m 25\u001b[0m parsed_doc \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m 26\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# Press Release\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mpress_release[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m# Transcript\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mtranscript[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 27\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m\"\u001b[39m: press_release[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m+\u001b[39m transcript[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtables\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 33\u001b[0m }\n\u001b[0;32m 34\u001b[0m }\n",
|
||||
"File \u001b[1;32m~\\semantica\\semantica\\parse\\docling_parser.py:137\u001b[0m, in \u001b[0;36mDoclingParser.parse\u001b[1;34m(self, file_path, **options)\u001b[0m\n\u001b[0;32m 135\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m DOCLING_AVAILABLE:\n\u001b[0;32m 136\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m DOCLING_IMPORT_ERROR:\n\u001b[1;32m--> 137\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(DOCLING_IMPORT_ERROR)\n\u001b[0;32m 138\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 139\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDocling is not installed\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
|
||||
"\u001b[1;31mImportError\u001b[0m: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies."
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"from pathlib import Path\n",
|
||||
"from semantica.parse import DoclingParser\n",
|
||||
"\n",
|
||||
"parser = DoclingParser(\n",
|
||||
" export_format=\"markdown\",\n",
|
||||
" enable_ocr=False,\n",
|
||||
" table_extraction_mode=\"auto\"\n",
|
||||
")\n",
|
||||
"parser = DoclingParser()\n",
|
||||
"\n",
|
||||
"press_release_url = \"https://filecache.investorroom.com/mr5ircnw_mda/677/MDA_Space_Ltd_Q3_2025_Press_Release_Nov_14_2025_FINAL.pdf\"\n",
|
||||
"transcript_url = \"https://filecache.investorroom.com/mr5ircnw_mda/681/MDA%20Space%20Ltd.%20Q3%202025%20Earnings%20Conference%20Call%20Transcript%20%28November%2014%202025%29.pdf\"\n",
|
||||
"\n",
|
||||
"download_dir = Path(\"downloads\")\n",
|
||||
"download_dir.mkdir(exist_ok=True)\n",
|
||||
"\n",
|
||||
"press_release_pdf = download_dir / \"mda_space_q3_2025_press_release.pdf\"\n",
|
||||
"transcript_pdf = download_dir / \"mda_space_q3_2025_transcript.pdf\"\n",
|
||||
"\n",
|
||||
"if not press_release_pdf.exists():\n",
|
||||
" press_release_pdf.write_bytes(requests.get(press_release_url).content)\n",
|
||||
"\n",
|
||||
"if not transcript_pdf.exists():\n",
|
||||
" transcript_pdf.write_bytes(requests.get(transcript_url).content)\n",
|
||||
"\n",
|
||||
"press_release = parser.parse(press_release_pdf)\n",
|
||||
"transcript = parser.parse(transcript_pdf)\n",
|
||||
"\n",
|
||||
"# Example parsed document (replace with actual PDF parsing)\n",
|
||||
"parsed_doc = {\n",
|
||||
" \"full_text\": \"\"\"Q1 2024 Earnings Call Transcript\n",
|
||||
"\n",
|
||||
"Company: TechCorp Inc.\n",
|
||||
"Date: January 25, 2024\n",
|
||||
"\n",
|
||||
"Prepared Remarks:\n",
|
||||
"Our revenue for Q1 2024 was $2.5 billion, representing 15% year-over-year growth. \n",
|
||||
"EPS was $1.25 per share. We expect Q2 revenue to be between $2.6 and $2.8 billion.\n",
|
||||
"\n",
|
||||
"Q&A Session:\n",
|
||||
"Analyst: What's your guidance for the full year?\n",
|
||||
"CEO: We're maintaining our full-year guidance of $10.5 to $11 billion in revenue.\"\"\",\n",
|
||||
" \"tables\": [],\n",
|
||||
" \"metadata\": {\"title\": \"Q1 2024 Earnings Call\"}\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(f\"✓ Document parsed: {parsed_doc['metadata'].get('title', 'Unknown')}\")\n",
|
||||
"print(f\" Text length: {len(parsed_doc['full_text'])} characters\")\n"
|
||||
" \"full_text\": f\"# Press Release\\n\\n{press_release['full_text']}\\n\\n# Transcript\\n\\n{transcript['full_text']}\",\n",
|
||||
" \"tables\": press_release['tables'] + transcript['tables'],\n",
|
||||
" \"metadata\": {\n",
|
||||
" \"title\": \"MDA Space Ltd. Q3 2025 Earnings Analysis\",\n",
|
||||
" \"company\": \"MDA Space Ltd.\",\n",
|
||||
" \"quarter\": \"Q3 2025\",\n",
|
||||
" \"date\": \"November 14, 2025\"\n",
|
||||
" }\n",
|
||||
"}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -138,26 +167,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>Progress</th><th>ETA</th><th>Rate</th><th>Time</th></tr><tr><td>✅</td><td>Semantica is normalizing</td><td>🔧 normalize</td><td>TextNormalizer</td><td>100.0%</td><td>-</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>NERExtractor</td><td>100.0%</td><td>-</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>RelationExtractor</td><td>100.0%</td><td>-</td><td>-</td><td>0.00s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>TripletExtractor</td><td>100.0% (1/1)</td><td>-</td><td>65.4/s</td><td>0.02s</td></tr></table></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🧠 Normalizing text 🔄🔧 (0.0s) ✓ Text normalized: 409 characters\n"
|
||||
"🧠 Semantica is parsing: Docling: mda_space_q3_2025_press_release.pdf 🔄🔍 (0.0s) | 🧠 Normalizing text 🔄🔧 (0.0s) ✓ Text normalized: 0 characters\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -186,16 +203,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Method llm failed: groq provider not available\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
@@ -236,23 +246,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🧠 Normalizing text 🔄🔧 (0.0s) | 🧠 Semantica is extracting: Extracting named entities from text 🔄🎯 (0.0s) "
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Method llm failed: groq provider not available\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
@@ -289,7 +285,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -334,14 +330,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🧠 Semantica is extracting: Extracting named entities from text 🔄🎯 (0.0s) | 🧠 Semantica is extracting: Extracting triplets using llm... (1/1, remaining: 0 methods) |███████████████| 100.0% [1/1] 🔄🎯 (88.5/s)"
|
||||
"🧠 Normalizing text 🔄🔧 (0.0s) | 🧠 Semantica is extracting: Extracting triplets using llm... (1/1, remaining: 0 methods) |███████████████| 100.0% [1/1] 🔄🎯 (61.7/s)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -32,6 +32,7 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
@@ -137,6 +138,14 @@ class DoclingParser:
|
||||
else:
|
||||
raise ImportError("Docling is not installed")
|
||||
|
||||
# Stage 1: Initialization (0-10%)
|
||||
self.progress_tracker.update_progress(
|
||||
tracking_id,
|
||||
processed=1,
|
||||
total=10,
|
||||
message="Initializing Docling converter..."
|
||||
)
|
||||
|
||||
# Lazy initialization of converter
|
||||
if self._converter is None:
|
||||
self._converter = DocumentConverter(
|
||||
@@ -148,18 +157,44 @@ class DoclingParser:
|
||||
# Determine export format
|
||||
export_format = options.get("export_format", self.export_format)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Converting document with Docling..."
|
||||
# Stage 2: Document conversion (10-70%) - This is the longest step
|
||||
# Note: This is a blocking operation, but we'll update progress after it completes
|
||||
self.progress_tracker.update_progress(
|
||||
tracking_id,
|
||||
processed=2,
|
||||
total=10,
|
||||
message=f"Converting document with Docling (this may take a while for large PDFs)..."
|
||||
)
|
||||
|
||||
# Convert document using Docling
|
||||
# Convert document using Docling (blocking operation)
|
||||
# Store start time for ETA calculation
|
||||
conversion_start = time.time()
|
||||
result = self._converter.convert(str(file_path))
|
||||
conversion_elapsed = time.time() - conversion_start
|
||||
|
||||
# Stage 3: Document conversion complete (70%)
|
||||
# Update progress immediately after conversion completes with timing info
|
||||
self.progress_tracker.update_progress(
|
||||
tracking_id,
|
||||
processed=7,
|
||||
total=10,
|
||||
message=f"Document conversion complete ({conversion_elapsed:.1f}s), extracting content..."
|
||||
)
|
||||
|
||||
# Extract content based on export format
|
||||
extract_text = options.get("extract_text", True)
|
||||
extract_tables = options.get("extract_tables", True)
|
||||
extract_images = options.get("extract_images", False)
|
||||
|
||||
# Stage 4: Text extraction (70-80%)
|
||||
if extract_text:
|
||||
self.progress_tracker.update_progress(
|
||||
tracking_id,
|
||||
processed=8,
|
||||
total=10,
|
||||
message=f"Extracting text content ({export_format} format)..."
|
||||
)
|
||||
|
||||
# Get document content
|
||||
if export_format == "markdown":
|
||||
full_text = result.document.export_to_markdown()
|
||||
@@ -172,20 +207,44 @@ class DoclingParser:
|
||||
else:
|
||||
full_text = result.document.export_to_markdown()
|
||||
|
||||
# Extract metadata
|
||||
# Stage 5: Metadata extraction (80-85%)
|
||||
self.progress_tracker.update_progress(
|
||||
tracking_id,
|
||||
processed=8,
|
||||
total=10,
|
||||
message="Extracting document metadata..."
|
||||
)
|
||||
metadata = self._extract_metadata(result, file_path)
|
||||
|
||||
# Extract tables
|
||||
# Stage 6: Table extraction (85-95%)
|
||||
tables = []
|
||||
if extract_tables:
|
||||
self.progress_tracker.update_progress(
|
||||
tracking_id,
|
||||
processed=9,
|
||||
total=10,
|
||||
message="Extracting tables from document..."
|
||||
)
|
||||
tables = self._extract_tables(result, export_format)
|
||||
|
||||
# Extract pages (for PDF-like structure)
|
||||
# Stage 7: Page extraction (95-98%)
|
||||
self.progress_tracker.update_progress(
|
||||
tracking_id,
|
||||
processed=9,
|
||||
total=10,
|
||||
message="Extracting page structure..."
|
||||
)
|
||||
pages = self._extract_pages(result, options)
|
||||
|
||||
# Extract images if requested
|
||||
# Stage 8: Image extraction (98-100%)
|
||||
images = []
|
||||
if extract_images:
|
||||
self.progress_tracker.update_progress(
|
||||
tracking_id,
|
||||
processed=9,
|
||||
total=10,
|
||||
message="Extracting images from document..."
|
||||
)
|
||||
images = self._extract_images(result)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
|
||||
Reference in New Issue
Block a user