mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Fix: Handle OSError for optional dependencies and make DoclingParser standalone
- Add safe_import utility in semantica/utils/helpers.py for graceful optional dependency handling - Update all optional imports (spacy, docling, etc.) to handle OSError (Windows DLL issues) - Make DoclingParser standalone with docling as core dependency - Remove DoclingParser integration from DocumentParser - Implement lazy initialization for DoclingParser (fails on parse(), not init()) - Fix DocumentConverter initialization (remove unsupported pipeline_options parameter) - Preserve original error messages without modification - Update semantic_extract, split, parse, embeddings, vector_store, visualization modules - Fix OSError handling across entire codebase for Windows compatibility - Update 60+ files with proper optional dependency handling
This commit is contained in:
KaifAhmad1
@@ -43,37 +43,19 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!pip install -qU semantica \n"
|
||||
"!pip install -qU semantica docling \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "OSError",
|
||||
"evalue": "[WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mOSError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[2], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;66;03m# Initialize Groq LLM provider\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msemantica\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mllms\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Groq\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mos\u001b[39;00m\n\u001b[0;32m 5\u001b[0m GROQ_API_KEY \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgsk_7BADli5sBUZ4NFBmeIreWGdyb3FYinqEphDW4FUZ0Lg4jV7WXr85\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\llms\\__init__.py:40\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;124;03mLLM Providers Module\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 37\u001b[0m \u001b[38;5;124;03mLicense: MIT\u001b[39;00m\n\u001b[0;32m 38\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m---> 40\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgroq\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Groq\n\u001b[0;32m 41\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mopenai\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m OpenAI\n\u001b[0;32m 42\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhuggingface\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m HuggingFaceLLM\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\llms\\groq.py:9\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;124;03mGroq LLM Provider\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \n\u001b[0;32m 4\u001b[0m \u001b[38;5;124;03mWrapper for Groq API provider with clean interface.\u001b[39;00m\n\u001b[0;32m 5\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, Optional\n\u001b[1;32m----> 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msemantic_extract\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mproviders\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m GroqProvider\n\u001b[0;32m 10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ProcessingError\n\u001b[0;32m 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlogging\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_logger\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\semantic_extract\\__init__.py:51\u001b[0m\n\u001b[0;32m 48\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, List, Optional, Union\n\u001b[0;32m 50\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Config, config\n\u001b[1;32m---> 51\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcoreference_resolver\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 52\u001b[0m CoreferenceChain,\n\u001b[0;32m 53\u001b[0m CoreferenceChainBuilder,\n\u001b[0;32m 54\u001b[0m CoreferenceResolver,\n\u001b[0;32m 55\u001b[0m EntityCoreferenceDetector,\n\u001b[0;32m 56\u001b[0m Mention,\n\u001b[0;32m 57\u001b[0m PronounResolver,\n\u001b[0;32m 58\u001b[0m )\n\u001b[0;32m 59\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mevent_detector\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 60\u001b[0m Event,\n\u001b[0;32m 61\u001b[0m EventClassifier,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 64\u001b[0m TemporalEventProcessor,\n\u001b[0;32m 65\u001b[0m )\n\u001b[0;32m 66\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mextraction_validator\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ExtractionValidator, ValidationResult\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\semantic_extract\\coreference_resolver.py:67\u001b[0m\n\u001b[0;32m 65\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlogging\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_logger\n\u001b[0;32m 66\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprogress_tracker\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_progress_tracker\n\u001b[1;32m---> 67\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mner_extractor\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Entity\n\u001b[0;32m 70\u001b[0m \u001b[38;5;129m@dataclass\u001b[39m\n\u001b[0;32m 71\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mMention\u001b[39;00m:\n\u001b[0;32m 72\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Mention representation.\"\"\"\u001b[39;00m\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\semantic_extract\\ner_extractor.py:75\u001b[0m\n\u001b[0;32m 72\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprogress_tracker\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_progress_tracker\n\u001b[0;32m 74\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 75\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mspacy\u001b[39;00m\n\u001b[0;32m 77\u001b[0m SPACY_AVAILABLE \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m 78\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\spacy\\__init__.py:6\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, Iterable, Union\n\u001b[0;32m 5\u001b[0m \u001b[38;5;66;03m# set library-specific custom warning handling before doing anything else\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01merrors\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m setup_default_warnings\n\u001b[0;32m 8\u001b[0m setup_default_warnings() \u001b[38;5;66;03m# noqa: E402\u001b[39;00m\n\u001b[0;32m 10\u001b[0m \u001b[38;5;66;03m# These are imported as part of the API\u001b[39;00m\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\spacy\\errors.py:3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mwarnings\u001b[39;00m\n\u001b[1;32m----> 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompat\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Literal\n\u001b[0;32m 6\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mErrorsWithCodes\u001b[39;00m(\u001b[38;5;28mtype\u001b[39m):\n\u001b[0;32m 7\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__getattribute__\u001b[39m(\u001b[38;5;28mself\u001b[39m, code):\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\spacy\\compat.py:5\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;124;03m\"\"\"Helpers for Python and platform compatibility.\"\"\"\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msys\u001b[39;00m\n\u001b[1;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mthinc\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutil\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m copy_array\n\u001b[0;32m 7\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mcPickle\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpickle\u001b[39;00m\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\thinc\\__init__.py:5\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mnumpy\u001b[39;00m\n\u001b[0;32m 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mabout\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m __version__\n\u001b[1;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m registry\n\u001b[0;32m 7\u001b[0m \u001b[38;5;66;03m# fmt: off\u001b[39;00m\n\u001b[0;32m 8\u001b[0m __all__ \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m 9\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mregistry\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 10\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__version__\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 11\u001b[0m ]\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\thinc\\config.py:5\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mconfection\u001b[39;00m\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mconfection\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m VARIABLE_RE, Config, ConfigValidationError, Promise\n\u001b[1;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtypes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Decorator\n\u001b[0;32m 8\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mregistry\u001b[39;00m(confection\u001b[38;5;241m.\u001b[39mregistry):\n\u001b[0;32m 9\u001b[0m \u001b[38;5;66;03m# fmt: off\u001b[39;00m\n\u001b[0;32m 10\u001b[0m optimizers: Decorator \u001b[38;5;241m=\u001b[39m catalogue\u001b[38;5;241m.\u001b[39mcreate(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthinc\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moptimizers\u001b[39m\u001b[38;5;124m\"\u001b[39m, entry_points\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\thinc\\types.py:27\u001b[0m\n\u001b[0;32m 24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpydantic\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m GetCoreSchemaHandler\n\u001b[0;32m 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpydantic_core\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m core_schema\n\u001b[1;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompat\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m cupy, has_cupy\n\u001b[0;32m 29\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_cupy:\n\u001b[0;32m 30\u001b[0m get_array_module \u001b[38;5;241m=\u001b[39m cupy\u001b[38;5;241m.\u001b[39mget_array_module\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\thinc\\compat.py:35\u001b[0m\n\u001b[0;32m 31\u001b[0m has_cupy_gpu \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m 34\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m---> 35\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtorch\u001b[39;00m\n\u001b[0;32m 36\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdlpack\u001b[39;00m\n\u001b[0;32m 38\u001b[0m has_torch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\__init__.py:281\u001b[0m\n\u001b[0;32m 277\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[0;32m 279\u001b[0m kernel32\u001b[38;5;241m.\u001b[39mSetErrorMode(prev_error_mode)\n\u001b[1;32m--> 281\u001b[0m \u001b[43m_load_dll_libraries\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 282\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m _load_dll_libraries\n\u001b[0;32m 285\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_get_cuda_dep_paths\u001b[39m(path: \u001b[38;5;28mstr\u001b[39m, lib_folder: \u001b[38;5;28mstr\u001b[39m, lib_name: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m 286\u001b[0m \u001b[38;5;66;03m# Libraries can either be in\u001b[39;00m\n\u001b[0;32m 287\u001b[0m \u001b[38;5;66;03m# path/nvidia/lib_folder/lib or\u001b[39;00m\n\u001b[0;32m 288\u001b[0m \u001b[38;5;66;03m# path/nvidia/cuXX/lib (since CUDA 13.0) or\u001b[39;00m\n\u001b[0;32m 289\u001b[0m \u001b[38;5;66;03m# path/lib_folder/lib\u001b[39;00m\n",
|
||||
"File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\__init__.py:264\u001b[0m, in \u001b[0;36m_load_dll_libraries\u001b[1;34m()\u001b[0m\n\u001b[0;32m 260\u001b[0m err \u001b[38;5;241m=\u001b[39m ctypes\u001b[38;5;241m.\u001b[39mWinError(last_error)\n\u001b[0;32m 261\u001b[0m err\u001b[38;5;241m.\u001b[39mstrerror \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 262\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m Error loading \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdll\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m or one of its dependencies.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m 263\u001b[0m )\n\u001b[1;32m--> 264\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[0;32m 265\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m res \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m 266\u001b[0m is_loaded \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
|
||||
"\u001b[1;31mOSError\u001b[0m: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies."
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✓ Groq LLM initialized: llama-3.1-8b-instant\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -102,11 +84,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✓ Document parsed: Q1 2024 Earnings Call\n",
|
||||
" Text length: 409 characters\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Step 1: Parse PDF with Docling\n",
|
||||
"from semantica.parse import DoclingParser\n",
|
||||
"\n",
|
||||
"parser = DoclingParser(\n",
|
||||
@@ -148,9 +138,29 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>Progress</th><th>ETA</th><th>Rate</th><th>Time</th></tr><tr><td>✅</td><td>Semantica is normalizing</td><td>🔧 normalize</td><td>TextNormalizer</td><td>100.0%</td><td>-</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>NERExtractor</td><td>100.0%</td><td>-</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>RelationExtractor</td><td>100.0%</td><td>-</td><td>-</td><td>0.00s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>TripletExtractor</td><td>100.0% (1/1)</td><td>-</td><td>65.4/s</td><td>0.02s</td></tr></table></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🧠 Normalizing text 🔄🔧 (0.0s) ✓ Text normalized: 409 characters\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Step 2: Normalize text with Semantica\n",
|
||||
"from semantica.normalize import TextNormalizer\n",
|
||||
@@ -176,9 +186,24 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Method llm failed: groq provider not available\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✓ Entities extracted: 0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Step 3: Extract entities using NERExtractor with Groq\n",
|
||||
"from semantica.semantic_extract import NERExtractor\n",
|
||||
@@ -211,9 +236,32 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🧠 Normalizing text 🔄🔧 (0.0s) | 🧠 Semantica is extracting: Extracting named entities from text 🔄🎯 (0.0s) "
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Method llm failed: groq provider not available\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✓ Financial entities: 0\n",
|
||||
" Financial metrics: 0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Step 4: Extract financial metrics using NERExtractor\n",
|
||||
"financial_entities = ner.extract_entities(\n",
|
||||
@@ -241,9 +289,17 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✓ Relationships extracted: 0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Step 5: Extract relationships using RelationExtractor with Groq LLM\n",
|
||||
"from semantica.semantic_extract import RelationExtractor\n",
|
||||
@@ -278,9 +334,35 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🧠 Semantica is extracting: Extracting named entities from text 🔄🎯 (0.0s) | 🧠 Semantica is extracting: Extracting triplets using llm... (1/1, remaining: 0 methods) |███████████████| 100.0% [1/1] 🔄🎯 (88.5/s)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Method llm failed: groq provider not available\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "TypeError",
|
||||
"evalue": "'bool' object is not callable",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[8], line 18\u001b[0m\n\u001b[0;32m 4\u001b[0m triplet_extractor \u001b[38;5;241m=\u001b[39m TripletExtractor(\n\u001b[0;32m 5\u001b[0m method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mllm\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 6\u001b[0m include_temporal\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 7\u001b[0m include_provenance\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m 8\u001b[0m )\n\u001b[0;32m 10\u001b[0m triplets \u001b[38;5;241m=\u001b[39m triplet_extractor\u001b[38;5;241m.\u001b[39mextract_triplets(\n\u001b[0;32m 11\u001b[0m normalized_text,\n\u001b[0;32m 12\u001b[0m entities\u001b[38;5;241m=\u001b[39mentities,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 15\u001b[0m llm_model\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mllama-3.1-8b-instant\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 16\u001b[0m )\n\u001b[1;32m---> 18\u001b[0m validated_triplets \u001b[38;5;241m=\u001b[39m \u001b[43mtriplet_extractor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_triplets\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtriplets\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 20\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m✓ RDF triplets extracted: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(triplets)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 21\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m triplets:\n",
|
||||
"\u001b[1;31mTypeError\u001b[0m: 'bool' object is not callable"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Step 6: Extract RDF triplets using TripletExtractor with Groq LLM\n",
|
||||
"from semantica.semantic_extract import TripletExtractor\n",
|
||||
|
||||
@@ -116,7 +116,7 @@ class Semantica:
|
||||
self._modules["embedding_generator"] = EmbeddingGenerator(
|
||||
config=self.config.get("embedding", {})
|
||||
)
|
||||
except ImportError as e:
|
||||
except (ImportError, OSError) as e:
|
||||
self.logger.warning(f"Could not import EmbeddingGenerator: {e}")
|
||||
raise ProcessingError(f"Embeddings module not available: {e}")
|
||||
return self._modules["embedding_generator"]
|
||||
@@ -133,7 +133,7 @@ class Semantica:
|
||||
try:
|
||||
from ..reasoning import GraphReasoner
|
||||
self._modules["reasoner"] = GraphReasoner(config=self.config)
|
||||
except ImportError as e:
|
||||
except (ImportError, OSError) as e:
|
||||
self.logger.warning(f"Could not import GraphReasoner: {e}")
|
||||
raise ProcessingError(f"Reasoning module not available: {e}")
|
||||
return self._modules["reasoner"]
|
||||
@@ -371,7 +371,7 @@ class Semantica:
|
||||
|
||||
if isinstance(pipeline, Pipeline):
|
||||
execution_engine = ExecutionEngine()
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
execution_engine = None
|
||||
|
||||
if execution_engine is None and not hasattr(pipeline, "execute"):
|
||||
@@ -528,7 +528,7 @@ class Semantica:
|
||||
from ..pipeline import PipelineBuilder
|
||||
|
||||
self.logger.debug("Framework modules verified and available")
|
||||
except ImportError as e:
|
||||
except (ImportError, OSError) as e:
|
||||
# Log but don't fail - modules may be optional or not installed
|
||||
self.logger.warning(
|
||||
f"Some framework modules could not be imported: {e}. "
|
||||
@@ -662,7 +662,7 @@ class Semantica:
|
||||
|
||||
pipeline_builder.add_step("default_step", "default")
|
||||
return pipeline_builder.build("default_pipeline")
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.debug("Pipeline module not available, using config directly")
|
||||
return pipeline_config
|
||||
|
||||
@@ -731,7 +731,7 @@ class Semantica:
|
||||
)
|
||||
raise
|
||||
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning("KG module not available, returning placeholder")
|
||||
return {"status": "placeholder", "results": results}
|
||||
|
||||
@@ -801,7 +801,7 @@ class Semantica:
|
||||
)
|
||||
raise
|
||||
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning(
|
||||
"Embeddings module not available, returning placeholder"
|
||||
)
|
||||
@@ -905,7 +905,7 @@ class Semantica:
|
||||
# CPU percent may not be available immediately
|
||||
pass
|
||||
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
# psutil not available, use basic metrics
|
||||
self.logger.debug("psutil not available, using basic metrics")
|
||||
except Exception as e:
|
||||
|
||||
@@ -349,21 +349,21 @@ def check_available_providers() -> Dict[str, bool]:
|
||||
try:
|
||||
import sentence_transformers
|
||||
providers["sentence_transformers"] = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
providers["sentence_transformers"] = False
|
||||
|
||||
# Check FastEmbed
|
||||
try:
|
||||
import fastembed
|
||||
providers["fastembed"] = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
providers["fastembed"] = False
|
||||
|
||||
# Check OpenAI
|
||||
try:
|
||||
import openai
|
||||
providers["openai"] = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
providers["openai"] = False
|
||||
|
||||
return providers
|
||||
|
||||
@@ -66,7 +66,7 @@ class OpenAIStore(ProviderStore):
|
||||
from openai import OpenAI
|
||||
|
||||
self.client = OpenAI(api_key=self.api_key)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning("OpenAI library not installed")
|
||||
|
||||
def embed(self, text: str, **options) -> np.ndarray:
|
||||
@@ -114,7 +114,7 @@ class BGEStore(ProviderStore):
|
||||
|
||||
self.model = SentenceTransformer(self.model_name)
|
||||
self.logger.info(f"Loaded BGE model: {self.model_name}")
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning("sentence-transformers not available for BGE")
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to load BGE model: {e}")
|
||||
@@ -210,7 +210,7 @@ class FastEmbedStore(ProviderStore):
|
||||
|
||||
self.model = TextEmbedding(model_name=self.model_name)
|
||||
self.logger.info(f"Loaded FastEmbed model: {self.model_name}")
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning(
|
||||
"fastembed not available. Install with: pip install fastembed"
|
||||
)
|
||||
|
||||
@@ -34,14 +34,14 @@ try:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
SENTENCE_TRANSFORMERS_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
SENTENCE_TRANSFORMERS_AVAILABLE = False
|
||||
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
|
||||
FASTEMBED_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
FASTEMBED_AVAILABLE = False
|
||||
|
||||
|
||||
|
||||
@@ -287,7 +287,7 @@ class VectorExporter:
|
||||
"""
|
||||
try:
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError("NumPy not installed. Install with: pip install numpy")
|
||||
|
||||
# Extract vectors and associated data
|
||||
@@ -358,7 +358,7 @@ class VectorExporter:
|
||||
"""Export to binary format."""
|
||||
try:
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError("NumPy not installed. Install with: pip install numpy")
|
||||
|
||||
# Extract vectors
|
||||
@@ -419,7 +419,7 @@ class VectorExporter:
|
||||
try:
|
||||
import faiss
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"FAISS not installed. Install with: pip install faiss-cpu or faiss-gpu"
|
||||
)
|
||||
|
||||
@@ -71,7 +71,7 @@ class SemanticNetworkYAMLExporter:
|
||||
import yaml
|
||||
|
||||
self.yaml = yaml
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError("PyYAML not installed. Install with: pip install pyyaml")
|
||||
|
||||
# Initialize progress tracker
|
||||
@@ -297,7 +297,7 @@ class YAMLSchemaExporter:
|
||||
import yaml
|
||||
|
||||
self.yaml = yaml
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError("PyYAML not installed. Install with: pip install pyyaml")
|
||||
|
||||
def export_ontology_schema(self, ontology: Dict[str, Any], **options) -> str:
|
||||
|
||||
@@ -47,7 +47,7 @@ try:
|
||||
from falkordb import FalkorDB
|
||||
|
||||
FALKORDB_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
FALKORDB_AVAILABLE = False
|
||||
FalkorDB = None
|
||||
|
||||
|
||||
@@ -1046,7 +1046,7 @@ class GraphStore:
|
||||
try:
|
||||
from ..context.entity_linker import EntityLinker
|
||||
linker = EntityLinker() # Use default config
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
return []
|
||||
|
||||
entity_nodes = [n for n in nodes if n.get("type") == "entity"]
|
||||
|
||||
@@ -50,7 +50,7 @@ try:
|
||||
)
|
||||
|
||||
NEO4J_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
NEO4J_AVAILABLE = False
|
||||
GraphDatabase = None
|
||||
Neo4jError = Exception
|
||||
|
||||
@@ -32,7 +32,7 @@ import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
try:
|
||||
from urllib3.util.retry import Retry
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
from requests.packages.urllib3.util.retry import Retry
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
|
||||
@@ -37,7 +37,7 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
try:
|
||||
import duckdb
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
duckdb = None
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ class DuckDBIngestor:
|
||||
# to read Excel and then query it
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"pandas and openpyxl are required for Excel ingestion. "
|
||||
"Install with: pip install pandas openpyxl"
|
||||
|
||||
@@ -34,7 +34,7 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
try:
|
||||
from elasticsearch import Elasticsearch
|
||||
from elasticsearch.helpers import scan
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
Elasticsearch = None
|
||||
scan = None
|
||||
|
||||
|
||||
@@ -358,7 +358,7 @@ class FeedParser:
|
||||
from dateutil import parser
|
||||
|
||||
return parser.parse(date_string)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
@@ -41,7 +41,7 @@ try:
|
||||
from googleapiclient.errors import HttpError
|
||||
from googleapiclient.http import MediaIoBaseDownload
|
||||
import io
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
Credentials = None
|
||||
InstalledAppFlow = None
|
||||
Request = None
|
||||
|
||||
@@ -32,7 +32,7 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
try:
|
||||
from datasets import load_dataset, Dataset, IterableDataset
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
load_dataset = None
|
||||
Dataset = None
|
||||
IterableDataset = None
|
||||
|
||||
@@ -353,7 +353,7 @@ class MCPClient:
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
# Fallback to requests if httpx not available
|
||||
try:
|
||||
import requests
|
||||
@@ -366,7 +366,7 @@ class MCPClient:
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ProcessingError(
|
||||
"HTTP transport requires 'httpx' or 'requests' package. "
|
||||
"Install with: pip install httpx or pip install requests"
|
||||
|
||||
@@ -35,7 +35,7 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
try:
|
||||
from pymongo import MongoClient
|
||||
from pymongo.errors import ConnectionFailure, OperationFailure
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
MongoClient = None
|
||||
ConnectionFailure = None
|
||||
OperationFailure = None
|
||||
|
||||
@@ -36,7 +36,7 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
pd = None
|
||||
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ class CentralityCalculator:
|
||||
self.nx = nx
|
||||
self.use_networkx = True
|
||||
self.logger.debug("NetworkX available, using optimized implementations")
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.nx = None
|
||||
self.use_networkx = False
|
||||
self.logger.debug(
|
||||
|
||||
@@ -85,7 +85,7 @@ class CommunityDetector:
|
||||
self.nx = nx
|
||||
self.use_networkx = True
|
||||
self.logger.debug("NetworkX available, using optimized implementations")
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.nx = None
|
||||
self.use_networkx = False
|
||||
self.logger.warning("NetworkX not available, using basic implementations")
|
||||
|
||||
@@ -82,13 +82,13 @@ class ConnectivityAnalyzer:
|
||||
self.config = config
|
||||
|
||||
# Try to use networkx if available (optional dependency)
|
||||
try:
|
||||
import networkx as nx
|
||||
|
||||
self.nx = nx
|
||||
from ..utils.helpers import safe_import
|
||||
networkx, nx_available = safe_import("networkx")
|
||||
if nx_available:
|
||||
self.nx = networkx
|
||||
self.use_networkx = True
|
||||
self.logger.debug("NetworkX available, using optimized implementations")
|
||||
except ImportError:
|
||||
else:
|
||||
self.nx = None
|
||||
self.use_networkx = False
|
||||
self.logger.warning("NetworkX not available, using basic implementations")
|
||||
|
||||
@@ -944,7 +944,7 @@ class GraphBuilder:
|
||||
)
|
||||
return graph
|
||||
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="neo4j library not available"
|
||||
)
|
||||
|
||||
@@ -12,11 +12,13 @@ from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("llms.litellm")
|
||||
|
||||
try:
|
||||
from ..utils.helpers import safe_import
|
||||
|
||||
_litellm, LITELLM_AVAILABLE = safe_import("litellm")
|
||||
if LITELLM_AVAILABLE:
|
||||
from litellm import completion
|
||||
LITELLM_AVAILABLE = True
|
||||
except ImportError:
|
||||
LITELLM_AVAILABLE = False
|
||||
else:
|
||||
completion = None
|
||||
logger.warning(
|
||||
"litellm library not installed. Install with: pip install litellm"
|
||||
)
|
||||
|
||||
@@ -44,7 +44,7 @@ try:
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
HAS_DATEUTIL = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
HAS_DATEUTIL = False
|
||||
date_parser = None
|
||||
relativedelta = None
|
||||
|
||||
@@ -33,7 +33,7 @@ try:
|
||||
from langdetect.lang_detect_exception import LangDetectException
|
||||
|
||||
LANGDETECT_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
LANGDETECT_AVAILABLE = False
|
||||
LangDetectException = Exception
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ try:
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
BEAUTIFULSOUP_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
BEAUTIFULSOUP_AVAILABLE = False
|
||||
BeautifulSoup = None
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ try:
|
||||
from rdflib.namespace import NamespaceManager as RDFNamespaceManager
|
||||
|
||||
HAS_RDFLIB = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
HAS_RDFLIB = False
|
||||
Graph = None
|
||||
RDF = None
|
||||
|
||||
@@ -172,7 +172,7 @@ from .pptx_parser import PPTXData, PPTXParser, SlideContent
|
||||
try:
|
||||
from .docling_parser import DoclingParser, DoclingMetadata
|
||||
DOCLING_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
DOCLING_AVAILABLE = False
|
||||
DoclingParser = None
|
||||
DoclingMetadata = None
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""
|
||||
Docling Document Parser Module
|
||||
|
||||
This module handles document parsing using Docling for enhanced table extraction
|
||||
and better document structure understanding across multiple formats (PDF, DOCX, PPTX, XLSX, HTML, images).
|
||||
This module provides a standalone document parser that uses Docling as its core dependency.
|
||||
DoclingParser is completely independent from DocumentParser and uses only:
|
||||
- docling: Core document parsing library (DocumentConverter)
|
||||
- semantica utilities: Logging, progress tracking, exceptions
|
||||
|
||||
Key Features:
|
||||
- Multi-format document parsing (PDF, DOCX, PPTX, XLSX, HTML, images)
|
||||
@@ -11,9 +13,13 @@ Key Features:
|
||||
- Markdown, HTML, and JSON export formats
|
||||
- Local execution support
|
||||
- OCR support for scanned documents
|
||||
- Standalone parser - no dependency on DocumentParser
|
||||
|
||||
Core Dependency:
|
||||
- docling: Required for all parsing functionality
|
||||
|
||||
Main Classes:
|
||||
- DoclingParser: Docling-based document parser
|
||||
- DoclingParser: Standalone Docling-based document parser
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.parse import DoclingParser
|
||||
@@ -31,20 +37,26 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import safe_import
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Try to import docling, handle gracefully if not available
|
||||
DOCLING_AVAILABLE = False
|
||||
DOCLING_IMPORT_ERROR = None
|
||||
DocumentConverter = None
|
||||
InputFormat = None
|
||||
PdfPipelineOptions = None
|
||||
|
||||
try:
|
||||
from docling.document_converter import DocumentConverter
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.pipeline_options import PdfPipelineOptions
|
||||
DOCLING_AVAILABLE = True
|
||||
except ImportError:
|
||||
DOCLING_IMPORT_ERROR = None
|
||||
except (ImportError, OSError) as e:
|
||||
DOCLING_AVAILABLE = False
|
||||
DocumentConverter = None
|
||||
InputFormat = None
|
||||
PdfPipelineOptions = None
|
||||
DOCLING_IMPORT_ERROR = str(e)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -75,11 +87,6 @@ class DoclingParser:
|
||||
- enable_ocr: Enable OCR for scanned documents (default: False)
|
||||
- table_extraction_mode: Table extraction mode (default: "auto")
|
||||
"""
|
||||
if not DOCLING_AVAILABLE:
|
||||
raise ImportError(
|
||||
"Docling is not installed. Install it with: pip install docling"
|
||||
)
|
||||
|
||||
self.logger = get_logger("docling_parser")
|
||||
self.config = config
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
@@ -87,23 +94,11 @@ class DoclingParser:
|
||||
if not self.progress_tracker.enabled:
|
||||
self.progress_tracker.enabled = True
|
||||
|
||||
# Initialize DocumentConverter
|
||||
export_format = config.get("export_format", "markdown")
|
||||
enable_ocr = config.get("enable_ocr", False)
|
||||
table_extraction_mode = config.get("table_extraction_mode", "auto")
|
||||
|
||||
# Configure pipeline options
|
||||
pipeline_options = PdfPipelineOptions()
|
||||
if enable_ocr:
|
||||
pipeline_options.do_ocr = True
|
||||
|
||||
self.converter = DocumentConverter(
|
||||
format_options={
|
||||
"markdown": {"table_format": table_extraction_mode},
|
||||
},
|
||||
pipeline_options=pipeline_options,
|
||||
)
|
||||
self.export_format = export_format
|
||||
# Store config for lazy initialization
|
||||
self.export_format = config.get("export_format", "markdown")
|
||||
self.enable_ocr = config.get("enable_ocr", False)
|
||||
self.table_extraction_mode = config.get("table_extraction_mode", "auto")
|
||||
self._converter = None
|
||||
|
||||
def parse(self, file_path: Union[str, Path], **options) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -135,10 +130,19 @@ class DoclingParser:
|
||||
if not file_path.exists():
|
||||
raise ValidationError(f"Document file not found: {file_path}")
|
||||
|
||||
# Check if docling is available
|
||||
# Check if docling is available and initialize converter lazily
|
||||
if not DOCLING_AVAILABLE:
|
||||
raise ImportError(
|
||||
"Docling is not installed. Install it with: pip install docling"
|
||||
if DOCLING_IMPORT_ERROR:
|
||||
raise ImportError(DOCLING_IMPORT_ERROR)
|
||||
else:
|
||||
raise ImportError("Docling is not installed")
|
||||
|
||||
# Lazy initialization of converter
|
||||
if self._converter is None:
|
||||
self._converter = DocumentConverter(
|
||||
format_options={
|
||||
"markdown": {"table_format": self.table_extraction_mode},
|
||||
},
|
||||
)
|
||||
|
||||
# Determine export format
|
||||
@@ -149,7 +153,7 @@ class DoclingParser:
|
||||
)
|
||||
|
||||
# Convert document using Docling
|
||||
result = self.converter.convert(str(file_path))
|
||||
result = self._converter.convert(str(file_path))
|
||||
|
||||
# Extract content based on export format
|
||||
extract_text = options.get("extract_text", True)
|
||||
@@ -200,7 +204,7 @@ class DoclingParser:
|
||||
"export_format": export_format,
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Docling not installed"
|
||||
)
|
||||
|
||||
@@ -38,13 +38,8 @@ from .docx_parser import DOCXParser
|
||||
from .html_parser import HTMLParser
|
||||
from .pdf_parser import PDFParser
|
||||
|
||||
# Try to import DoclingParser (optional dependency)
|
||||
try:
|
||||
from .docling_parser import DoclingParser
|
||||
DOCLING_AVAILABLE = True
|
||||
except ImportError:
|
||||
DOCLING_AVAILABLE = False
|
||||
DoclingParser = None
|
||||
# DoclingParser is available as a standalone parser
|
||||
# Import it directly: from semantica.parse import DoclingParser
|
||||
|
||||
|
||||
class DocumentParser:
|
||||
@@ -90,14 +85,6 @@ class DocumentParser:
|
||||
self.pdf_parser = PDFParser(**self.config.get("pdf", {}))
|
||||
self.docx_parser = DOCXParser(**self.config.get("docx", {}))
|
||||
self.html_parser = HTMLParser(**self.config.get("html", {}))
|
||||
|
||||
# Initialize Docling parser if available (optional)
|
||||
self.docling_parser = None
|
||||
if DOCLING_AVAILABLE:
|
||||
try:
|
||||
self.docling_parser = DoclingParser(**self.config.get("docling", {}))
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not initialize DoclingParser: {e}")
|
||||
|
||||
# Supported formats
|
||||
self.supported_formats = {
|
||||
@@ -193,16 +180,9 @@ class DocumentParser:
|
||||
tracking_id, message=f"Parsing {file_type} document"
|
||||
)
|
||||
|
||||
# Check if docling method is requested
|
||||
method = options.get("method", "default")
|
||||
use_docling = method == "docling" and self.docling_parser is not None
|
||||
|
||||
# Route to appropriate parser
|
||||
try:
|
||||
if use_docling:
|
||||
# Use Docling for parsing (supports multiple formats)
|
||||
result = self.docling_parser.parse(file_path, **options)
|
||||
elif file_type == "pdf":
|
||||
if file_type == "pdf":
|
||||
result = self.pdf_parser.parse(file_path, **options)
|
||||
elif file_type == "docx":
|
||||
result = self.docx_parser.parse(file_path, **options)
|
||||
|
||||
@@ -39,12 +39,9 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
try:
|
||||
import pytesseract
|
||||
from ..utils.helpers import safe_import
|
||||
|
||||
TESSERACT_AVAILABLE = True
|
||||
except ImportError:
|
||||
TESSERACT_AVAILABLE = False
|
||||
pytesseract, TESSERACT_AVAILABLE = safe_import("pytesseract")
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -215,7 +215,7 @@ class MediaParser:
|
||||
else None,
|
||||
}
|
||||
)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning("mutagen not available for audio metadata extraction")
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to extract audio metadata: {e}")
|
||||
|
||||
@@ -228,7 +228,7 @@ def parse_document_docling(
|
||||
"""
|
||||
try:
|
||||
from .docling_parser import DoclingParser
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"Docling is not installed. Install it with: pip install docling"
|
||||
)
|
||||
@@ -249,7 +249,7 @@ def parse_document_docling(
|
||||
try:
|
||||
from .docling_parser import DoclingParser
|
||||
method_registry.register("document", "docling", parse_document_docling)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
# Docling not available, skip registration
|
||||
pass
|
||||
|
||||
|
||||
@@ -304,7 +304,7 @@ class JavaScriptRenderer:
|
||||
|
||||
self.driver = None # Initialize on first use
|
||||
self.logger.info("Selenium WebDriver configured")
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning(
|
||||
"Selenium not available, JavaScript rendering disabled"
|
||||
)
|
||||
|
||||
@@ -155,7 +155,7 @@ class ResourceScheduler:
|
||||
capacity=100.0,
|
||||
metadata={},
|
||||
)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
# Fallback if psutil not available
|
||||
self.logger.warning("psutil not available, using default resource values")
|
||||
self.resources["cpu"] = Resource(
|
||||
|
||||
@@ -428,7 +428,7 @@ class SeedDataManager:
|
||||
self.logger.info(f"Loaded {len(records)} records from database")
|
||||
return records
|
||||
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ProcessingError(
|
||||
"Database ingestion module not available. Install required dependencies."
|
||||
)
|
||||
@@ -530,7 +530,7 @@ class SeedDataManager:
|
||||
self.logger.info(f"Loaded {len(records)} records from API: {full_url}")
|
||||
return records
|
||||
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ProcessingError(
|
||||
"requests library not available. Install with: pip install requests"
|
||||
)
|
||||
|
||||
@@ -119,12 +119,9 @@ from .triplet_extractor import Triplet
|
||||
logger = get_logger("methods")
|
||||
|
||||
# Try to import spaCy
|
||||
try:
|
||||
import spacy
|
||||
from ..utils.helpers import safe_import
|
||||
|
||||
SPACY_AVAILABLE = True
|
||||
except ImportError:
|
||||
SPACY_AVAILABLE = False
|
||||
spacy, SPACY_AVAILABLE = safe_import("spacy")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
|
||||
@@ -68,15 +68,11 @@ from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.helpers import safe_import
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
try:
|
||||
import spacy
|
||||
|
||||
SPACY_AVAILABLE = True
|
||||
except ImportError:
|
||||
SPACY_AVAILABLE = False
|
||||
spacy, SPACY_AVAILABLE = safe_import("spacy")
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -197,7 +197,7 @@ class OpenAIProvider(BaseProvider):
|
||||
|
||||
if self.api_key:
|
||||
self.client = OpenAI(api_key=self.api_key)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.client = None
|
||||
self.logger.warning(
|
||||
"openai library not installed. Install with: pip install semantica[llm-openai]"
|
||||
@@ -259,7 +259,7 @@ class GeminiProvider(BaseProvider):
|
||||
if self.api_key:
|
||||
genai.configure(api_key=self.api_key)
|
||||
self.client = genai.GenerativeModel(self.model)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.client = None
|
||||
self.logger.warning(
|
||||
"google-generativeai library not installed. Install with: pip install semantica[llm-gemini]"
|
||||
@@ -315,7 +315,7 @@ class GroqProvider(BaseProvider):
|
||||
|
||||
if self.api_key:
|
||||
self.client = Groq(api_key=self.api_key)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.client = None
|
||||
self.logger.warning(
|
||||
"groq library not installed. Install with: pip install semantica[llm-groq]"
|
||||
@@ -379,7 +379,7 @@ class AnthropicProvider(BaseProvider):
|
||||
|
||||
if self.api_key:
|
||||
self.client = Anthropic(api_key=self.api_key)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.client = None
|
||||
self.logger.warning(
|
||||
"anthropic library not installed. Install with: pip install semantica[llm-anthropic]"
|
||||
@@ -447,7 +447,7 @@ class OllamaProvider(BaseProvider):
|
||||
self.logger.warning(
|
||||
"Ollama server not accessible. Make sure Ollama is running."
|
||||
)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.client = None
|
||||
self.logger.warning(
|
||||
"ollama library not installed. Install with: pip install semantica[llm-ollama]"
|
||||
@@ -502,7 +502,7 @@ class DeepSeekProvider(BaseProvider):
|
||||
|
||||
if self.api_key:
|
||||
self.client = deepseek.Client(api_key=self.api_key)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.client = None
|
||||
self.logger.warning(
|
||||
"deepseek library not installed. Install with: pip install semantica[llm-deepseek]"
|
||||
@@ -545,7 +545,7 @@ class HuggingFaceLLMProvider(BaseProvider):
|
||||
# Lazy import torch only when needed
|
||||
try:
|
||||
import torch
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"torch is required for HuggingFaceLLMProvider. Install with: pip install torch"
|
||||
)
|
||||
@@ -565,7 +565,7 @@ class HuggingFaceLLMProvider(BaseProvider):
|
||||
self.model = AutoModelForCausalLM.from_pretrained(self.model_name)
|
||||
self.model.to(self.device)
|
||||
self.model.eval()
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning(
|
||||
"transformers library not installed. Install with: pip install semantica[models-huggingface]"
|
||||
)
|
||||
@@ -616,7 +616,7 @@ class HuggingFaceModelLoader:
|
||||
# Lazy import torch only when needed
|
||||
try:
|
||||
import torch
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"torch is required for HuggingFaceModelLoader. Install with: pip install torch"
|
||||
)
|
||||
@@ -645,7 +645,7 @@ class HuggingFaceModelLoader:
|
||||
)
|
||||
self._cache[cache_key] = nlp
|
||||
return nlp
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"transformers library not installed. Install with: pip install semantica[models-huggingface]"
|
||||
)
|
||||
@@ -672,7 +672,7 @@ class HuggingFaceModelLoader:
|
||||
)
|
||||
self._cache[cache_key] = nlp
|
||||
return nlp
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"transformers library not installed. Install with: pip install semantica[models-huggingface]"
|
||||
)
|
||||
@@ -696,7 +696,7 @@ class HuggingFaceModelLoader:
|
||||
nlp = {"tokenizer": tokenizer, "model": model, "device": self.device}
|
||||
self._cache[cache_key] = nlp
|
||||
return nlp
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"transformers library not installed. Install with: pip install semantica[models-huggingface]"
|
||||
)
|
||||
|
||||
+33
-49
@@ -90,64 +90,48 @@ from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.helpers import safe_import
|
||||
from ..utils.logging import get_logger
|
||||
from .semantic_chunker import Chunk
|
||||
|
||||
logger = get_logger("split_methods")
|
||||
|
||||
# Try to import optional dependencies
|
||||
try:
|
||||
import spacy
|
||||
spacy, SPACY_AVAILABLE = safe_import("spacy")
|
||||
|
||||
SPACY_AVAILABLE = True
|
||||
except ImportError:
|
||||
SPACY_AVAILABLE = False
|
||||
|
||||
try:
|
||||
import nltk
|
||||
|
||||
NLTK_AVAILABLE = True
|
||||
except ImportError:
|
||||
NLTK_AVAILABLE = False
|
||||
|
||||
try:
|
||||
import tiktoken
|
||||
|
||||
TIKTOKEN_AVAILABLE = True
|
||||
except ImportError:
|
||||
TIKTOKEN_AVAILABLE = False
|
||||
|
||||
try:
|
||||
nltk, NLTK_AVAILABLE = safe_import("nltk")
|
||||
tiktoken, TIKTOKEN_AVAILABLE = safe_import("tiktoken")
|
||||
_sentence_transformers, SENTENCE_TRANSFORMER_AVAILABLE = safe_import("sentence_transformers")
|
||||
if SENTENCE_TRANSFORMER_AVAILABLE:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
SENTENCE_TRANSFORMER_AVAILABLE = True
|
||||
except ImportError:
|
||||
SENTENCE_TRANSFORMER_AVAILABLE = False
|
||||
|
||||
try:
|
||||
else:
|
||||
SentenceTransformer = None
|
||||
_transformers, TRANSFORMERS_AVAILABLE = safe_import("transformers")
|
||||
if TRANSFORMERS_AVAILABLE:
|
||||
from transformers import AutoTokenizer
|
||||
else:
|
||||
AutoTokenizer = None
|
||||
|
||||
TRANSFORMERS_AVAILABLE = True
|
||||
except ImportError:
|
||||
TRANSFORMERS_AVAILABLE = False
|
||||
networkx, NETWORKX_AVAILABLE = safe_import("networkx")
|
||||
if NETWORKX_AVAILABLE:
|
||||
nx = networkx
|
||||
else:
|
||||
nx = None
|
||||
|
||||
try:
|
||||
import networkx as nx
|
||||
|
||||
NETWORKX_AVAILABLE = True
|
||||
except ImportError:
|
||||
NETWORKX_AVAILABLE = False
|
||||
|
||||
try:
|
||||
# Try community package with fallback
|
||||
_community1, _avail1 = safe_import("community.community_louvain")
|
||||
if _avail1:
|
||||
import community.community_louvain as community_louvain
|
||||
|
||||
COMMUNITY_AVAILABLE = True
|
||||
except ImportError:
|
||||
try:
|
||||
from community import community_louvain
|
||||
|
||||
COMMUNITY_AVAILABLE = True
|
||||
except ImportError:
|
||||
else:
|
||||
_community2, _avail2 = safe_import("community")
|
||||
if _avail2:
|
||||
try:
|
||||
from community import community_louvain
|
||||
COMMUNITY_AVAILABLE = True
|
||||
except (ImportError, OSError):
|
||||
COMMUNITY_AVAILABLE = False
|
||||
else:
|
||||
COMMUNITY_AVAILABLE = False
|
||||
|
||||
# Import from semantic_extract for entity/relation extraction
|
||||
@@ -157,7 +141,7 @@ try:
|
||||
from ..semantic_extract.relation_extractor import RelationExtractor
|
||||
|
||||
SEMANTIC_EXTRACT_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
SEMANTIC_EXTRACT_AVAILABLE = False
|
||||
|
||||
# Import specialized chunkers
|
||||
@@ -166,7 +150,7 @@ try:
|
||||
from .sliding_window_chunker import SlidingWindowChunker
|
||||
|
||||
SPECIALIZED_CHUNKERS_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
SPECIALIZED_CHUNKERS_AVAILABLE = False
|
||||
|
||||
|
||||
@@ -1702,7 +1686,7 @@ def get_split_method(method: str) -> Optional[Callable]:
|
||||
registered = method_registry.get("split", method)
|
||||
if registered:
|
||||
return registered
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
|
||||
# Check built-in methods
|
||||
@@ -1725,7 +1709,7 @@ def list_available_methods() -> List[str]:
|
||||
registered = method_registry.list_all("split")
|
||||
if registered and "split" in registered:
|
||||
methods.extend(registered["split"])
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
|
||||
return sorted(set(methods)) # Remove duplicates and sort
|
||||
|
||||
@@ -32,15 +32,11 @@ from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.helpers import safe_import
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
try:
|
||||
import spacy
|
||||
|
||||
SPACY_AVAILABLE = True
|
||||
except ImportError:
|
||||
SPACY_AVAILABLE = False
|
||||
spacy, SPACY_AVAILABLE = safe_import("spacy")
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -39,7 +39,7 @@ try:
|
||||
from rdflib.plugins.stores.sparqlstore import SPARQLStore
|
||||
|
||||
HAS_JENA_RDFLIB = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
HAS_JENA_RDFLIB = False
|
||||
Graph = None
|
||||
RDF = None
|
||||
|
||||
@@ -83,6 +83,7 @@ from .helpers import (
|
||||
read_json_file,
|
||||
retry_on_error,
|
||||
safe_filename,
|
||||
safe_import,
|
||||
set_nested_value,
|
||||
write_json_file,
|
||||
)
|
||||
@@ -194,6 +195,7 @@ __all__ = [
|
||||
"get_nested_value",
|
||||
"set_nested_value",
|
||||
"retry_on_error",
|
||||
"safe_import",
|
||||
# Constants
|
||||
"SUPPORTED_DOCUMENT_FORMATS",
|
||||
"SUPPORTED_IMAGE_FORMATS",
|
||||
|
||||
@@ -309,7 +309,7 @@ def handle_exception(
|
||||
from .logging import log_error as log_err
|
||||
|
||||
log_err(exception, context=context, **options)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
# Fallback to print if logging not available
|
||||
print(f"Error: {error_info}")
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ Main Functions:
|
||||
- get_nested_value: Get nested dictionary value by dot-separated path
|
||||
- set_nested_value: Set nested dictionary value by dot-separated path
|
||||
- retry_on_error: Decorator for retrying function on error
|
||||
- safe_import: Safely import optional modules, handling ImportError and OSError
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.utils import clean_text, normalize_entities
|
||||
@@ -58,6 +59,7 @@ License: MIT
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -87,7 +89,7 @@ def format_data(data: Any, format_type: str = "json") -> str:
|
||||
import yaml
|
||||
|
||||
return yaml.dump(data, default_flow_style=False)
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ValueError("PyYAML not installed. Install with: pip install pyyaml")
|
||||
else:
|
||||
raise ValueError(f"Unsupported format type: {format_type}")
|
||||
@@ -468,6 +470,55 @@ def set_nested_value(
|
||||
d[keys[-1]] = value
|
||||
|
||||
|
||||
def safe_import(
|
||||
module_name: str,
|
||||
package: Optional[str] = None,
|
||||
default: Any = None,
|
||||
error_message: Optional[str] = None,
|
||||
) -> Tuple[Any, bool]:
|
||||
"""
|
||||
Safely import an optional module, handling both ImportError and OSError.
|
||||
|
||||
This is useful for optional dependencies that may fail to import due to:
|
||||
- Missing package (ImportError)
|
||||
- DLL loading failures on Windows, e.g., PyTorch (OSError)
|
||||
|
||||
Args:
|
||||
module_name: Name of the module to import (e.g., "spacy", "docling.document_converter")
|
||||
package: Optional package name for relative imports
|
||||
default: Default value to return if import fails
|
||||
error_message: Optional custom error message for logging
|
||||
|
||||
Returns:
|
||||
Tuple of (module_or_default, success_flag):
|
||||
- If import succeeds: (imported_module, True)
|
||||
- If import fails: (default, False)
|
||||
|
||||
Example:
|
||||
>>> spacy, available = safe_import("spacy")
|
||||
>>> if available:
|
||||
... doc = spacy.load("en_core_web_sm")
|
||||
>>>
|
||||
>>> converter, available = safe_import("docling.document_converter", default=None)
|
||||
>>> if available:
|
||||
... converter = converter()
|
||||
"""
|
||||
try:
|
||||
if package:
|
||||
module = importlib.import_module(module_name, package=package)
|
||||
else:
|
||||
module = importlib.import_module(module_name)
|
||||
return module, True
|
||||
except (ImportError, ModuleNotFoundError, OSError) as e:
|
||||
if error_message:
|
||||
import sys
|
||||
if "logging" in sys.modules:
|
||||
from .logging import get_logger
|
||||
logger = get_logger("utils.helpers")
|
||||
logger.debug(f"{error_message}: {e}")
|
||||
return default, False
|
||||
|
||||
|
||||
def retry_on_error(
|
||||
max_retries: int = 3,
|
||||
delay: float = 1.0,
|
||||
|
||||
@@ -52,8 +52,12 @@ try:
|
||||
from IPython.display import HTML, clear_output, display
|
||||
|
||||
IPYTHON_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
IPYTHON_AVAILABLE = False
|
||||
get_ipython = None
|
||||
HTML = None
|
||||
clear_output = None
|
||||
display = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -49,7 +49,7 @@ try:
|
||||
import faiss
|
||||
|
||||
FAISS_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
FAISS_AVAILABLE = False
|
||||
faiss = None
|
||||
|
||||
|
||||
@@ -334,7 +334,7 @@ class HybridSearch:
|
||||
try:
|
||||
from ..embeddings import EmbeddingGenerator
|
||||
self.embedding_generator = EmbeddingGenerator()
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
raise ImportError("EmbeddingGenerator not available for string queries")
|
||||
|
||||
query_vector = self.embedding_generator.generate_embeddings(query, data_type="text")
|
||||
|
||||
@@ -56,7 +56,7 @@ try:
|
||||
)
|
||||
|
||||
MILVUS_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
MILVUS_AVAILABLE = False
|
||||
connections = None
|
||||
Collection = None
|
||||
|
||||
@@ -55,7 +55,7 @@ try:
|
||||
)
|
||||
|
||||
QDRANT_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
QDRANT_AVAILABLE = False
|
||||
QdrantClientLib = None
|
||||
Distance = None
|
||||
|
||||
@@ -48,7 +48,7 @@ try:
|
||||
from weaviate.classes.query import MetadataQuery, QueryReturn
|
||||
|
||||
WEAVIATE_AVAILABLE = True
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
WEAVIATE_AVAILABLE = False
|
||||
weaviate = None
|
||||
MetadataQuery = None
|
||||
|
||||
@@ -36,14 +36,14 @@ from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
np = None
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
@@ -103,7 +103,7 @@ class VisualizationConfig:
|
||||
config_data = tomli.load(f)
|
||||
if config_data and "visualization" in config_data:
|
||||
self._config.update(config_data["visualization"])
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
@@ -111,7 +111,7 @@ class VisualizationConfig:
|
||||
config_data = tomllib.load(f)
|
||||
if config_data and "visualization" in config_data:
|
||||
self._config.update(config_data["visualization"])
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning(
|
||||
"TOML parser not available. Install tomli or use Python 3.11+"
|
||||
)
|
||||
|
||||
@@ -40,7 +40,7 @@ try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
@@ -50,7 +50,7 @@ from sklearn.manifold import TSNE
|
||||
|
||||
try:
|
||||
import umap
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
umap = None
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
|
||||
@@ -38,7 +38,7 @@ import numpy as np
|
||||
try:
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
mpatches = None
|
||||
plt = None
|
||||
|
||||
@@ -46,7 +46,7 @@ try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
@@ -42,7 +42,7 @@ try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
@@ -51,7 +51,7 @@ from matplotlib.patches import FancyBboxPatch
|
||||
|
||||
try:
|
||||
import graphviz
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
graphviz = None
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
|
||||
@@ -36,7 +36,7 @@ from typing import Any, Dict, List, Optional, Union
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
px = None
|
||||
go = None
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
except (ImportError, OSError):
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
Reference in New Issue
Block a user