diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec6e888..88ac6ec8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **ArangoDB AQL Export Support** (PR #342 by @tibisabau) + - Full-featured ArangoDB AQL exporter with 642 lines of production-ready code + - Comprehensive AQL INSERT statement generation for vertices and edges + - Configurable collection names with validation and sanitization + - Batch processing support for large knowledge graphs (default: 1000) + - Added `export_arango()` convenience function for easy access + - Enhanced unified export with AQL format support and `.aql` auto-detection + - Integrated with method registry for extensibility + - 17 comprehensive test cases with 100% pass rate + - Enterprise-grade ArangoDB multi-model database integration + +### Fixed + +- Fixed NameError: missing Type import in utils/helpers.py + - Added Type to typing imports to fix retry_on_error decorator + - Removed unused Type import from config_manager.py + - Resolves ImportError when importing semantica modules + - Fixes capability gap analysis notebook execution + ## [0.3.0-alpha] - 2026-02-19 ### Added / Changed diff --git a/cookbook/use_cases/capability_gap_defense/01_Capability_Gap_Analysis_Context_Graphs_POC.ipynb b/cookbook/use_cases/capability_gap_defense/01_Capability_Gap_Analysis_Context_Graphs_POC.ipynb index d3e0cd1a..092cc683 100644 --- a/cookbook/use_cases/capability_gap_defense/01_Capability_Gap_Analysis_Context_Graphs_POC.ipynb +++ b/cookbook/use_cases/capability_gap_defense/01_Capability_Gap_Analysis_Context_Graphs_POC.ipynb @@ -154,20 +154,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "07b5e471", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: semantica==0.3.0a0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (0.3.0a0)\n", + "Requirement already satisfied: numpy>=1.21.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (1.26.4)\n", + "Requirement already satisfied: pandas>=1.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (2.3.3)\n", + "Requirement already satisfied: scipy>=1.9.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (1.15.2)\n", + "Requirement already satisfied: scikit-learn>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (1.8.0)\n", + "Requirement already satisfied: umap-learn>=0.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (0.5.9.post2)\n", + "Requirement already satisfied: spacy>=3.4.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (3.8.11)\n", + "Requirement already satisfied: transformers>=4.20.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (4.53.2)\n", + "Requirement already satisfied: torch>=1.12.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (2.9.1+cpu)\n", + "Requirement already satisfied: sentence-transformers>=2.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (5.2.0)\n", + "Requirement already satisfied: rdflib>=6.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (7.5.0)\n", + "Requirement already satisfied: networkx>=2.8.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (3.6.1)\n", + "Requirement already satisfied: matplotlib>=3.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (3.10.8)\n", + "Requirement already satisfied: seaborn>=0.11.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (0.13.2)\n", + "Requirement already satisfied: plotly>=5.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (6.5.0)\n", + "Requirement already satisfied: ipywidgets>=8.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (8.1.1)\n", + "Requirement already satisfied: requests>=2.28.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (2.32.5)\n", + "Requirement already satisfied: GitPython>=3.1.30 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (3.1.45)\n", + "Requirement already satisfied: chardet>=5.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (5.2.0)\n", + "Requirement already satisfied: protobuf<7.0,>=5.29.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (5.29.2)\n", + "Requirement already satisfied: grpcio>=1.71.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (1.71.2)\n", + "Requirement already satisfied: beautifulsoup4>=4.11.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (4.14.3)\n", + "Requirement already satisfied: lxml>=4.9.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (6.0.2)\n", + "Requirement already satisfied: pypdf2>=2.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (3.0.1)\n", + "Requirement already satisfied: python-docx>=0.8.11 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (1.2.0)\n", + "Requirement already satisfied: openpyxl>=3.0.10 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (3.1.5)\n", + "Requirement already satisfied: pillow>=9.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (10.4.0)\n", + "Requirement already satisfied: librosa>=0.9.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (0.11.0)\n", + "Requirement already satisfied: opencv-python>=4.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (4.9.0.80)\n", + "Requirement already satisfied: faiss-cpu>=1.7.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (1.13.2)\n", + "Requirement already satisfied: fastembed>=0.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (0.7.3)\n", + "Requirement already satisfied: onnxruntime>=1.17.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (1.22.1)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (0.21.4)\n", + "Requirement already satisfied: pydantic>=2.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (2.12.3)\n", + "Requirement already satisfied: click>=8.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (8.2.1)\n", + "Requirement already satisfied: rich>=12.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (14.2.0)\n", + "Requirement already satisfied: tqdm>=4.64.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (4.67.1)\n", + "Requirement already satisfied: pyyaml>=6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (6.0.1)\n", + "Requirement already satisfied: toml>=0.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (0.10.2)\n", + "Requirement already satisfied: python-dotenv>=0.20.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (1.1.1)\n", + "Requirement already satisfied: loguru>=0.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (0.7.3)\n", + "Requirement already satisfied: structlog>=22.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (24.4.0)\n", + "Requirement already satisfied: gensim>=4.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica==0.3.0a0) (4.4.0)\n", + "Requirement already satisfied: soupsieve>=1.6.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from beautifulsoup4>=4.11.0->semantica==0.3.0a0) (2.5)\n", + "Requirement already satisfied: typing-extensions>=4.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from beautifulsoup4>=4.11.0->semantica==0.3.0a0) (4.15.0)\n", + "Requirement already satisfied: colorama in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from click>=8.1.0->semantica==0.3.0a0) (0.4.6)\n", + "Requirement already satisfied: packaging in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from faiss-cpu>=1.7.0->semantica==0.3.0a0) (24.2)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.20 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from fastembed>=0.2.0->semantica==0.3.0a0) (0.30.2)\n", + "Requirement already satisfied: mmh3<6.0.0,>=4.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from fastembed>=0.2.0->semantica==0.3.0a0) (5.2.0)\n", + "Requirement already satisfied: py-rust-stemmers<0.2.0,>=0.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from fastembed>=0.2.0->semantica==0.3.0a0) (0.1.5)\n", + "Requirement already satisfied: filelock in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from huggingface-hub<1.0,>=0.20->fastembed>=0.2.0->semantica==0.3.0a0) (3.16.1)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from huggingface-hub<1.0,>=0.20->fastembed>=0.2.0->semantica==0.3.0a0) (2023.10.0)\n", + "Requirement already satisfied: win32-setctime>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from loguru>=0.6.0->semantica==0.3.0a0) (1.1.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests>=2.28.0->semantica==0.3.0a0) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests>=2.28.0->semantica==0.3.0a0) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests>=2.28.0->semantica==0.3.0a0) (1.26.20)\n", + "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests>=2.28.0->semantica==0.3.0a0) (2025.8.3)\n", + "Requirement already satisfied: smart_open>=1.8.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gensim>=4.3.0->semantica==0.3.0a0) (7.1.0)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from GitPython>=3.1.30->semantica==0.3.0a0) (4.0.11)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gitdb<5,>=4.0.1->GitPython>=3.1.30->semantica==0.3.0a0) (5.0.1)\n", + "Requirement already satisfied: comm>=0.1.3 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from ipywidgets>=8.0.0->semantica==0.3.0a0) (0.2.0)\n", + "Requirement already satisfied: ipython>=6.1.0 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from ipywidgets>=8.0.0->semantica==0.3.0a0) (8.17.2)\n", + "Requirement already satisfied: traitlets>=4.3.1 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from ipywidgets>=8.0.0->semantica==0.3.0a0) (5.13.0)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.9 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipywidgets>=8.0.0->semantica==0.3.0a0) (4.0.9)\n", + "Requirement already satisfied: jupyterlab-widgets~=3.0.9 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipywidgets>=8.0.0->semantica==0.3.0a0) (3.0.9)\n", + "Requirement already satisfied: decorator in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (0.19.1)\n", + "Requirement already satisfied: matplotlib-inline in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (0.1.6)\n", + "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (3.0.40)\n", + "Requirement already satisfied: pygments>=2.4.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (2.19.2)\n", + "Requirement already satisfied: stack-data in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (0.6.3)\n", + "Requirement already satisfied: wcwidth in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (0.2.9)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (0.8.3)\n", + "Requirement already satisfied: audioread>=2.1.9 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica==0.3.0a0) (3.0.1)\n", + "Requirement already satisfied: numba>=0.51.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica==0.3.0a0) (0.61.2)\n", + "Requirement already satisfied: joblib>=1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica==0.3.0a0) (1.3.2)\n", + "Requirement already satisfied: soundfile>=0.12.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica==0.3.0a0) (0.13.1)\n", + "Requirement already satisfied: pooch>=1.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica==0.3.0a0) (1.8.2)\n", + "Requirement already satisfied: soxr>=0.3.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica==0.3.0a0) (0.5.0.post1)\n", + "Requirement already satisfied: lazy_loader>=0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica==0.3.0a0) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica==0.3.0a0) (1.1.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica==0.3.0a0) (1.3.2)\n", + "Requirement already satisfied: cycler>=0.10 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica==0.3.0a0) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica==0.3.0a0) (4.57.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica==0.3.0a0) (1.4.8)\n", + "Requirement already satisfied: pyparsing>=3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica==0.3.0a0) (3.2.0)\n", + "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica==0.3.0a0) (2.9.0.post0)\n", + "Requirement already satisfied: llvmlite<0.45,>=0.44.0dev0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from numba>=0.51.0->librosa>=0.9.0->semantica==0.3.0a0) (0.44.0)\n", + "Requirement already satisfied: coloredlogs in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from onnxruntime>=1.17.0->semantica==0.3.0a0) (15.0.1)\n", + "Requirement already satisfied: flatbuffers in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from onnxruntime>=1.17.0->semantica==0.3.0a0) (23.5.26)\n", + "Requirement already satisfied: sympy in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from onnxruntime>=1.17.0->semantica==0.3.0a0) (1.13.3)\n", + "Requirement already satisfied: et-xmlfile in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from openpyxl>=3.0.10->semantica==0.3.0a0) (2.0.0)\n", + "Requirement already satisfied: pytz>=2020.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas>=1.3.0->semantica==0.3.0a0) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas>=1.3.0->semantica==0.3.0a0) (2025.2)\n", + "Requirement already satisfied: narwhals>=1.15.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from plotly>=5.10.0->semantica==0.3.0a0) (1.25.2)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pooch>=1.1->librosa>=0.9.0->semantica==0.3.0a0) (4.5.1)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=2.0.0->semantica==0.3.0a0) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.41.4 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=2.0.0->semantica==0.3.0a0) (2.41.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=2.0.0->semantica==0.3.0a0) (0.4.2)\n", + "Requirement already satisfied: six>=1.5 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from python-dateutil>=2.7->matplotlib>=3.5.0->semantica==0.3.0a0) (1.16.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich>=12.5.0->semantica==0.3.0a0) (3.0.0)\n", + "Requirement already satisfied: mdurl~=0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from markdown-it-py>=2.2.0->rich>=12.5.0->semantica==0.3.0a0) (0.1.2)\n", + "Requirement already satisfied: threadpoolctl>=3.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from scikit-learn>=1.0.0->semantica==0.3.0a0) (3.2.0)\n", + "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from transformers>=4.20.0->semantica==0.3.0a0) (2024.11.6)\n", + "Requirement already satisfied: safetensors>=0.4.3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from transformers>=4.20.0->semantica==0.3.0a0) (0.5.3)\n", + "Requirement already satisfied: wrapt in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from smart_open>=1.8.1->gensim>=4.3.0->semantica==0.3.0a0) (1.17.2)\n", + "Requirement already satisfied: cffi>=1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from soundfile>=0.12.1->librosa>=0.9.0->semantica==0.3.0a0) (1.17.1)\n", + "Requirement already satisfied: pycparser in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.9.0->semantica==0.3.0a0) (2.22)\n", + "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (3.0.12)\n", + "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (1.0.5)\n", + "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (1.0.15)\n", + "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (2.0.13)\n", + "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (3.0.12)\n", + "Requirement already satisfied: thinc<8.4.0,>=8.3.4 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (8.3.10)\n", + "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (1.1.3)\n", + "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (2.5.2)\n", + "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (2.0.10)\n", + "Requirement already satisfied: weasel<0.5.0,>=0.4.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (0.4.3)\n", + "Requirement already satisfied: typer-slim<1.0.0,>=0.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (0.20.0)\n", + "Requirement already satisfied: jinja2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (3.1.6)\n", + "Requirement already satisfied: setuptools in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica==0.3.0a0) (80.9.0)\n", + "Requirement already satisfied: blis<1.4.0,>=1.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from thinc<8.4.0,>=8.3.4->spacy>=3.4.0->semantica==0.3.0a0) (1.3.3)\n", + "Requirement already satisfied: confection<1.0.0,>=0.0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from thinc<8.4.0,>=8.3.4->spacy>=3.4.0->semantica==0.3.0a0) (0.1.5)\n", + "Requirement already satisfied: cloudpathlib<1.0.0,>=0.7.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from weasel<0.5.0,>=0.4.2->spacy>=3.4.0->semantica==0.3.0a0) (0.23.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from sympy->onnxruntime>=1.17.0->semantica==0.3.0a0) (1.3.0)\n", + "Requirement already satisfied: pynndescent>=0.5 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from umap-learn>=0.5.0->semantica==0.3.0a0) (0.5.13)\n", + "Requirement already satisfied: humanfriendly>=9.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from coloredlogs->onnxruntime>=1.17.0->semantica==0.3.0a0) (10.0)\n", + "Requirement already satisfied: pyreadline3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from humanfriendly>=9.1->coloredlogs->onnxruntime>=1.17.0->semantica==0.3.0a0) (3.4.1)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jinja2->spacy>=3.4.0->semantica==0.3.0a0) (2.1.3)\n", + "Requirement already satisfied: executing>=1.2.0 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (2.0.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (2.4.1)\n", + "Requirement already satisfied: pure-eval in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=8.0.0->semantica==0.3.0a0) (0.2.2)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "\n", + "[notice] A new release of pip is available: 25.3 -> 26.0.1\n", + "[notice] To update, run: C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\python.exe -m pip install --upgrade pip\n" + ] + } + ], "source": [ "!pip install semantica==0.3.0a0" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "9ad777f9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Setup completed:\n", + "Base Directory: c:\\Users\\Mohd Kaif\\semantica\\cookbook\\use_cases\\capability_gap_defense\n", + "Data Directory: c:\\Users\\Mohd Kaif\\semantica\\cookbook\\use_cases\\capability_gap_defense\\cookbook\\use_cases\\capability_gap_defense\\data\n", + "Output Directory: c:\\Users\\Mohd Kaif\\semantica\\cookbook\\use_cases\\capability_gap_defense\\cookbook\\use_cases\\capability_gap_defense\\outputs\n" + ] + }, + { + "data": { + "text/plain": [ + "(WindowsPath('c:/Users/Mohd Kaif/semantica/cookbook/use_cases/capability_gap_defense'),\n", + " WindowsPath('c:/Users/Mohd Kaif/semantica/cookbook/use_cases/capability_gap_defense/cookbook/use_cases/capability_gap_defense/data'),\n", + " WindowsPath('c:/Users/Mohd Kaif/semantica/cookbook/use_cases/capability_gap_defense/cookbook/use_cases/capability_gap_defense/outputs'))" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Setup and Configuration (Consolidated)\n", "from pathlib import Path\n", @@ -195,10 +375,28 @@ "execution_count": null, "id": "89f73b8f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'Type' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[3], line 4\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;66;03m# Multi-Source Data Ingestion\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;66;03m# Import ingestion classes where they are used\u001b[39;00m\n\u001b[1;32m----> 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msemantica\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mingest\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m FileIngestor, WebIngestor, OntologyIngestor\n\u001b[0;32m 6\u001b[0m \u001b[38;5;66;03m# Initialize ingestion classes\u001b[39;00m\n\u001b[0;32m 7\u001b[0m file_ingestor \u001b[38;5;241m=\u001b[39m FileIngestor()\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\semantica\\__init__.py:24\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, List, Optional, Union\n\u001b[0;32m 23\u001b[0m \u001b[38;5;66;03m# Core imports\u001b[39;00m\n\u001b[1;32m---> 24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Config, ConfigManager, LifecycleManager, PluginRegistry, Semantica\n\u001b[0;32m 26\u001b[0m \u001b[38;5;66;03m# KG Quality Assurance\u001b[39;00m\n\u001b[0;32m 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mkg_qa\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 28\u001b[0m AutomatedFixer,\n\u001b[0;32m 29\u001b[0m AutoMerger,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 42\u001b[0m ValidationEngine,\n\u001b[0;32m 43\u001b[0m )\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\semantica\\core\\__init__.py:55\u001b[0m\n\u001b[0;32m 52\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpathlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Path\n\u001b[0;32m 53\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, List, Optional, Union\n\u001b[1;32m---> 55\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig_manager\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Config, ConfigManager\n\u001b[0;32m 56\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlifecycle\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m HealthStatus, LifecycleManager, SystemState\n\u001b[0;32m 57\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmethods\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 58\u001b[0m build_knowledge_base,\n\u001b[0;32m 59\u001b[0m get_orchestration_method,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 63\u001b[0m run_pipeline,\n\u001b[0;32m 64\u001b[0m )\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\semantica\\core\\config_manager.py:61\u001b[0m\n\u001b[0;32m 57\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Any, Dict, Optional, Union\n\u001b[0;32m 59\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01myaml\u001b[39;00m\n\u001b[1;32m---> 61\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconstants\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DEFAULT_CONFIG\n\u001b[0;32m 62\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ConfigurationError\n\u001b[0;32m 63\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhelpers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 64\u001b[0m get_nested_value,\n\u001b[0;32m 65\u001b[0m merge_dicts,\n\u001b[0;32m 66\u001b[0m read_json_file,\n\u001b[0;32m 67\u001b[0m set_nested_value,\n\u001b[0;32m 68\u001b[0m )\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\semantica\\utils\\__init__.py:70\u001b[0m\n\u001b[0;32m 40\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconstants\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 41\u001b[0m API_ENDPOINTS,\n\u001b[0;32m 42\u001b[0m CACHE_CONFIG,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 59\u001b[0m SUPPORTED_VIDEO_FORMATS,\n\u001b[0;32m 60\u001b[0m )\n\u001b[0;32m 61\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 62\u001b[0m ConfigurationError,\n\u001b[0;32m 63\u001b[0m ProcessingError,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 68\u001b[0m handle_exception,\n\u001b[0;32m 69\u001b[0m )\n\u001b[1;32m---> 70\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhelpers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 71\u001b[0m chunk_list,\n\u001b[0;32m 72\u001b[0m clean_text,\n\u001b[0;32m 73\u001b[0m ensure_directory,\n\u001b[0;32m 74\u001b[0m flatten_dict,\n\u001b[0;32m 75\u001b[0m format_data,\n\u001b[0;32m 76\u001b[0m format_timestamp,\n\u001b[0;32m 77\u001b[0m get_file_size,\n\u001b[0;32m 78\u001b[0m get_nested_value,\n\u001b[0;32m 79\u001b[0m hash_data,\n\u001b[0;32m 80\u001b[0m merge_dicts,\n\u001b[0;32m 81\u001b[0m normalize_entities,\n\u001b[0;32m 82\u001b[0m parse_timestamp,\n\u001b[0;32m 83\u001b[0m read_json_file,\n\u001b[0;32m 84\u001b[0m retry_on_error,\n\u001b[0;32m 85\u001b[0m safe_filename,\n\u001b[0;32m 86\u001b[0m set_nested_value,\n\u001b[0;32m 87\u001b[0m write_json_file,\n\u001b[0;32m 88\u001b[0m )\n\u001b[0;32m 89\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlogging\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 90\u001b[0m get_logger,\n\u001b[0;32m 91\u001b[0m log_data_quality,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 95\u001b[0m setup_logging,\n\u001b[0;32m 96\u001b[0m )\n\u001b[0;32m 97\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprogress_tracker\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m 98\u001b[0m ConsoleProgressDisplay,\n\u001b[0;32m 99\u001b[0m FileProgressDisplay,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 106\u001b[0m track_progress,\n\u001b[0;32m 107\u001b[0m )\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\semantica\\utils\\helpers.py:473\u001b[0m\n\u001b[0;32m 464\u001b[0m d \u001b[38;5;241m=\u001b[39m d[key]\n\u001b[0;32m 466\u001b[0m d[keys[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]] \u001b[38;5;241m=\u001b[39m value\n\u001b[0;32m 469\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mretry_on_error\u001b[39m(\n\u001b[0;32m 470\u001b[0m max_retries: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m3\u001b[39m,\n\u001b[0;32m 471\u001b[0m delay: \u001b[38;5;28mfloat\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1.0\u001b[39m,\n\u001b[0;32m 472\u001b[0m backoff_factor: \u001b[38;5;28mfloat\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m2.0\u001b[39m,\n\u001b[1;32m--> 473\u001b[0m exceptions: Tuple[\u001b[43mType\u001b[49m[\u001b[38;5;167;01mException\u001b[39;00m], \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (\u001b[38;5;167;01mException\u001b[39;00m,),\n\u001b[0;32m 474\u001b[0m ):\n\u001b[0;32m 475\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 476\u001b[0m \u001b[38;5;124;03m Decorator for retrying function on error.\u001b[39;00m\n\u001b[0;32m 477\u001b[0m \n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 485\u001b[0m \u001b[38;5;124;03m Decorator function\u001b[39;00m\n\u001b[0;32m 486\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m 487\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mfunctools\u001b[39;00m\n", + "\u001b[1;31mNameError\u001b[0m: name 'Type' is not defined" + ] + } + ], "source": [ "# Multi-Source Data Ingestion\n", "# Import ingestion classes where they are used\n", + "# NOTE: The Type import issue has been fixed in semantica/utils/helpers.py\n", "\n", "from semantica.ingest import FileIngestor, WebIngestor, OntologyIngestor\n", "\n", @@ -2617,8 +2815,16 @@ "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" } }, "nbformat": 4, diff --git a/semantica/core/config_manager.py b/semantica/core/config_manager.py index a59bf31b..219bd157 100644 --- a/semantica/core/config_manager.py +++ b/semantica/core/config_manager.py @@ -54,7 +54,7 @@ License: MIT import json import os from pathlib import Path -from typing import Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional, Union import yaml diff --git a/semantica/export/__init__.py b/semantica/export/__init__.py index 51e304ac..4366544f 100644 --- a/semantica/export/__init__.py +++ b/semantica/export/__init__.py @@ -3,13 +3,16 @@ Export and Reporting Module This module provides comprehensive export and reporting capabilities for the Semantica framework, supporting multiple formats and use cases including RDF, -JSON, CSV, Graph, YAML, OWL, Vector, and LPG (Labeled Property Graph) formats. +JSON, CSV, Graph, YAML, OWL, Vector, LPG (Labeled Property Graph), and +ArangoDB AQL formats. Algorithms Used: RDF Export: - RDF Serialization: Multiple format serialization (Turtle, RDF/XML, JSON-LD, N-Triples, N3) + - RDF Serialization: Multiple format serialization (Turtle, RDF/XML, + JSON-LD, N-Triples, N3) - Namespace Management: Namespace registration, conflict resolution, declaration generation - RDF Validation: RDF syntax validation, triplet validation, @@ -21,6 +24,8 @@ RDF Export: LPG (Labeled Property Graph) Export: - LPG Serialization: Labeled Property Graph format for Neo4j, Memgraph, and similar databases + - LPG Serialization: Labeled Property Graph format for Neo4j, Memgraph, + and similar databases - Node Label Assignment: Entity type to node label mapping - Relationship Type Mapping: Relationship type to edge label conversion - Property Serialization: Entity and relationship properties to LPG property format @@ -28,11 +33,21 @@ LPG (Labeled Property Graph) Export: - Batch Node/Relationship Export: Efficient batch processing for large graphs - Index Generation: Index and constraint generation for graph databases +ArangoDB AQL Export: + - AQL Serialization: AQL INSERT statements for ArangoDB multi-model databases + - Vertex Collection Export: Entity to vertex collection conversion + - Edge Collection Export: Relationship to edge collection conversion + - Key Sanitization: Automatic sanitization of keys for ArangoDB compliance + - Batch Insert Generation: Efficient batch INSERT operations for large graphs + - Configurable Collections: Support for custom vertex and edge collection names + - Property Preservation: Full preservation of entity and relationship properties + JSON/JSON-LD Export: - JSON Serialization: Standard JSON serialization with configurable indentation - JSON-LD Context Management: @context generation and management - Knowledge Graph Serialization: Graph structure to JSON/JSON-LD conversion - - Metadata Embedding: Provenance and metadata serialization in JSON structure + - Metadata Embedding: Provenance and metadata serialization in JSON + structure - Pretty Printing: Formatted JSON output with indentation CSV Export: @@ -41,6 +56,8 @@ CSV Export: - Delimiter Handling: Configurable delimiter support (comma, tab, semicolon) - Header Generation: Automatic CSV header row generation - Metadata Serialization: JSON string serialization for complex metadata fields + - Metadata Serialization: JSON string serialization for complex + metadata fields - Multi-file Export: Knowledge graph split into multiple CSV files (entities, relationships) @@ -62,7 +79,8 @@ OWL Export: - Turtle Serialization: OWL in Turtle format - Class Hierarchy Export: Class definition and hierarchy serialization - Property Export: Object and data property definition export - - OWL 2.0 Feature Support: Advanced OWL features (cardinality, restrictions, etc.) + - OWL 2.0 Feature Support: Advanced OWL features (cardinality, + restrictions, etc.) - Ontology Validation: OWL syntax and semantic validation Parquet Export: @@ -92,6 +110,8 @@ Report Generation: Key Features: - Multiple export formats (RDF, JSON, CSV, Graph, YAML, OWL, Vector, LPG, Parquet) + - Multiple export formats (RDF, JSON, CSV, Graph, YAML, OWL, Vector, + LPG, ArangoDB AQL) - Knowledge graph export with format auto-detection - Report generation (HTML, Markdown, JSON, Text) - Vector store integration @@ -109,7 +129,10 @@ Main Classes: - YAMLExporter: YAML format export for semantic networks - OWLExporter: OWL format export for ontologies - VectorExporter: Vector embedding export for vector stores - - LPGExporter: LPG format export for Neo4j, Memgraph, and similar databases + - LPGExporter: LPG format export for Neo4j, Memgraph, and similar + databases + - ArangoAQLExporter: ArangoDB AQL format export for multi-model + databases - ReportGenerator: Report generation (HTML, Markdown, JSON, Text) - MethodRegistry: Registry for custom export methods - ExportConfig: Configuration manager for export module @@ -138,12 +161,13 @@ Author: Semantica Contributors License: MIT """ +from .arango_aql_exporter import ArangoAQLExporter from .config import ExportConfig, export_config try: from .arrow_exporter import ArrowExporter except ImportError: - # ArrowExporter is not available in CI environment - create a dummy class + # ArrowExporter is not available in CI environment - dummy class class ArrowExporter: def __init__(self, *args, **kwargs): pass @@ -157,6 +181,7 @@ from .graph_exporter import GraphExporter from .json_exporter import JSONExporter from .lpg_exporter import LPGExporter from .methods import ( + export_arango, export_arrow, export_csv, export_graph, @@ -194,6 +219,7 @@ from .yaml_exporter import SemanticNetworkYAMLExporter, YAMLSchemaExporter __all__ = [ # Core Exporters "ArrowExporter", + "ArangoAQLExporter", "RDFExporter", "RDFSerializer", "RDFValidator", @@ -221,6 +247,7 @@ __all__ = [ "export_owl", "export_vector", "export_lpg", + "export_arango", "generate_report", "get_export_method", "list_available_methods", diff --git a/semantica/export/arango_aql_exporter.py b/semantica/export/arango_aql_exporter.py new file mode 100644 index 00000000..712cb1d3 --- /dev/null +++ b/semantica/export/arango_aql_exporter.py @@ -0,0 +1,641 @@ +""" +ArangoDB AQL Export Module + +This module provides comprehensive AQL export capabilities for the Semantica framework, +enabling export to ArangoDB multi-model graph databases. + +Key Features: + - AQL format export for ArangoDB + - INSERT statement generation for vertex and edge collections + - Configurable collection names + - Entity and relationship identifier preservation + - Batch insert support for performance + - Proper string escaping and special character handling + +Example Usage: + >>> from semantica.export import ArangoAQLExporter + >>> exporter = ArangoAQLExporter() + >>> exporter.export_knowledge_graph(kg, "output.aql") + >>> # With custom collection names + >>> exporter = ArangoAQLExporter( + ... vertex_collection="nodes", + ... edge_collection="links" + ... ) + >>> exporter.export(kg, "graph.aql") +""" + +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from ..utils.helpers import ensure_directory +from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker + + +class ArangoAQLExporter: + """ + ArangoDB AQL exporter for knowledge graphs. + + This class provides comprehensive AQL export functionality for knowledge graphs, + supporting export to ArangoDB multi-model databases via AQL INSERT statements. + + Features: + - AQL INSERT statement generation for vertices and edges + - Configurable collection names + - Entity and relationship identifier preservation + - Batch insert support for performance + - Proper string escaping and special character handling + - Support for nested properties via JSON serialization + + Example Usage: + >>> exporter = ArangoAQLExporter( + ... vertex_collection="entities", + ... edge_collection="relationships" + ... ) + >>> exporter.export_knowledge_graph(kg, "output.aql") + """ + + def __init__( + self, + vertex_collection: str = "vertices", + edge_collection: str = "edges", + batch_size: int = 1000, + include_collection_creation: bool = True, + config: Optional[Dict[str, Any]] = None, + **kwargs, + ): + """ + Initialize ArangoDB AQL exporter. + + Sets up the exporter with collection names and batch processing options. + + Args: + vertex_collection: Name of the vertex collection + (default: "vertices") + edge_collection: Name of the edge collection (default: "edges") + batch_size: Batch size for INSERT operations (default: 1000) + include_collection_creation: Whether to include collection + creation statements (default: True) + config: Optional configuration dictionary (merged with kwargs) + **kwargs: Additional configuration options + """ + self.logger = get_logger("arango_aql_exporter") + self.config = config or {} + self.config.update(kwargs) + + # Validate collection names + self._validate_collection_name(vertex_collection, "vertex_collection") + self._validate_collection_name(edge_collection, "edge_collection") + + # AQL export configuration + self.vertex_collection = vertex_collection + self.edge_collection = edge_collection + self.batch_size = batch_size + self.include_collection_creation = include_collection_creation + + # Initialize progress tracker + self.progress_tracker = get_progress_tracker() + + self.logger.debug( + f"ArangoDB AQL exporter initialized: " + f"vertex_collection={vertex_collection}, " + f"edge_collection={edge_collection}, batch_size={batch_size}" + ) + + def export( + self, knowledge_graph: Dict[str, Any], file_path: Union[str, Path], **options + ) -> None: + """ + Export knowledge graph to ArangoDB AQL format. + + This method exports a knowledge graph to AQL INSERT statements that can + be imported into ArangoDB multi-model databases. + + Args: + knowledge_graph: Knowledge graph dictionary containing: + - entities: List of entity dictionaries + - relationships: List of relationship dictionaries + - nodes: List of node dictionaries (optional, alternative to + entities) + - edges: List of edge dictionaries (optional, alternative to + relationships) + file_path: Output AQL file path + **options: Additional export options: + - vertex_collection: Override default vertex collection name + - edge_collection: Override default edge collection name + + Example: + >>> kg = { + ... "entities": [...], + ... "relationships": [...] + ... } + >>> exporter.export(kg, "graph.aql") + """ + file_path = Path(file_path) + ensure_directory(file_path.parent) + + tracking_id = self.progress_tracker.start_tracking( + file=str(file_path), + module="export", + submodule="ArangoAQLExporter", + message=f"Exporting knowledge graph to ArangoDB AQL format: {file_path}", + ) + + try: + # Override collection names if provided in options + vertex_collection = options.pop("vertex_collection", self.vertex_collection) + edge_collection = options.pop("edge_collection", self.edge_collection) + + # Validate overridden collection names + if vertex_collection != self.vertex_collection: + self._validate_collection_name(vertex_collection, "vertex_collection") + if edge_collection != self.edge_collection: + self._validate_collection_name(edge_collection, "edge_collection") + + # Generate AQL statements + aql_statements = self._generate_aql_statements( + knowledge_graph, vertex_collection, edge_collection, **options + ) + + # Write to file + with open(file_path, "w", encoding="utf-8") as f: + f.write("\n".join(aql_statements)) + + self.progress_tracker.stop_tracking( + tracking_id, + status="completed", + message=f"Exported {len(aql_statements)} AQL statements", + ) + self.logger.info( + f"Exported knowledge graph to ArangoDB AQL format: " f"{file_path}" + ) + + except Exception as e: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message=str(e) + ) + raise + + def _generate_aql_statements( + self, + knowledge_graph: Dict[str, Any], + vertex_collection: str, + edge_collection: str, + **options, + ) -> List[str]: + """ + Generate AQL INSERT statements from knowledge graph. + + Args: + knowledge_graph: Knowledge graph dictionary + vertex_collection: Vertex collection name + edge_collection: Edge collection name + **options: Additional options + + Returns: + List of AQL statement strings + """ + statements = [] + + # Add collection creation statements if requested + if self.include_collection_creation: + statements.extend( + self._generate_collection_creation(vertex_collection, edge_collection) + ) + + # Extract entities and relationships + entities = knowledge_graph.get("entities", []) + relationships = knowledge_graph.get("relationships", []) + nodes = knowledge_graph.get("nodes", entities) + edges = knowledge_graph.get("edges", relationships) + + # Use nodes/edges if entities/relationships are empty + if not entities and nodes: + entities = nodes + if not relationships and edges: + relationships = edges + + # Generate vertex INSERT statements + vertex_statements = self._generate_vertex_inserts(entities, vertex_collection) + statements.extend(vertex_statements) + + # Generate edge INSERT statements + edge_statements = self._generate_edge_inserts( + relationships, edge_collection, vertex_collection + ) + statements.extend(edge_statements) + + return statements + + def _generate_collection_creation( + self, vertex_collection: str, edge_collection: str + ) -> List[str]: + """ + Generate AQL collection creation statements. + + Args: + vertex_collection: Vertex collection name + edge_collection: Edge collection name + + Returns: + List of collection creation statements + """ + statements = [ + "// Create vertex collection if it doesn't exist", + f"// db._createDocumentCollection('{vertex_collection}');", + "", + "// Create edge collection if it doesn't exist", + f"// db._createEdgeCollection('{edge_collection}');", + "", + ] + return statements + + def _generate_vertex_inserts( + self, vertices: List[Dict[str, Any]], collection: str + ) -> List[str]: + """ + Generate AQL INSERT statements for vertices. + + Args: + vertices: List of vertex/entity dictionaries + collection: Vertex collection name + + Returns: + List of AQL INSERT statements + """ + statements = [] + + # Add header comment + statements.append(f"// Inserting {len(vertices)} vertices into {collection}") + statements.append("") + + # Process vertices in batches + for i in range(0, len(vertices), self.batch_size): + batch = vertices[i : i + self.batch_size] + batch_statement = self._create_vertex_batch_insert(batch, collection) + statements.append(batch_statement) + statements.append("") + + return statements + + def _create_vertex_batch_insert( + self, vertices: List[Dict[str, Any]], collection: str + ) -> str: + """ + Create a batch INSERT statement for vertices. + + Args: + vertices: Batch of vertex dictionaries + collection: Vertex collection name + + Returns: + AQL INSERT statement + """ + if not vertices: + return "" + + # Build document list + documents = [] + for idx, vertex in enumerate(vertices): + doc = self._convert_vertex_to_document(vertex, idx) + documents.append(doc) + + # Format as AQL + docs_json = json.dumps(documents, indent=2, ensure_ascii=False) + statement = f"FOR doc IN {docs_json}\n INSERT doc INTO {collection}" + + return statement + + def _convert_vertex_to_document( + self, vertex: Dict[str, Any], idx: int + ) -> Dict[str, Any]: + """ + Convert a vertex/entity to an ArangoDB document. + + Additional properties from the input entity are preserved as-is in the + document root (not flattened or merged). Nested dictionaries and lists + are preserved as JSON-serializable structures. The 'properties' field, + if present, is also preserved as-is rather than being flattened into + the document root. + + Args: + vertex: Vertex/entity dictionary + idx: Index for generating fallback IDs + + Returns: + ArangoDB document dictionary + """ + document = {} + + # Set _key from id or generate one + vertex_id = vertex.get("id") or vertex.get("entity_id") or f"vertex_{idx}" + document["_key"] = self._sanitize_key(str(vertex_id)) + + # Add original ID if different from _key + if str(vertex_id) != document["_key"]: + document["original_id"] = str(vertex_id) + + # Add type/label information + vertex_type = vertex.get("type") or vertex.get("entity_type", "Entity") + document["type"] = vertex_type + + # Add name/label + label = ( + vertex.get("label") + or vertex.get("name") + or vertex.get("text") + or document["_key"] + ) + document["name"] = label + + # Add all other properties + for key, value in vertex.items(): + if key not in [ + "_key", + "_id", + "_rev", + "id", + "entity_id", + "type", + "entity_type", + "label", + "name", + "text", + ]: + # Handle nested dictionaries and lists + if isinstance(value, (dict, list)): + document[key] = value + elif value is not None: + document[key] = value + + return document + + def _generate_edge_inserts( + self, edges: List[Dict[str, Any]], collection: str, vertex_collection: str + ) -> List[str]: + """ + Generate AQL INSERT statements for edges. + + Args: + edges: List of edge/relationship dictionaries + collection: Edge collection name + vertex_collection: Vertex collection name for _from/_to references + + Returns: + List of AQL INSERT statements + """ + statements = [] + + # Add header comment + statements.append( + f"// Attempting to insert {len(edges)} edges into {collection}" + ) + statements.append("") + + # Process edges in batches + for i in range(0, len(edges), self.batch_size): + batch = edges[i : i + self.batch_size] + batch_statement = self._create_edge_batch_insert( + batch, collection, vertex_collection + ) + if batch_statement: # Only add non-empty statements + statements.append(batch_statement) + statements.append("") + + return statements + + def _create_edge_batch_insert( + self, edges: List[Dict[str, Any]], collection: str, vertex_collection: str + ) -> str: + """ + Create a batch INSERT statement for edges. + + Args: + edges: Batch of edge dictionaries + collection: Edge collection name + vertex_collection: Vertex collection name for _from/_to references + + Returns: + AQL INSERT statement + """ + # Build document list + documents = [] + for idx, edge in enumerate(edges): + doc = self._convert_edge_to_document(edge, idx, vertex_collection) + if doc: # Only add valid edges + documents.append(doc) + + if not documents: + return "" + + # Format as AQL + docs_json = json.dumps(documents, indent=2, ensure_ascii=False) + statement = f"FOR doc IN {docs_json}\n INSERT doc INTO {collection}" + + return statement + + def _convert_edge_to_document( + self, edge: Dict[str, Any], idx: int, vertex_collection: str + ) -> Optional[Dict[str, Any]]: + """ + Convert an edge/relationship to an ArangoDB edge document. + + Args: + edge: Edge/relationship dictionary + idx: Index for generating fallback IDs + vertex_collection: Vertex collection name for _from/_to references + + Returns: + ArangoDB edge document dictionary, or None if source/target missing + """ + # Extract source and target + source_id = edge.get("source") or edge.get("source_id") + target_id = edge.get("target") or edge.get("target_id") + + if not source_id or not target_id: + self.logger.warning( + f"Skipping edge {idx}: missing source or target " + f"(source={source_id}, target={target_id})" + ) + return None + + document = {} + + # Set _key from id or generate one + edge_id = edge.get("id") or edge.get("relationship_id") or f"edge_{idx}" + document["_key"] = self._sanitize_key(str(edge_id)) + + # Set _from and _to (required for edges in ArangoDB) + document["_from"] = f"{vertex_collection}/{self._sanitize_key(str(source_id))}" + document["_to"] = f"{vertex_collection}/{self._sanitize_key(str(target_id))}" + + # Add original ID if different from _key + if str(edge_id) != document["_key"]: + document["original_id"] = str(edge_id) + + # Add relationship type + rel_type = edge.get("type") or edge.get("relationship_type", "RELATED_TO") + document["type"] = rel_type + + # Add all other properties + for key, value in edge.items(): + if key not in [ + "_key", + "_id", + "_rev", + "_from", + "_to", + "id", + "relationship_id", + "source", + "source_id", + "target", + "target_id", + "type", + "relationship_type", + ]: + # Handle nested dictionaries and lists + if isinstance(value, (dict, list)): + document[key] = value + elif value is not None: + document[key] = value + + return document + + def _validate_collection_name(self, name: str, param_name: str) -> None: + """ + Validate an ArangoDB collection name. + + ArangoDB collection names must: + - Start with a letter or underscore + - Contain only alphanumeric characters, hyphens, and underscores + - Not exceed 256 characters + + Args: + name: Collection name to validate + param_name: Parameter name for error messages + + Raises: + ValueError: If the collection name is invalid + """ + if not name: + raise ValueError(f"{param_name} cannot be empty") + + if len(name) > 256: + raise ValueError( + f"{param_name} '{name}' exceeds maximum length of 256 characters" + ) + + # Check first character + if not (name[0].isalpha() or name[0] == "_"): + raise ValueError( + f"{param_name} '{name}' must start with a letter or underscore" + ) + + # Check remaining characters + for char in name: + if not (char.isalnum() or char in ("-", "_")): + raise ValueError( + f"{param_name} '{name}' contains invalid character " + f"'{char}'. Only alphanumeric characters, hyphens, and " + "underscores are allowed." + ) + + def _sanitize_key(self, key: str) -> str: + """ + Sanitize a key for use as ArangoDB _key. + + ArangoDB _key must contain only alphanumeric characters, hyphens, + and underscores. It cannot start with an underscore (unless it's + a system collection). + + Args: + key: Original key string + + Returns: + Sanitized key string + """ + # Replace invalid characters with underscores + sanitized = "" + for char in key: + if char.isalnum() or char in ("-", "_"): + sanitized += char + else: + sanitized += "_" + + # Ensure key doesn't start with underscore + if sanitized.startswith("_"): + sanitized = "k" + sanitized + + # Ensure key is not empty + if not sanitized: + sanitized = "key" + + return sanitized + + def export_knowledge_graph( + self, knowledge_graph: Dict[str, Any], file_path: Union[str, Path], **options + ) -> None: + """ + Export knowledge graph to ArangoDB AQL format. + + Convenience method that calls export(). + + Args: + knowledge_graph: Knowledge graph dictionary + file_path: Output AQL file path + **options: Additional export options + + Example: + >>> kg = { + ... "entities": [...], + ... "relationships": [...] + ... } + >>> exporter.export_knowledge_graph(kg, "output.aql") + """ + self.export(knowledge_graph, file_path, **options) + + def export_entities( + self, entities: List[Dict[str, Any]], file_path: Union[str, Path], **options + ) -> None: + """ + Export entities to ArangoDB AQL format (vertices only). + + Args: + entities: List of entity dictionaries + file_path: Output AQL file path + **options: Additional export options + + Example: + >>> entities = [ + ... {"id": "e1", "type": "Person", "name": "Alice"}, + ... {"id": "e2", "type": "Organization", "name": "Acme Corp"} + ... ] + >>> exporter.export_entities(entities, "entities.aql") + """ + knowledge_graph = {"entities": entities, "relationships": []} + self.export(knowledge_graph, file_path, **options) + + def export_relationships( + self, + relationships: List[Dict[str, Any]], + file_path: Union[str, Path], + **options, + ) -> None: + """ + Export relationships to ArangoDB AQL format (edges only). + + Args: + relationships: List of relationship dictionaries + file_path: Output AQL file path + **options: Additional export options + + Example: + >>> relationships = [ + ... {"id": "r1", "source": "e1", "target": "e2", "type": "WORKS_FOR"} + ... ] + >>> exporter.export_relationships(relationships, "relationships.aql") + """ + knowledge_graph = {"entities": [], "relationships": relationships} + self.export(knowledge_graph, file_path, **options) diff --git a/semantica/export/arrow_exporter.py b/semantica/export/arrow_exporter.py index 85e4b1cf..d2e3a4d1 100644 --- a/semantica/export/arrow_exporter.py +++ b/semantica/export/arrow_exporter.py @@ -24,6 +24,8 @@ Author: Semantica Contributors License: MIT """ +from __future__ import annotations + import json from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union @@ -31,6 +33,10 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union if TYPE_CHECKING: import pyarrow as pa # noqa: F401 + +if TYPE_CHECKING: + import pyarrow as pa + try: import pyarrow as pa # noqa: F811 import pyarrow.ipc as ipc @@ -82,12 +88,59 @@ except ImportError: ARROW_AVAILABLE = False ENTITY_SCHEMA = None RELATIONSHIP_SCHEMA = None + pa = None # Set pa to None when not available from ..utils.exceptions import ProcessingError, ValidationError from ..utils.helpers import ensure_directory from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +# Explicit Arrow Schemas (no inference) +if ARROW_AVAILABLE: + ENTITY_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False), + pa.field("text", pa.string(), nullable=True), + pa.field("type", pa.string(), nullable=True), + pa.field("confidence", pa.float64(), nullable=True), + pa.field("start", pa.int64(), nullable=True), + pa.field("end", pa.int64(), nullable=True), + pa.field( + "metadata", + pa.struct( + [ + pa.field("keys", pa.list_(pa.string())), + pa.field("values", pa.list_(pa.string())), + ] + ), + nullable=True, + ), + ] + ) + + RELATIONSHIP_SCHEMA = pa.schema( + [ + pa.field("id", pa.string(), nullable=False), + pa.field("source_id", pa.string(), nullable=False), + pa.field("target_id", pa.string(), nullable=False), + pa.field("type", pa.string(), nullable=True), + pa.field("confidence", pa.float64(), nullable=True), + pa.field( + "metadata", + pa.struct( + [ + pa.field("keys", pa.list_(pa.string())), + pa.field("values", pa.list_(pa.string())), + ] + ), + nullable=True, + ), + ] + ) +else: + ENTITY_SCHEMA = None + RELATIONSHIP_SCHEMA = None + class ArrowExporter: """ @@ -206,7 +259,8 @@ class ArrowExporter: # Export each key as separate Arrow file exported_files = [] self.progress_tracker.update_tracking( - tracking_id, message=f"Exporting {len(data)} data groups..." + tracking_id, + message=f"Exporting {len(data)} data groups...", ) for key, value in data.items(): if isinstance(value, list): @@ -241,7 +295,8 @@ class ArrowExporter: elif isinstance(data, list): # Single Arrow file self.progress_tracker.update_tracking( - tracking_id, message=f"Exporting {len(data)} records..." + tracking_id, + message=f"Exporting {len(data)} records...", ) self._write_arrow(data, file_path, schema=schema, **options) self.logger.info(f"Exported Arrow to: {file_path}") @@ -322,6 +377,7 @@ class ArrowExporter: confidence = float(confidence) except (TypeError, ValueError): msg = ( + self.logger.warning( f"Invalid confidence value for entity {i}: " f"{confidence}. Setting to None." ) @@ -430,6 +486,7 @@ class ArrowExporter: confidence = float(confidence) except (TypeError, ValueError): msg = ( + self.logger.warning( f"Invalid confidence value for relationship {i}: " f"{confidence}. Setting to None." ) diff --git a/semantica/export/methods.py b/semantica/export/methods.py index 72ba7470..8fafc9af 100644 --- a/semantica/export/methods.py +++ b/semantica/export/methods.py @@ -161,6 +161,7 @@ from typing import Any, Callable, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from .arango_aql_exporter import ArangoAQLExporter from .arrow_exporter import ArrowExporter from .config import export_config from .csv_exporter import CSVExporter @@ -692,6 +693,56 @@ def export_lpg( raise +def export_arango( + knowledge_graph: Dict[str, Any], + file_path: Union[str, Path], + method: str = "default", + **kwargs, +) -> None: + """ + Export knowledge graph to ArangoDB AQL format (convenience function). + + This is a user-friendly wrapper that exports knowledge graphs to ArangoDB + multi-model databases via AQL INSERT statements. + + Args: + knowledge_graph: Knowledge graph dictionary with entities and relationships + file_path: Output AQL file path + method: Export method (default: "default") + **kwargs: Additional options passed to ArangoAQLExporter: + - vertex_collection: Override default vertex collection name + - edge_collection: Override default edge collection name + - batch_size: Batch size for INSERT operations + - include_collection_creation: Whether to include collection creation statements + + Examples: + >>> from semantica.export.methods import export_arango + >>> export_arango(kg, "output.aql") + >>> export_arango(kg, "graph.aql", vertex_collection="nodes", edge_collection="links") + """ + # Check for custom method in registry + custom_method = method_registry.get("arango", method) + if custom_method and custom_method is not export_arango: + try: + return custom_method(knowledge_graph, file_path, **kwargs) + except Exception as e: + logger.warning( + f"Custom method {method} failed: {e}, falling back to default" + ) + + try: + # Get config + config = export_config.get_method_config("arango") + config.update(kwargs) + + exporter = ArangoAQLExporter(**config) + exporter.export_knowledge_graph(knowledge_graph, file_path, **kwargs) + + except Exception as e: + logger.error(f"Failed to export ArangoDB AQL: {e}") + raise + + def generate_report( data: Dict[str, Any], file_path: Union[str, Path], @@ -774,6 +825,7 @@ def export_knowledge_graph( - "yaml", "yml": YAML exporters - "owl": OWLExporter - "cypher": LPGExporter + - "aql": ArangoAQLExporter method: Optional specific export method **kwargs: Additional options passed to exporter @@ -800,6 +852,7 @@ def export_knowledge_graph( ".yml": "yaml", ".owl": "owl-xml", ".cypher": "cypher", + ".aql": "aql", } format = format_map.get(ext, "json") @@ -818,6 +871,8 @@ def export_knowledge_graph( export_owl(knowledge_graph, file_path, format=format, method=method, **kwargs) elif format == "cypher": export_lpg(knowledge_graph, file_path, method=method, **kwargs) + elif format == "aql": + export_arango(knowledge_graph, file_path, method=method, **kwargs) else: raise ProcessingError(f"Unknown export format: {format}") @@ -886,6 +941,8 @@ method_registry.register("vector", "json", export_vector) method_registry.register("vector", "numpy", export_vector) method_registry.register("lpg", "default", export_lpg) method_registry.register("lpg", "cypher", export_lpg) +method_registry.register("arango", "default", export_arango) +method_registry.register("arango", "aql", export_arango) method_registry.register("report", "default", generate_report) method_registry.register("report", "html", generate_report) method_registry.register("report", "markdown", generate_report) diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.py index e88b4eba..3a23f81b 100644 --- a/semantica/utils/helpers.py +++ b/semantica/utils/helpers.py @@ -65,7 +65,7 @@ import os import re from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union def format_data(data: Any, format_type: str = "json") -> str: diff --git a/tests/test_arango_aql_exporter.py b/tests/test_arango_aql_exporter.py new file mode 100644 index 00000000..36e2e000 --- /dev/null +++ b/tests/test_arango_aql_exporter.py @@ -0,0 +1,538 @@ +""" +Tests for ArangoDB AQL Exporter Module + +This module contains comprehensive tests for the ArangoDB AQL exporter, +validating AQL syntax generation, node and edge handling, and edge cases. +""" + +import json +import re +import shutil +import tempfile +import unittest +from pathlib import Path + +from semantica.export import ArangoAQLExporter + + +class TestArangoAQLExporter(unittest.TestCase): + """Test cases for ArangoDB AQL Exporter.""" + + def setUp(self): + """Set up test fixtures.""" + self.test_dir = tempfile.mkdtemp() + + # Sample entities for testing + self.entities = [ + { + "id": "e1", + "type": "Person", + "name": "Alice", + "label": "Alice", + "properties": {"age": 30, "email": "alice@example.com"}, + }, + { + "id": "e2", + "type": "Organization", + "name": "Acme Corp", + "label": "Acme Corp", + "properties": {"location": "New York", "founded": 2010}, + }, + { + "id": "e3", + "type": "Person", + "name": "Bob", + "label": "Bob", + "properties": {"age": 25}, + }, + ] + + # Sample relationships for testing + self.relationships = [ + { + "id": "r1", + "source": "e1", + "target": "e2", + "type": "WORKS_FOR", + "properties": {"role": "Engineer", "since": 2020}, + }, + { + "id": "r2", + "source": "e3", + "target": "e2", + "type": "WORKS_FOR", + "properties": {"role": "Manager"}, + }, + { + "id": "r3", + "source": "e1", + "target": "e3", + "type": "KNOWS", + }, + ] + + # Complete knowledge graph + self.kg = { + "entities": self.entities, + "relationships": self.relationships, + "metadata": {"version": "1.0", "created": "2024-01-01"}, + } + + def tearDown(self): + """Clean up test fixtures.""" + shutil.rmtree(self.test_dir) + + def test_exporter_initialization(self): + """Test exporter initialization with default and custom parameters.""" + # Default initialization + exporter = ArangoAQLExporter() + self.assertEqual(exporter.vertex_collection, "vertices") + self.assertEqual(exporter.edge_collection, "edges") + self.assertEqual(exporter.batch_size, 1000) + self.assertTrue(exporter.include_collection_creation) + + # Custom initialization + exporter = ArangoAQLExporter( + vertex_collection="nodes", + edge_collection="links", + batch_size=500, + include_collection_creation=False, + ) + self.assertEqual(exporter.vertex_collection, "nodes") + self.assertEqual(exporter.edge_collection, "links") + self.assertEqual(exporter.batch_size, 500) + self.assertFalse(exporter.include_collection_creation) + + def test_export_knowledge_graph(self): + """Test exporting a complete knowledge graph.""" + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "graph.aql" + + exporter.export_knowledge_graph(self.kg, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Check for collection creation comments + self.assertIn("Create vertex collection", content) + self.assertIn("Create edge collection", content) + + # Check for INSERT statements + self.assertIn("INSERT doc INTO vertices", content) + self.assertIn("INSERT doc INTO edges", content) + + # Check for entity data + self.assertIn("Alice", content) + self.assertIn("Acme Corp", content) + self.assertIn("Bob", content) + self.assertIn("Person", content) + self.assertIn("Organization", content) + + # Check for relationship data + self.assertIn("WORKS_FOR", content) + self.assertIn("KNOWS", content) + self.assertIn("vertices/e1", content) + self.assertIn("vertices/e2", content) + + def test_export_entities_only(self): + """Test exporting only entities (vertices).""" + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "entities.aql" + + exporter.export_entities(self.entities, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Check for vertices INSERT + self.assertIn("INSERT doc INTO vertices", content) + self.assertIn("Alice", content) + self.assertIn("Acme Corp", content) + + # Check that edges are NOT present (empty relationships) + self.assertIn("Attempting to insert 0 edges", content) + + def test_export_relationships_only(self): + """Test exporting only relationships (edges).""" + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "relationships.aql" + + exporter.export_relationships(self.relationships, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Check for edges INSERT + self.assertIn("INSERT doc INTO edges", content) + self.assertIn("WORKS_FOR", content) + self.assertIn("KNOWS", content) + + # Check that vertices section indicates 0 vertices + self.assertIn("Inserting 0 vertices", content) + + def test_custom_collection_names(self): + """Test exporting with custom collection names.""" + exporter = ArangoAQLExporter( + vertex_collection="custom_nodes", edge_collection="custom_edges" + ) + output_path = Path(self.test_dir) / "custom.aql" + + exporter.export(self.kg, str(output_path)) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Check for custom collection names + self.assertIn("INSERT doc INTO custom_nodes", content) + self.assertIn("INSERT doc INTO custom_edges", content) + self.assertIn("custom_nodes/e1", content) + + def test_special_characters_in_properties(self): + """Test handling of special characters in node and edge properties.""" + special_entities = [ + { + "id": "special_1", + "type": "Person", + "name": "O'Brien", + "properties": {"quote": 'She said "hello"', "path": "C:\\Users\\test"}, + } + ] + + special_relationships = [ + { + "id": "special_r1", + "source": "special_1", + "target": "e1", + "type": "KNOWS", + "properties": {"note": "Uses 'quotes' and \"escapes\""}, + } + ] + + kg = {"entities": special_entities, "relationships": special_relationships} + + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "special_chars.aql" + + exporter.export(kg, str(output_path)) + + # Verify file was created and is valid JSON structure + self.assertTrue(output_path.exists()) + + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # The content should contain the special characters properly escaped in JSON + self.assertIn("O'Brien", content) + self.assertIn("She said", content) + + def test_empty_collections(self): + """Test handling of empty entity and relationship collections.""" + empty_kg = {"entities": [], "relationships": []} + + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "empty.aql" + + exporter.export(empty_kg, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Should have collection creation comments but no INSERT statements + self.assertIn("Inserting 0 vertices", content) + self.assertIn("Attempting to insert 0 edges", content) + + def test_missing_source_or_target(self): + """Test handling of edges with missing source or target.""" + invalid_relationships = [ + {"id": "r_invalid_1", "source": "e1", "type": "KNOWS"}, # Missing target + {"id": "r_invalid_2", "target": "e2", "type": "RELATED"}, # Missing source + { + "id": "r_valid", + "source": "e1", + "target": "e2", + "type": "VALID", + }, # Valid + ] + + kg = {"entities": self.entities, "relationships": invalid_relationships} + + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "invalid_edges.aql" + + # Should not raise an exception, but should skip invalid edges + exporter.export(kg, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # The data section should only contain the valid relationship + self.assertIn("VALID", content) + # The header comment should count all input edges, including invalid ones + self.assertIn("Attempting to insert 3 edges", content) + + def test_key_sanitization(self): + """Test sanitization of keys with invalid characters.""" + entities_with_invalid_keys = [ + { + "id": "e1@domain.com", + "type": "Email", + "name": "Test Email", + }, + { + "id": "user/123/profile", + "type": "Profile", + "name": "User Profile", + }, + { + "id": "_system", + "type": "System", + "name": "System Node", + }, + ] + + kg = {"entities": entities_with_invalid_keys, "relationships": []} + + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "sanitized.aql" + + exporter.export(kg, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Keys should be sanitized (@ and / replaced with _) + self.assertIn("e1_domain_com", content) + self.assertIn("user_123_profile", content) + # _system should become k_system (no leading underscore) + self.assertIn("k_system", content) + + def test_batch_processing(self): + """Test batch processing with small batch size.""" + # Create many entities to test batching + many_entities = [ + {"id": f"e{i}", "type": "Node", "name": f"Node {i}"} for i in range(250) + ] + + kg = {"entities": many_entities, "relationships": []} + + # Use small batch size + exporter = ArangoAQLExporter(batch_size=100) + output_path = Path(self.test_dir) / "batched.aql" + + exporter.export(kg, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Count number of INSERT statements (should be 3 batches: 100, 100, 50) + insert_count = content.count("INSERT doc INTO vertices") + self.assertEqual(insert_count, 3, "Should have 3 batches for 250 entities") + + def test_nested_properties(self): + """Test handling of nested dictionaries and lists in properties.""" + entities_with_nested = [ + { + "id": "complex_1", + "type": "ComplexNode", + "name": "Complex", + "properties": { + "nested_dict": {"key1": "value1", "key2": "value2"}, + "nested_list": [1, 2, 3, 4, 5], + "mixed": {"list": [1, 2], "value": "test"}, + }, + } + ] + + kg = {"entities": entities_with_nested, "relationships": []} + + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "nested.aql" + + exporter.export(kg, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Nested structures should be properly serialized as JSON + self.assertIn("nested_dict", content) + self.assertIn("nested_list", content) + # Check for the list values (may be formatted on separate lines) + self.assertIn("1", content) + self.assertIn("2", content) + self.assertIn("3", content) + self.assertIn("4", content) + self.assertIn("5", content) + + def test_aql_syntax_validity(self): + """Test that generated AQL has valid syntax structure.""" + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "syntax_test.aql" + + exporter.export(self.kg, str(output_path)) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Check for valid AQL structure + # Should have FOR doc IN [array] INSERT doc INTO collection pattern + # Using re.search with DOTALL flag + pattern = r"FOR doc IN \[.*?\]\s+INSERT doc INTO \w+" + self.assertTrue( + re.search(pattern, content, re.DOTALL), + f"Pattern '{pattern}' not found in generated AQL", + ) + + # Check that JSON arrays in the INSERT statements are valid + # Find all JSON arrays in the content + json_arrays = re.findall(r"FOR doc IN (\[.*?\])\s+INSERT", content, re.DOTALL) + for json_array_match in json_arrays: + # Extract just the array part + json_str = json_array_match.replace("\n INSERT", "").strip() + try: + # This should parse without errors + parsed = json.loads(json_str) + self.assertIsInstance(parsed, list) + except json.JSONDecodeError as e: + self.fail(f"Invalid JSON in AQL: {e}") + + def test_without_collection_creation(self): + """Test export without collection creation statements.""" + exporter = ArangoAQLExporter(include_collection_creation=False) + output_path = Path(self.test_dir) / "no_creation.aql" + + exporter.export(self.kg, str(output_path)) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Should still have INSERT statements + self.assertIn("INSERT doc INTO vertices", content) + self.assertIn("INSERT doc INTO edges", content) + + def test_export_with_nodes_edges_keys(self): + """Test export using 'nodes' and 'edges' keys. + + Instead of 'entities' and 'relationships'. + """ + kg_alt = { + "nodes": self.entities, + "edges": self.relationships, + } + + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "nodes_edges.aql" + + exporter.export(kg_alt, str(output_path)) + + # Verify file was created + self.assertTrue(output_path.exists()) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Should have all the data + self.assertIn("Alice", content) + self.assertIn("WORKS_FOR", content) + + def test_override_collection_names_in_export(self): + """Test overriding collection names via export options.""" + exporter = ArangoAQLExporter( + vertex_collection="default_v", edge_collection="default_e" + ) + output_path = Path(self.test_dir) / "override.aql" + + # Override via options + exporter.export( + self.kg, + str(output_path), + vertex_collection="override_v", + edge_collection="override_e", + ) + + # Read and verify content + with open(output_path, "r", encoding="utf-8") as f: + content = f.read() + + # Should use overridden names + self.assertIn("INSERT doc INTO override_v", content) + self.assertIn("INSERT doc INTO override_e", content) + self.assertIn("override_v/e1", content) + + def test_invalid_collection_name_on_init(self): + """Test that invalid collection names raise ValueError on initialization.""" + # Test collection name starting with number + with self.assertRaises(ValueError) as context: + ArangoAQLExporter(vertex_collection="123invalid") + self.assertIn("must start with a letter or underscore", str(context.exception)) + + # Test collection name with invalid characters + with self.assertRaises(ValueError) as context: + ArangoAQLExporter(edge_collection="invalid@name") + self.assertIn("contains invalid character", str(context.exception)) + + # Test empty collection name + with self.assertRaises(ValueError) as context: + ArangoAQLExporter(vertex_collection="") + self.assertIn("cannot be empty", str(context.exception)) + + # Test too long collection name + with self.assertRaises(ValueError) as context: + ArangoAQLExporter(vertex_collection="a" * 257) + self.assertIn("exceeds maximum length", str(context.exception)) + + def test_invalid_collection_name_on_export(self): + """Test invalid collection names raise ValueError when overriding.""" + exporter = ArangoAQLExporter() + output_path = Path(self.test_dir) / "test.aql" + + kg = {"entities": self.entities, "relationships": self.relationships} + + # Test invalid vertex collection override + with self.assertRaises(ValueError) as context: + exporter.export(kg, str(output_path), vertex_collection="123invalid") + self.assertIn("must start with a letter or underscore", str(context.exception)) + + # Test invalid edge collection override + with self.assertRaises(ValueError) as context: + exporter.export(kg, str(output_path), edge_collection="invalid@name") + self.assertIn("contains invalid character", str(context.exception)) + + +if __name__ == "__main__": + unittest.main()