chore: bump version to 0.6.0

Promotes the Unreleased changelog section (Databricks connector, SQLite
vector store, SPARQL CONSTRUCT templates, JenaStore named-graph support)
to 0.6.0 and syncs version references across pyproject.toml, __init__.py,
and docs.

.checkov.yaml

@@ -0,0 +1,5 @@
+# Checkov configuration.
+# Cloud Run false-positives (CKV_K8S_21/28/30) are suppressed via per-file
+# inline checkov:skip comments in deploy/gcp/cloudrun-service.yaml rather than
+# globally here, so future real Kubernetes manifests are not silently exempted.
+skip-check: []

.dockerignore

@@ -0,0 +1,103 @@
+# Start with a tiny Docker context and opt in only files used by Dockerfile.
+*
+!Dockerfile
+!.dockerignore
+!pyproject.toml
+!README.md
+!LICENSE
+!MANIFEST.in
+!semantica/
+!semantica/**
+!integrations/
+!integrations/**
+!explorer/
+!explorer/**
+
+# VCS, local config, and secrets.
+.git
+.git/**
+.github
+.github/**
+.claude
+.claude/**
+.codex
+.codex/**
+.agents
+.agents/**
+.env
+.env.*
+*.env
+
+# Python build/test/cache artifacts.
+__pycache__
+**/__pycache__
+*.py[cod]
+.pytest_cache
+.pytest_cache/**
+.mypy_cache
+.mypy_cache/**
+.ruff_cache
+.ruff_cache/**
+.tox
+.tox/**
+.venv
+.venv/**
+venv
+venv/**
+coverage
+coverage/**
+htmlcov
+htmlcov/**
+*.egg-info
+*.egg-info/**
+build
+build/**
+dist
+dist/**
+
+# Frontend dependency/build artifacts.
+node_modules
+node_modules/**
+explorer/node_modules
+explorer/node_modules/**
+explorer/dist
+explorer/dist/**
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Local outputs and large generated samples.
+logs
+logs/**
+*.log
+*.tmp
+*.bak
+*.backup
+tests
+tests/**
+explorer/tests
+explorer/tests/**
+docs
+docs/**
+site
+site/**
+.mkdocs_cache
+.mkdocs_cache/**
+cookbook
+cookbook/**
+examples
+examples/**
+demo_assets
+demo_assets/**
+demo_out
+demo_out/**
+demo_out_*
+demo_out_*/**
+outputs
+outputs/**
+pytest-cache-files-*
+pytest-cache-files-*/**
+test_data
+test_data/**
+sample_data
+sample_data/**

.gitattributes

@@ -1,3 +1,11 @@
+# Line endings — force LF so Mintlify/Linux CI parses frontmatter correctly
+* text=auto eol=lf
+*.md text eol=lf
+*.json text eol=lf
+*.yml text eol=lf
+*.yaml text eol=lf
+*.py text eol=lf
+
 # Linguist documentation and generated files
 # This ensures GitHub language statistics reflect the core Python code
 

.github/workflows/benchmark.yml

@@ -1,13 +1,6 @@
 name: Semantica Performance Suite
 
 on:
-  push:
-    branches: [main]
-    paths-ignore:
-      - 'docs/**'
-      - 'mkdocs.yml'
-      - 'requirements-docs.txt'
-      - '**/*.md'
   workflow_dispatch:
 
 permissions:
@@ -20,7 +13,7 @@ jobs:
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 

.github/workflows/ci.yml

@@ -1,28 +1,56 @@
 name: CI
 
+permissions:
+  contents: read
+
 on:
   push:
     branches: [main]
     paths-ignore:
       - 'docs/**'
-      - 'mkdocs.yml'
-      - 'requirements-docs.txt'
+      - 'docs_check.py'
       - '**/*.md'
   pull_request:
     branches: [main]
     paths-ignore:
       - 'docs/**'
-      - 'mkdocs.yml'
-      - 'requirements-docs.txt'
+      - 'docs_check.py'
       - '**/*.md'
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: explorer/package-lock.json
+      - name: Build Explorer frontend
+        working-directory: explorer
+        run: |
+          npm ci
+          npm run build
       - run: pip install build
       - run: python -m build
+      - name: Verify Explorer frontend is packaged
+        run: |
+          python - <<'PY'
+          import zipfile
+          from pathlib import Path
+
+          wheels = list(Path("dist").glob("*.whl"))
+          assert wheels, "No wheel was built"
+
+          with zipfile.ZipFile(wheels[0]) as wheel:
+              names = set(wheel.namelist())
+
+          assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel"
+          assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel"
+
+          print("Explorer frontend is packaged")
+          PY

.github/workflows/codeql.yml

@@ -20,9 +20,38 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
-      - name: Initialize CodeQL
+      # The CodeQL bundle download (github/codeql-action/init's "Setup CodeQL
+      # tools" step) streams a ~1GB tarball from GitHub's release CDN and
+      # does not retry on a transient connection reset (ECONNRESET) itself
+      # (github/codeql-action, unresolved as of v4 / CLI 2.26.1: the HTTP
+      # error is retryable but isn't retried internally). Since a `uses:`
+      # step can't be wrapped by a shell-level retry action, attempt init
+      # up to 3 times; each retry is a fresh download attempt with no
+      # meaningful state carried over from a failed attempt.
+      - name: Initialize CodeQL (attempt 1)
+        id: codeql-init-1
+        uses: github/codeql-action/init@v4
+        continue-on-error: true
+        with:
+          languages: python
+          queries: security-and-quality
+          config-file: .github/codeql/codeql-config.yml
+
+      - name: Initialize CodeQL (attempt 2)
+        id: codeql-init-2
+        if: steps.codeql-init-1.outcome == 'failure'
+        uses: github/codeql-action/init@v4
+        continue-on-error: true
+        with:
+          languages: python
+          queries: security-and-quality
+          config-file: .github/codeql/codeql-config.yml
+
+      - name: Initialize CodeQL (attempt 3)
+        id: codeql-init-3
+        if: steps.codeql-init-2.outcome == 'failure'
         uses: github/codeql-action/init@v4
         with:
           languages: python

.github/workflows/defender-for-devops.yml

@@ -0,0 +1,88 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+#
+# Microsoft Security DevOps (MSDO) is a command line application which integrates static analysis tools into the development cycle.
+# MSDO installs, configures and runs the latest versions of static analysis tools
+# (including, but not limited to, SDL/security and compliance tools).
+#
+# The Microsoft Security DevOps action is currently in beta and runs on the windows-latest queue,
+# as well as Windows self hosted agents. ubuntu-latest support coming soon.
+#
+# For more information about the action , check out https://github.com/microsoft/security-devops-action
+#
+# Please note this workflow do not integrate your GitHub Org with Microsoft Defender For DevOps. You have to create an integration
+# and provide permission before this can report data back to azure.
+# Read the official documentation here : https://learn.microsoft.com/en-us/azure/defender-for-cloud/quickstart-onboard-github
+
+name: "Microsoft Defender For Devops"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '43 17 * * 6'
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  MSDO:
+    # currently only windows-latest is supported
+    runs-on: windows-latest
+
+    steps:
+    - uses: actions/checkout@v7
+    - uses: actions/setup-dotnet@v5
+      with:
+        dotnet-version: |
+          5.0.x
+          6.0.x
+    - name: Run Microsoft Security DevOps
+      uses: microsoft/security-devops-action@v1.12.0
+      id: msdo
+      with:
+        # checkov is intentionally excluded from this MSDO step.
+        # MSDO 0.215.0's guardian.cmd wrapper treats checkov's exit code 1
+        # (emitted whenever any violation is found, even below the active severity
+        # threshold) as a fatal "tool error" and breaks the build even when
+        # "Active results: 0" and "Found no breaking results." The .checkov.yaml
+        # soft-fail setting is never read by the guardian wrapper.
+        # IaC security scanning continues below in this same MSDO job identity.
+        # That preserves the existing GitHub code-scanning configuration while
+        # avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
+        tools: eslint,templateanalyzer,terrascan
+    - name: Upload results to Security tab
+      uses: github/codeql-action/upload-sarif@v4
+      with:
+        sarif_file: ${{ steps.msdo.outputs.sarifFile }}
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Install Checkov
+      run: python -m pip install checkov==3.3.1
+
+    - name: Run Checkov
+      shell: pwsh
+      env:
+        PYTHONUTF8: "1"
+      run: |
+        New-Item -ItemType Directory -Force reports | Out-Null
+        checkov --directory . --framework kubernetes helm dockerfile github_actions secrets bicep arm --soft-fail --output sarif --output-file-path reports/checkov.sarif
+        if (-not (Test-Path reports/checkov.sarif)) {
+          $sarif = Get-ChildItem -Path reports -Recurse -Filter *.sarif | Select-Object -First 1
+          if ($null -eq $sarif) { throw "Checkov did not produce a SARIF file" }
+          Copy-Item $sarif.FullName reports/checkov.sarif
+        }
+
+    - name: Upload Checkov results to Security tab
+      uses: github/codeql-action/upload-sarif@v4
+      if: always()
+      with:
+        sarif_file: reports/checkov.sarif

.github/workflows/docs.yml

@@ -1,80 +1,68 @@
 name: Build and Deploy Documentation
 
-# This workflow builds the documentation site and deploys it to GitHub Pages
-# It runs when changes are pushed to the 'docs' folder on the main branch
-
 on:
   push:
     branches: [main]
     paths:
       - 'docs/**'
-      - 'mkdocs.yml'
-      - 'requirements-docs.txt'
+      - 'docs_check.py'
       - 'CHANGELOG.md'
       - 'RELEASE.md'
-  release:
-    types: [published]
+  pull_request:
+    branches: [main]
+    paths:
+      - 'docs/**'
+      - 'docs_check.py'
   workflow_dispatch:
 
-# Permissions needed to deploy to GitHub Pages
 permissions:
   contents: read
   pages: write
   id-token: write
 
-# Prevent concurrent deployments
 concurrency:
   group: "pages"
   cancel-in-progress: false
 
 jobs:
-  build:
-    name: Build Documentation
+  validate:
+    name: Validate Documentation
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '20'
+      - run: python docs_check.py
 
-      - name: Install documentation dependencies
+  deploy:
+    name: Build and Deploy to GitHub Pages
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    needs: validate
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '20'
+
+      - name: Export static site
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-docs.txt
+          cd docs
+          npx mintlify export --output ../export.zip
+          cd ..
+          unzip -q export.zip -d site
 
-      - name: Build documentation
-        # Builds the static site using MkDocs
-        run: mkdocs build --strict
+      - uses: actions/configure-pages@v6
 
-      - name: Check for broken links
-        # Optional: checks if any links in the docs are broken
-        run: |
-          pip install linkchecker || echo "Skipping link check"
-          if [ -d "site" ]; then
-            linkchecker site/ --check-extern || echo "Link check completed"
-          fi
-        continue-on-error: true
-
-      - name: Setup Pages
-        uses: actions/configure-pages@v6
-        continue-on-error: true
-
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v5
+      - uses: actions/upload-pages-artifact@v5
         with:
           path: ./site
 
-  deploy:
-    name: Deploy to GitHub Pages
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
       - name: Deploy to GitHub Pages
         id: deployment
         uses: actions/deploy-pages@v5

.github/workflows/release.yml

@@ -13,12 +13,39 @@ jobs:
     runs-on: ubuntu-latest
     environment: pypi
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: explorer/package-lock.json
+      - name: Build Explorer frontend
+        working-directory: explorer
+        run: |
+          npm ci
+          npm run build
       - run: pip install build
       - run: python -m build
+      - name: Verify Explorer frontend is packaged
+        run: |
+          python - <<'PY'
+          import zipfile
+          from pathlib import Path
+
+          wheels = list(Path("dist").glob("*.whl"))
+          assert wheels, "No wheel was built"
+
+          with zipfile.ZipFile(wheels[0]) as wheel:
+              names = set(wheel.namelist())
+
+          assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel"
+          assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel"
+
+          print("Explorer frontend is packaged")
+          PY
       - uses: softprops/action-gh-release@v3
         with:
           files: dist/*

.github/workflows/security-scan.yml

@@ -18,6 +18,9 @@ on:
       - 'requirements-docs.txt'
       - '**/*.md'
 
+permissions:
+  contents: read
+
 jobs:
   security-scan:
     runs-on: ubuntu-latest
@@ -28,7 +31,7 @@ jobs:
     
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
       
       - name: Set up Python
         uses: actions/setup-python@v4

.github/workflows/security.yml

@@ -12,7 +12,7 @@ jobs:
   audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'

ARCHITECTURE.md

@@ -0,0 +1,108 @@
+# Semantica — Architecture
+
+Complete data flow from every source type to every final output, and the decision intelligence lifecycle.
+
+---
+
+## Full Data Pipeline
+
+Every source, every processing step, every final artifact — in one diagram.
+
+```mermaid
+flowchart TD
+    %% ── SOURCES ──────────────────────────────────────────────────────
+    subgraph SRC["🗂️ Sources  (semantica.ingest)"]
+        direction LR
+        F["📄 Files\nPDF · DOCX · PPTX · HTML\nTXT · CSV · JSON · Excel · XML"]
+        W["🌐 Web\nPages · RSS/Atom Feeds\nPublic REST APIs"]
+        DB["🗃️ Databases\nPostgreSQL · MySQL · SQLite\nOracle · DuckDB · MongoDB"]
+        CL["☁️ Cloud\nSnowflake · Google Drive\nElasticsearch · HuggingFace"]
+        RT["⚡ Streams\nKafka · RabbitMQ\nAWS Kinesis · Pulsar"]
+        DV["🛠️ Dev\nGit Repos · Email IMAP/POP3\nMCP Resources · Parquet · Pandas"]
+    end
+
+    %% ── INGEST ───────────────────────────────────────────────────────
+    F  --> FI["FileIngestor"]
+    W  --> WI["WebIngestor"]
+    DB --> DI["DBIngestor"]
+    CL --> PI["ParquetIngestor\nSnowflakeIngestor"]
+    RT --> SI["StreamIngestor"]
+    DV --> RI["RepoIngestor\nEmailIngestor · MCPIngestor"]
+
+    FI & WI & DI & PI & SI & RI --> RAW[/"📦 Raw Documents"/]
+
+    %% ── PARSE ────────────────────────────────────────────────────────
+    RAW --> PRS["🔍 Parse  (semantica.parse)\nDocumentParser · StructuredDataParser\nCodeParser · WebParser · EmailParser"]
+
+    PRS --> NRM["🧹 Normalize  (semantica.normalize)\nTextNormalizer · EntityNormalizer\nDateNormalizer · NumberNormalizer · DataCleaner"]
+
+    NRM --> SPL["✂️ Split  (semantica.split)\nentity_aware · relation_aware\ngraph_based · ontology_aware · hierarchical"]
+
+    %% ── EXTRACT ──────────────────────────────────────────────────────
+    SPL --> EXT["🔬 Extract  (semantica.semantic_extract)\nNamedEntityRecognizer · RelationExtractor\nEventDetector · TripletExtractor · CoreferenceResolver"]
+
+    EXT --> CFT["⚠️ Conflict Detection  (semantica.conflicts)\nConflictDetector · ConflictResolver · SourceTracker"]
+
+    CFT --> DDP["🔁 Deduplication  (semantica.deduplication)\nDuplicateDetector · EntityMerger"]
+
+    DDP --> KGB["🕸️ KG Construction  (semantica.kg)\nGraphBuilder · EntityResolver\nBiTemporalFact · TemporalGraphQuery"]
+
+    KGB --> KG[/"🗺️ Knowledge Graph\nnodes · edges · temporal facts · provenance"/]
+
+    %% ── INTELLIGENCE LAYER ───────────────────────────────────────────
+    KG --> ONT["Ontology  (semantica.ontology)\nOntologyGenerator · OntologyValidator\nOWL · SHACL · SKOS"]
+    KG --> RSN["Reasoning  (semantica.reasoning)\nReteEngine · DatalogReasoner\nSPARQLReasoner · ExplanationGenerator"]
+    KG --> PRV["Provenance  (semantica.provenance)\nProvenanceManager · W3C PROV-O"]
+    KG --> CTX["Context & Decisions  (semantica.context)\nContextGraph · AgentContext\nDecisionRecorder · CausalChainAnalyzer · PolicyEngine"]
+
+    ONT & RSN & PRV & CTX --> EKG[/"🗃️ Enriched KG\n+ ontology · inferences · provenance · decisions"/]
+
+    %% ── STORAGE ──────────────────────────────────────────────────────
+    EKG --> VS["Vector Store  (semantica.vector_store)\nFAISS · Qdrant · Weaviate · Milvus · Pinecone · PgVector\nHybrid Search · RRF Fusion"]
+    EKG --> GS["Graph Store  (semantica.graph_store)\nNeo4j · FalkorDB · Apache AGE · Amazon Neptune"]
+
+    %% ── OUTPUTS ──────────────────────────────────────────────────────
+    VS & GS --> EXP["📦 Export  (semantica.export)\nRDF Turtle · JSON-LD · N-Triples · OWL · SHACL\nParquet · Cypher · ArangoDB AQL · GraphML · CSV · HTML"]
+    VS & GS --> VIZ["📊 Visualize  (semantica.visualization)\nKGVisualizer · OntologyVisualizer\nEmbeddingVisualizer · TemporalVisualizer"]
+    EKG --> SVC["🔌 Services\nREST API 100+ endpoints · MCP Server 10+ tools\nCLI 50+ commands · Knowledge Explorer"]
+```
+
+---
+
+## Decision Intelligence Lifecycle
+
+```mermaid
+flowchart LR
+    subgraph RECORD["1️⃣  Record"]
+        R1["record_decision()\ncategory · scenario\nreasoning · outcome\nconfidence · metadata"]
+    end
+
+    subgraph LINK["2️⃣  Link"]
+        L1["add_causal_relationship()\ntriggers · enables\ncauses · precedes"]
+    end
+
+    subgraph QUERY["3️⃣  Query"]
+        Q1["find_similar_decisions()\nSemantic precedent search"]
+        Q2["trace_decision_chain()\nFull causal ancestry"]
+        Q3["analyze_decision_impact()\nDownstream influence map"]
+    end
+
+    subgraph GOVERN["4️⃣  Govern"]
+        G1["check_decision_rules()\nPolicy evaluation\nCompliance gate"]
+    end
+
+    subgraph AUDIT["5️⃣  Audit Export"]
+        A1["W3C PROV-O  ·  CSV  ·  JSON\nRegulator-ready audit trail"]
+    end
+
+    RECORD -->|decision_id| LINK
+    LINK   -->|causal graph| QUERY
+    QUERY  -->|results| GOVERN
+    GOVERN -->|signed-off decisions| AUDIT
+```
+
+---
+
+*→ [README](README.md) · [Docs](https://docs.getsemantica.ai/) · [Cookbook](https://github.com/semantica-agi/semantica/tree/main/cookbook)*
+
+> Note: `Docs` and `Cookbook` are external resources maintained outside this file and may change over time. If a link is unavailable, refer to the repository `README.md` and in-repo documentation as canonical fallbacks.

CHANGELOG.md

@@ -7,6 +7,273 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [Unreleased]
+
+## [0.6.0] - 2026-07-21
+
+### Added
+
+- **Named-graph support for `JenaStore` via `Dataset` migration** (#756, #757) by @Sameer6305 and @KaifAhmad1
+  - `JenaStore` now backs onto `rdflib.Dataset(default_union=False)` instead of `rdflib.Graph`, closing #756 and fully closing out the #754/#756 cross-backend named-graph parity effort across Blazegraph, RDF4J, and Jena
+  - `default_union=False` is explicitly set so existing `execute_sparql()`/`get_triplets()` calls that don't pass `graph=` keep seeing only the default graph, not a union across all named graphs
+  - `add_triplets()` accepts a `graph=` option: when supplied, triples are written to that named graph (4-tuple add via `Dataset.graph(uri)`); when omitted, behavior is unchanged (3-tuple add routes to the default graph)
+  - Fixed a pre-existing bug where the remote-endpoint path instantiated the read-only rdflib `SPARQLStore` instead of `SPARQLUpdateStore`, so every `add_triplets()` call against a remote Fuseki endpoint silently failed (`TypeError` swallowed, `success=True`/`added=0` returned); also fixed a constructor bug where `self.endpoint` was always `None` regardless of how `JenaStore` was called, making the remote path unreachable in practice
+  - `serialize()` now logs a warning instead of silently dropping named-graph content when the requested format (`turtle`, `xml`, `n3`, …) can only serialize the default graph; use `format="trig"` or `format="nquads"` to include all graphs
+  - `create_model()`'s `triplet_count` now documented as counting across all graphs (default + named), not just the default graph, matching the `Dataset`-wide semantics
+  - `delete_triplet()` remains scoped to the default graph only (named-graph parity for delete is an explicit follow-up, matching the maintainer's scoping of this migration to `add_triplets`); the removal is passed `self.graph.default_graph` explicitly as its context, since `Dataset.remove()` on a bare 3-tuple resolves to a wildcard context internally and would otherwise delete matching triples out of every named graph too — a follow-up fix to the initial PR #757 for a bug that had no test coverage
+  - 9 new tests covering `Dataset` construction, `default_union=False` confirmation, named-graph write isolation, `serialize()` warning behavior, and `delete_triplet()`'s default-graph scoping
+
+- **SPARQL CONSTRUCT query templates** (#752, #322, #755, #754) by @Sameer6305
+  - Added parameterized, injection-safe `CONSTRUCT` templates (`ConstructTemplate`, `ParameterDescriptor`, `ConstructTemplateRegistry`)
+  - Extended CONSTRUCT execution support from Blazegraph-only to the RDF4J and Jena backends (#755), closing #754
+    - `RDF4JStore.execute_sparql` gains a CONSTRUCT-aware path (`Accept: text/turtle`, rdflib Turtle parsing, the same `(s, p, o, metadata)` 4-tuple contract) and named-graph writes via RDF4J's REST `context` parameter
+    - `JenaStore.execute_sparql` gains the equivalent CONSTRUCT-aware path over its in-process `rdflib.Graph`
+    - `_CONSTRUCT_QUERY_RE` moved to `sparql_escaping.py` as a shared, backend-agnostic constant used by all three backends
+  - Added pipeline integration via the `construct_template` step type
+
+- **Databricks Connector (Unity Catalog + Delta Lake ingestion)** (#747) by @KaifAhmad1
+  - Added `DatabricksIngestor` (`semantica/ingest/databricks_ingestor.py`), mirroring `SnowflakeIngestor`'s structure and public API shape: a `DatabricksConnector` connection handler, a `DatabricksData` dataclass, and an optional-import guard for `databricks-sdk`/`databricks-sql-connector`
+  - Supports personal access token and OAuth M2M (service principal `client_id`/`client_secret`) authentication, configurable via constructor args or `DATABRICKS_*` environment variables
+  - `ingest_table()`/`ingest_query()` run against a SQL warehouse or cluster via `databricks-sql-connector`, with `where`/`order_by`/`limit`/`offset` support and the same identifier-escaping and unsafe-`ORDER BY` rejection as `SnowflakeIngestor`; each call closes the SQL connection it opened unless one is already open (e.g. via the `with DatabricksIngestor(...)` context manager), which reuses and closes it exactly once instead of leaking a second connection per call
+  - `get_table_schema()`, `list_catalogs()`, `list_schemas()`, and `list_tables()` introspect Unity Catalog via `databricks-sdk`'s `WorkspaceClient`, validating both catalog and schema are resolved before calling the SDK; `get_table_lineage()` calls Unity Catalog's table-lineage REST API for upstream/downstream `Table --DEPENDS_ON--> Table` dependencies, plus an opt-in `include_column_lineage=True` that resolves per-column lineage via the column-lineage API
+  - `export_as_documents()` converts ingested rows into Semantica document dicts for KG construction, matching `SnowflakeIngestor.export_as_documents()`'s shape
+  - Registered as a lazy export in `semantica.ingest` (`DatabricksIngestor`, `DatabricksData`, `DatabricksConnector`) and as the `db-databricks` optional extra (`pip install "semantica[db-databricks]"`) in `pyproject.toml`, included in `db-all`
+  - New `docs/integrations/databricks.md` page modeled on `docs/integrations/snowflake.md`, plus a `DatabricksIngestor` section and table row in `docs/reference/ingest.md` and cross-links between the two integration pages
+  - 35 unit tests in `tests/test_databricks_ingestor.py` covering both auth methods, table/query ingestion, connection lifecycle (including reuse under the context manager), pagination, unsafe `ORDER BY` rejection, catalog/schema validation, schema/catalog/table listing, table and column lineage, document export, and the missing-dependency error path, closing #747
+
+- **SQLite Vector Store Backend (`sqlite-vec`)** (#726) by @Luffy2208 and @KaifAhmad1
+  - Added `SQLiteVecStore` (`semantica/vector_store/sqlite_vec_store.py`), a disk-backed local vector store using the `sqlite-vec` extension's `vec0` virtual tables, closing #240
+  - Supports Cosine and L2 distance metrics, dynamic JSON metadata filtering, read-only mode, and an in-memory (`:memory:`) mode
+  - Registered as the `"sqlite"` backend in `VectorStore.SUPPORTED_BACKENDS`, with `db_path`/`sqlite_path` config and a `VECTOR_STORE_SQLITE_PATH` environment variable
+  - Batched `add`/`delete`/`get` and `executemany`-based `update` to avoid per-row round trips; optional `use_wal=True` enables `journal_mode=WAL` + `synchronous=NORMAL` for improved write concurrency
+  - Lazy-imports `sqlite-vec` so the dependency stays fully optional (`pip install semantica[vectorstore-sqlite]`); table names and metadata filter keys are validated against a strict identifier pattern before SQL interpolation
+  - Fixes `VectorStore.update_vectors`/`delete_vectors` to delegate to the active backend store instead of only mutating in-memory state, correcting existing behavior for all non-`inmemory` backends
+  - 25 unit and integration tests in `tests/vector_store/test_sqlite_vec_store.py` covering init, add, search, get, update, delete, read-only mode, and stats
+
+### Fixed
+
+- **`kg.ProvenanceTracker` compatibility wrapper out of sync with `ProvenanceManager`, causing 9 pre-existing test failures** (#744, #751) by @Sameer6305 and @KaifAhmad1
+  - `kg.ProvenanceTracker` was a standalone in-memory implementation that never delegated to the unified `ProvenanceManager` backend; its own test suite asserted the existence of `get_lineage`, `track_relationship`, `track_entities_batch`, `get_provenance`, and `_use_unified`, none of which were ever implemented, plus a stale `get_all_sources()` assertion expecting `"timestamp"` instead of the actual `"recorded_at"` key
+  - Rather than completing the abandoned compatibility layer, `kg.ProvenanceTracker` and its remaining supported methods (`track_entity`, `get_all_sources`, `query_recorded_between`, `revision_history`, `export_audit_log`) now emit `DeprecationWarning`s pointing callers to `semantica.provenance.ProvenanceManager`
+  - Removed/rewrote the 9 tests that only exercised the never-implemented compatibility methods to instead verify the observable behavior of the still-supported API, and corrected the stale `get_all_sources()` assertion
+  - Added the previously-missing `docs/migration/kg-provenance-tracker.md` migration guide referenced by every new deprecation warning, with a method-mapping table to `ProvenanceManager` and a before/after example, closing #744
+
+- **`ProvenanceManager.track_entity` silently overrides an explicit `parent_entity_id`/`derived_from` on re-track** (#742) by @Sameer6305
+  - `track_entity()` resolved `parent_id` via a documented precedence chain (`parent_entity_id` kwarg > `metadata["derived_from"]` > source-as-known-entity-id fallback), but the history-preservation block that runs afterward unconditionally overwrote that resolved value with an auto-generated `f"{entity_id}:v:{existing.last_updated}"` history pointer whenever the entity was being re-tracked, discarding whatever parent the caller had just explicitly supplied with no warning
+  - `track_entity()` now records whether the precedence chain already resolved an explicit parent (`parent_entity_id` kwarg, `metadata["derived_from"]`, or the source-as-known-entity-id fallback) before the history block runs, and only falls back to the auto-generated history pointer when the caller supplied no explicit parent on that call
+  - The archived history entry for the previous version is still kept reachable in `get_lineage()` via `used_entities` (BFS-traversed by `InMemoryStorage.trace_lineage()`) even when an explicit parent is supplied, so re-tracking with a new parent no longer orphans the prior version from the lineage chain; when no explicit parent is supplied, `used_entities` is left alone since `parent_entity_id` already points at the same history id, avoiding a duplicate self-reference
+  - Added `test_retrack_with_explicit_parent_overrides_history_link`, `test_retrack_without_explicit_parent_still_uses_history_link`, `test_retrack_with_derived_from_overrides_history_link`, and `test_retrack_history_reachable_via_used_entities` regression tests, closing #742
+
+- **`ProvenanceManager.get_lineage` does not link entities that share a source URL** (#735) by @KaifAhmad1
+  - `track_entity()`'s only auto-linking logic looked up `source` as if it were an existing entity's `entity_id`, so passing the same real URL/DOI as `source` for two conceptually linked entities (e.g. a document and a decision derived from it) never produced a parent link, leaving `get_lineage()` returning a chain of length 1
+  - `metadata["derived_from"]` was preserved and echoed back in the output JSON but was never consulted by any linking or traversal code, so the caller's explicit relationship was silently inert
+  - `track_entity()` now treats `metadata["derived_from"]` as an explicit parent link (unless `parent_entity_id` was already passed directly), so `InMemoryStorage.trace_lineage()`'s existing BFS over `parent_entity_id` picks it up for free
+  - `metadata["derived_from"]` is now recognized on any `collections.abc.Mapping`, not just a concrete `dict`, so e.g. `types.MappingProxyType` metadata still creates the parent link
+  - `get_lineage()`'s metadata aggregation now applies the queried entity's own metadata last so it wins over ancestor metadata on conflicting keys, matching the documented "most recent entry's metadata takes precedence" behavior — previously `trace_lineage()`'s BFS order caused ancestor metadata (now reachable via `derived_from` chains) to silently overwrite the queried entity's own values
+  - Added 9 regression/edge-case tests in `tests/provenance/test_manager.py` covering the happy path, explicit `parent_entity_id` precedence over `derived_from`, precedence over the `source`-as-known-entity-id fallback, a `derived_from` pointing at a never-tracked entity, non-string/empty-string `derived_from` values being ignored, a self-referencing `derived_from` not hanging traversal, multi-hop `derived_from` chains, metadata precedence between a queried entity and its ancestors, and non-`dict` `Mapping` metadata, closing #735
+
+- **`Reasoner.add_rule` had no deduplication, doubling rules and silently emptying `forward_chain()` on rerun** (#732) by @KaifAhmad1
+  - `add_rule()` unconditionally appended to `self.rules`, so re-running the same setup code on an existing `Reasoner` instance (e.g. re-executing a Jupyter cell) duplicated every rule; since `forward_chain()` only records a conclusion if it isn't already in `self.facts`, the second run's duplicated rules matched but produced no new results, with no error or warning
+  - `add_rule()` now compares an incoming rule's `rule_type`, `conditions`, and `conclusion` against existing rules and returns the existing `Rule` instead of appending a duplicate, keeping repeated `add_rule()` calls with the same definition idempotent
+  - Added `test_add_rule_deduplicates_identical_rule`, `test_add_rule_deduplication_is_idempotent_across_forward_chain`, and `test_add_rule_does_not_dedupe_distinct_rules` regression tests
+
+- **`InferenceResult.premises` always empty from `forward_chain`/`backward_chain`** (#739) by @Sameer6305
+  - `_match_rule()` discarded matched facts and returned only instantiated conclusions, so `ExplanationGenerator` always produced empty premises lists regardless of which facts actually satisfied a rule, closing #733
+  - `_match_rule()` now returns `(conclusion, matched_facts)` tuples; `forward_chain()` threads those facts into `InferenceResult(premises=...)`, merging premises when the same conclusion is derived more than once within a pass
+  - `_prove_goal()`'s base cases (goal already a known fact; goal matched via pattern unification) now return `premises=[goal]`/`premises=[fact]` instead of `[]`
+  - Facts are matched against a `sorted()` snapshot instead of the raw `set` so rule matching and premise selection are deterministic
+  - Added `test_forward_chaining_premises` regression test mirroring the existing backward-chaining premises test
+
+- **Missing `shacl` optional-dependency extra** (#736) by @Sameer6305
+  - `pip install semantica[shacl]` referenced no matching extra in `pyproject.toml`, so `pyshacl` was never installed despite being documented as the fix in `ontology_validator.py`'s `ImportError` message, the Explorer API, the healthcare cookbook notebook, and the changelog
+  - Added `shacl = ["pyshacl>=0.25.0"]` to `[project.optional-dependencies]` and folded `shacl` into the `all` extra
+
+- **`NodeEmbedder` `AttributeError` masked in `ContextGraph.analyze_graph_with_kg`** (#734) by @Sameer6305
+  - `analyze_graph_with_kg()` called a non-existent `NodeEmbedder.generate_embeddings()`, and the surrounding broad `except Exception` swallowed the resulting `AttributeError`, silently returning `{"error": "Graph analysis failed due to an internal error"}` from `get_causal_chain()`'s supporting analytics and `get_decision_insights()`
+  - Rewired the call site to the real `NodeEmbedder.compute_embeddings(graph_store, node_labels, relationship_types)` API, deriving `node_labels`/`relationship_types` from `self.node_type_index`/`self.edge_type_index`
+  - Added a dedicated `except AttributeError` branch that logs distinctly and re-raises, so a broken internal method call surfaces as a diagnosable error instead of being indistinguishable from a legitimately empty analysis result
+
+---
+
+## [0.5.1] - 2026-06-29
+
+### Added
+
+- **Apache Arrow & Feather File Ingestion** (#705) by @Luffy2208
+  - Added `ArrowIngestor` (`semantica/ingest/arrow_ingestor.py`) for reading `.arrow`, `.feather`, and `.ipc` files via PyArrow
+  - Supports Arrow IPC File format (random-access), Arrow IPC Stream format, Feather v1 and v2
+  - Selective column reads, optional row limits, and batch-aware iteration that stops early without scanning the full file
+  - `extract_schema()` and `extract_metadata()` convenience methods for schema/metadata inspection without reading row data
+  - `_ArrowReaderWrapper` provides a unified interface across all three reader types, preventing stream exhaustion during schema inspection
+  - `ingest_arrow()` convenience function and `ingest(..., source_type="arrow")` unified dispatch
+  - Automatic Arrow format detection in `ingest()` by file extension (`.arrow`, `.feather`, `.ipc`) and by Arrow IPC magic bytes (`ARROW1\x00\x00`) in `FileTypeDetector`
+  - Registry integration under the `arrow` task namespace with `file`, `schema`, and `metadata` methods
+  - Lazy-import exports of `ArrowIngestor`, `ArrowData`, and `ingest_arrow` from `semantica.ingest`
+  - Optional dependency group: `pip install semantica[ingest-arrow]`; included in `pip install semantica[all]`
+  - 34 tests covering schema extraction, metadata inspection, row limits, column selection, multi-batch reading, IPC stream format, Feather ingestion, empty datasets, null values, magic-byte detection, and failure modes
+
+- **Knowledge Explorer Deployment Templates** (#684) by @ZohaibHassan16 and @KaifAhmad1
+  - Added `deploy/` directory with ready-to-use templates for 7 platforms, closing #681
+  - **Docker** — fixed `Dockerfile` path (was broken on clean checkout), added non-root user, `HEALTHCHECK`, `.dockerignore`; fixed `docker-compose.yml` to start Explorer alongside FalkorDB on a shared network; added `docker-compose.dev.yml` with source volume-mounts for hot-reload (`docker compose up` brings up the full stack in one command)
+  - **Railway** — `deploy/railway/railway.toml` with Dockerfile builder, healthcheck path, restart policy, and env vars wired from the Railway Redis plugin
+  - **Render** — `deploy/render/render.yaml` Blueprint provisioning the web service and a Redis instance together with cross-linked env vars
+  - **Fly.io** — `deploy/fly/fly.toml` with region, 512 MB VM, auto-stop, HTTP healthcheck, and a short README with four `flyctl` commands to deploy from zero
+  - **GCP Cloud Run** — `deploy/gcp/cloudbuild.yaml` (build → push → deploy pipeline) and `deploy/gcp/cloudrun-service.yaml` (scale-to-zero, Secret Manager env vars, liveness probe)
+  - **Azure Container Apps** — `deploy/azure/azure.yaml`, `main.bicep` (Container App + managed environment, HTTP ingress, HPA min 0 / max 10, liveness probe), and `main.parameters.json`; deployable with `azd up`
+  - **Kubernetes + Helm** — raw manifests (`namespace`, `configmap`, `secret.example`, `deployment` with 2 replicas + rolling update, `service`, `ingress` with cert-manager TLS, `kustomization`); Helm chart with `Chart.yaml`, `values.yaml`, `values.prod.yaml`, HPA template, and `helm lint`-passing templates; all templates carry `namespace: {{ .Release.Namespace }}`
+  - Added `/api/health` endpoint returning `{"status": "ok"}` used by all platform healthchecks
+  - Wired `ALLOWED_ORIGINS`, `FALKORDB_HOST`, and `FALKORDB_PORT` from environment variables in `semantica/explorer/app.py`
+  - Security hardened: non-root containers, `readOnlyRootFilesystem`, `NetworkPolicy` with explicit ingress/egress selectors, `seccompProfile: RuntimeDefault`, capabilities dropped; secrets via `secret.yaml.example` templates only — no committed credentials
+
+### Fixed
+
+- **Arrow ingestion double full-scan on every data read** (#705) by @KaifAhmad1
+  - `ingest_file` previously called `_file_metadata` (a full batch scan) before `_read_batches`, meaning every read scanned the entire file twice; for a `limit=1` read on a large file the metadata pass visited every batch while the data pass read only one; replaced with a single-pass `_read_batches_with_info` that collects batch metadata as a side effect of the data read; `_file_metadata` is now only invoked for `include_data=False`
+
+- **Dead `num_record_batches` property on `_ArrowReaderWrapper` materialised all table batches** (#705) by @KaifAhmad1
+  - The property was never called by production code but its `is_table` branch called `to_batches()` purely to take `len()`, materialising the entire table in memory just for a count; property removed
+
+- **Arrow `_open_file` chained the wrong exception** (#705) by @KaifAhmad1
+  - The fallback cascade (IPC file → IPC stream → Feather) raised `from feather_err`, surfacing the least diagnostic error in the Python traceback chain; changed to `from file_err` so the IPC file open error — the most informative signal for unrecognised formats — appears as `__cause__`
+
+- **Neo4j Bulk CSV Export** (#665) by @Luffy2208
+  - Added `Neo4jCSVExporter` for generating Neo4j bulk-import CSV files compatible with `neo4j-admin database import`
+  - Produces deterministic `nodes.csv` and `relationships.csv` with stable node IDs — reuses existing graph IDs or derives reproducible SHA-256 content-based IDs when none are present
+  - Multi-label support via Neo4j `:LABEL` convention with configurable `label_separator` (default `;`)
+  - Alphabetically sorted property columns and deterministic row ordering for reproducible output across permuted inputs
+  - Relationship endpoint resolution: aliases (`name`, `text`, `label`) automatically mapped to stable node IDs
+  - Nested property serialisation to canonical JSON; flat scalar values written directly
+  - `dry_run()` method for pre-flight CSV validation without writing files
+  - `validate_export()` for post-write integrity checks (unique `:id`, consistent column widths, valid endpoint references)
+  - `export_nodes()` and `export_relationships()` for partial exports
+  - `strict=True` mode raises `ValidationError` on unresolved relationship endpoints
+  - `export_neo4j_csv()` convenience function and `format="neo4j_csv"` / `format="neo4j-csv"` dispatch in `export_knowledge_graph()`
+  - Registry integration under the `neo4j_csv` task namespace
+  - Documentation added to `semantica/export/export_usage.md` with usage examples, mapping assumptions, and `neo4j-admin` import command
+  - 13 tests covering headers, node/relationship CSV structure, multi-label, missing properties, deterministic output, CSV quoting/escaping, Unicode, empty graphs, dry-run, duplicate ID detection, ambiguous alias handling, nested property serialisation, and `KnowledgeGraph` integration
+
+### Fixed
+
+- **Neo4j CSV exporter `_write_csv` crashed with `TypeError` on dialect kwargs** (#665) by @KaifAhmad1
+  - Passing `delimiter=`, `encoding=`, or any caller kwarg to `export_neo4j_csv` caused `csv.writer` to receive unknown or duplicate keyword arguments; `_write_csv` now whitelists only valid `csv.writer` dialect params (`quotechar`, `doublequote`, `skipinitialspace`, `escapechar`, `strict`)
+
+- **`export_neo4j_csv` double-passed kwargs to both the constructor and `export()`** (#665) by @KaifAhmad1
+  - Constructor-level settings (`node_file_name`, `relationship_file_name`, `encoding`, `delimiter`, `label_separator`, `strict`) were merged into config for the constructor then re-forwarded as `**kwargs` to `export_knowledge_graph`, causing dialect params to collide; kwargs are now split into `init_kwargs` and `call_kwargs` before forwarding
+
+- **Dead `node_id_lookup` dict removed from `_prepare_export`** (#665) by @KaifAhmad1
+  - The `{original_index → stable_id}` mapping was built on every export but never consumed; removed to avoid misleading future readers
+
+- **Dropped ambiguous `format="neo4j"` alias from `export_knowledge_graph` dispatch** (#665) by @KaifAhmad1
+  - `"neo4j"` is used throughout the codebase to identify the live Bolt/Cypher graph store backend; routing it silently to the offline bulk-CSV exporter would have confused callers; only `"neo4j_csv"` and `"neo4j-csv"` are accepted
+
+- **`export_usage.md` documented non-existent constructor and function parameters** (#665) by @KaifAhmad1
+  - Examples showed `node_label_sep` (correct: `label_separator`), `strict_validation` (correct: `strict`), and `nodes_path`/`rels_path` kwargs that do not exist; all three examples corrected to match the actual API
+
+- **Public API Ingestion Support** (#602) by @Luffy2208
+  - Added `PublicAPIIngestor` class built on top of `RESTIngestor` for credential-free REST endpoints
+  - Added `PublicAPIExample` and `PublicAPIExamples` catalog with 6 pre-configured no-auth examples:
+    - `jsonplaceholder_posts`, `jsonplaceholder_users`, `jsonplaceholder_todos` — fake REST resources for testing
+    - `rest_countries_all` — country reference data
+    - `data_gov_datasets` — Data.gov CKAN catalog search
+    - `open_meteo_forecast` — weather forecast (Berlin sample)
+  - Added `PublicAPIDetection` dataclass for endpoint-level public/no-auth detection
+  - Endpoint-level public API detection via `detect_public_api()` (informational, never raises)
+  - No-auth validation: rejects `Authorization`, `X-Api-Key`, and all common auth headers before sending the request
+  - Auth credential detection in URL query strings (`api_key=`, `token=`, `access_token=`, etc.)
+  - Polite rate limiting with per-request and per-ingestor `rate_limit_delay` controls
+  - Response parsing for JSON, CSV, and XML with `response_format="auto"` content-type detection
+  - HTML response guard — `text/html` responses are never misclassified as XML
+  - Nested `record_path` dot-notation extraction (e.g. `"result.results"` for Data.gov envelope)
+  - `_to_records` normalization with automatic envelope unwrapping for `items`, `data`, `results`, `records` keys
+  - `batch_public_apis()` for multi-endpoint ingestion with optional `fail_fast`
+  - `ingest_examples()` for bulk example ingestion
+  - `sample_response()` fixtures on `PublicAPIExamples` for mocked unit tests without live network calls
+  - `ingest_public_api()` convenience function and `ingest(..., source_type="public_api")` unified dispatch
+  - `source_type="api"` alias supported in `ingest()`
+  - Registry integration: `public_api` and `api` task namespaces with `endpoint`, `example`, `detect`, `batch`, `examples` methods
+  - Lazy-import exports of `RESTIngestor`, `APIData`, `PublicAPIIngestor`, `PublicAPIExample`, `PublicAPIExamples`, `PublicAPIDetection` from `semantica.ingest`
+  - Documentation: updated `docs/reference/ingest.md`, `docs/modules.md`, and `semantica/ingest/ingest_usage.md` with full usage examples
+  - 18 mocked tests covering JSON/CSV/XML parsing, nested record extraction, auth rejection, detection, string boolean config, batch dispatch, and unified `ingest()` routing
+  - 3 optional-import tests covering `defusedxml` fallback path and import isolation without web-scraping backends
+
+### Fixed
+
+- **Public API XML parsing hardened against malicious payloads** (#602) by @Luffy2208
+  - Replaced stdlib `xml.etree.ElementTree` with `defusedxml.ElementTree` (XXE/entity-expansion safe); falls back to a hardened `lxml` parser (`resolve_entities=False`, `no_network=True`, `load_dtd=False`, `huge_tree=False`) when `defusedxml` is not installed
+  - Added regression test asserting XXE entity payloads raise `ProcessingError`
+
+- **`validate_no_auth` config value not honoured when passed as a string** (#602) by @Luffy2208
+  - `bool("false")` evaluated to `True`, making `validate_no_auth=False` impossible via config files or environment variables; replaced with explicit `_coerce_bool()` that maps `"false"`, `"0"`, `"no"`, `"off"` → `False` and rejects unrecognised strings with `ValidationError`
+
+- **Auth credential detection extended to URL query strings** (#602) by @Sameer6305
+  - `detect_public_api()` and `ingest_public_api()` now scan the endpoint URL itself for auth parameters (`api_key`, `token`, `access_token`, etc.) via `urllib.parse.parse_qs`, not only request headers and explicit `params=` dicts
+  - Added regression tests for URL auth rejection (3 parametrized cases)
+
+- **`ingest_examples` and `batch_public_apis` mutable options mutation** (#602)
+  - Shared `**options` dict was passed by reference across loop iterations; mutable values such as `params` dicts were silently mutated after the first call, causing subsequent calls to receive a different (partially modified) options set; fixed by deep-copying options on each iteration
+
+- **`rate_limit_delay` forwarded twice in `ingest_public_api` method dispatcher** (#602)
+  - `rate_limit_delay` was consumed by the `PublicAPIIngestor` constructor via `config` but also leaked into `request_kwargs` forwarded to the ingestor method; added to the `config_only_key` strip list so it is consumed once at construction time only
+
+- **XML File Ingestion Support** (#560) by @Luffy2208
+  - Added `XMLIngestor` class with `lxml` backend for parsing local XML files
+  - Nested element hierarchy and flat element list extraction
+  - Namespace and prefix extraction with collision handling
+  - Attribute and element metadata extraction
+  - Optional XSD schema validation with detailed error reporting
+  - Optional DTD validation (internal and external)
+  - Secure-by-default parser (`resolve_entities=False`, `no_network=True`) blocking XXE attacks
+  - `ingest_xml()` convenience function and `ingest_file(..., method="xml")` support
+  - Unified `.xml` auto-detection via `ingest("file.xml")`
+  - Directory ingestion with recursive scanning and `fail_fast` support
+  - `ingest_string()` for in-memory XML bytes/str ingestion
+  - Comprehensive test coverage (8/8 tests passing)
+
+### Fixed
+
+- **NERExtractor LLM method returning pattern-based output on custom gateways** (#554, PR #556) by @KaifAhmad1
+
+  `NERExtractor(method="llm")` silently fell back to regex/pattern extraction when used with OpenAI-compatible enterprise or self-hosted gateways (Qwen, LLaMA proxies, internal routing layers). Returned entities carried `extraction_method='pattern'` even though the LLM itself was producing correct tool-call output. Three root causes fixed:
+
+  - **Silent exception swallowing** — `exc_info=True` was missing from the method-failure `WARNING` in `NERExtractor.extract_entities`. The full gateway-rejection traceback was invisible in logs even with `DEBUG` level enabled, making the failure impossible to diagnose without reading source code.
+
+  - **`response_format=json_object` sent to incompatible gateways** — `OpenAIProvider.generate_structured` unconditionally included `response_format={"type": "json_object"}` in every API call. Custom/enterprise gateways frequently reject this parameter, causing both the `instructor` path and the manual repair loop to fail with the same error on every retry, eventually triggering `_extract_fallback` (pattern extraction).
+
+  - **No fallback in the `generate_typed` manual repair loop** — when `generate_structured` itself raised (due to gateway rejection), the repair loop retried the identical failing call up to `max_retries` times before giving up. There was no path to recover via plain `generate()` + JSON parsing.
+
+  **Additional fixes applied during PR review:**
+
+  - Mode.JSON retry in `generate_typed` now strips `response_format` from `create_kwargs` before forwarding to the retry client, preventing incompatible kwargs from being sent to a client configured for a different instructor mode.
+  - `exc_info=True` added to the `generate_structured` fallback warning in the manual repair loop for consistent observability across all failure paths.
+  - Removed dead duplicate `is_available` definition in `GroqProvider` — Python silently kept only the second definition; the first was unreachable.
+  - `OpenAIProvider._init_client` now validates `base_url` scheme at construction time. Non-HTTP(S) schemes (`file://`, `ftp://`, `javascript:`, etc.) raise `ValueError` immediately, preventing SSRF if `base_url` originates from configuration rather than hardcoded values.
+
+  **17 regression tests** added in `tests/test_issue_554_fixes.py` covering all bug paths, including harshalizode's exact gateway configuration.
+
+### Security
+
+- **GitHub Actions workflow permissions hardened** — added explicit `permissions: contents: read` + `security-events: write` block to `defender-for-devops.yml`, resolving CodeQL alert [actions/missing-workflow-permissions](https://github.com/semantica-agi/semantica/security/code-scanning/25) (CWE: principle of least privilege).
+
+- **DOMPurify upgraded to 3.4.0+ via npm overrides** — `monaco-editor` pinned `dompurify` at 3.2.7; added `overrides` in `explorer/package.json` to force `^3.4.0` (resolved to 3.4.10). Fixes 6 Dependabot alerts:
+  - Prototype pollution → XSS bypass via `CUSTOM_ELEMENT_HANDLING` fallback (CVE-2026-41238 / GHSA-v9jr-rg53-9pgp)
+  - Mutation-XSS via re-contextualization into raw-text wrappers (GHSA-h8r8-wccr-v5f2)
+  - `SAFE_FOR_TEMPLATES` bypass in `RETURN_DOM` mode (CVE-2026-41239 / GHSA-crv5-9vww-q3g8)
+  - `ADD_TAGS` function-predicate bypasses `FORBID_TAGS` (GHSA-39q2-94rc-95cp / GHSA-h7mw-gpvr-xq4m)
+  - `ADD_ATTR` predicate skips URI validation, allowing `javascript:` URLs (GHSA-cjmm-f4jc-qw8r)
+  - `USE_PROFILES` prototype pollution allows event handlers (GHSA-cj63-jhhr-wcxv)
+
+- **`uuid` upgraded to 13.0.1+ via npm overrides** — bumped from 13.0.0 to 13.0.2, fixing missing buffer bounds check in `v3`/`v5`/`v6` APIs that allowed silent partial writes into caller-provided buffers (CVE-2026-41907 / GHSA-w5hq-g745-h8pq).
+
+- **Vite upgraded from 5.x to 6.4.3** — resolves path traversal in optimised-deps `.map` handling (CVE-2026-39365 / GHSA-4w7w-66w2-5vf9) and the esbuild dev-server CORS issue (GHSA-4w7w-66w2-5vf9). Bundled esbuild updated from 0.21.5 → 0.25.12.
+
+- **esbuild forced to 0.28.1+ via npm override** — vite 6.4.3 bundles esbuild 0.25.12 which is vulnerable to missing binary integrity verification in the Deno distribution module (GHSA-gv7w-rqvm-qjhr); added `"esbuild": "^0.28.1"` to `overrides` in `explorer/package.json`. `npm audit` now reports 0 vulnerabilities (Dependabot #15).
+
+- **Leaked Groq API keys removed from cookbook notebooks** — 6 hardcoded `GROQ_API_KEY` values (`gsk_...`) stripped from configuration cells in `supply_chain/01`, `intelligence/01`, `cybersecurity/01`, `cybersecurity/02`, `finance/01`, and `blockchain/02`; fallback replaced with empty string (secret scanning alerts #1–#6). Keys were already publicly exposed — rotate them in the Groq console.
+
+- **Leaked Groq API keys removed from additional cookbook notebooks** — 6 distinct hardcoded `GROQ_API_KEY` values stripped from 4 additional notebooks: `advanced_rag/01_GraphRAG_Complete`, `advanced_rag/02_RAG_vs_GraphRAG_Comparison`, `blockchain/01_DeFi_Protocol_Intelligence`, and `biomedical/01_Drug_Discovery_Pipeline` (secret scanning alerts #1–#6). Affected keys: `gsk_SLLE0...`, `gsk_S4dBVJ...`, `gsk_SLOv6...`, `gsk_lR6Qcj...`, `gsk_ToJis6...`, `gsk_LmbQBr...`; all publicly exposed since Dec 2025 — revoke in the Groq console and close the GitHub secret scanning alerts as "Revoked" in the Security tab.
+
+---
+
 ## [0.5.0] - 2026-05-11
 
 ### Added
@@ -173,13 +440,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
-## [Unreleased]
-
-### Added
-- Placeholder for future features and improvements
-
----
-
 ## [0.4.0] - 2026-04-08
 
 ### Added

Dockerfile

@@ -1,29 +1,40 @@
-FROM node:25-alpine AS frontend-builder
+# syntax=docker/dockerfile:1
+FROM node:26-alpine AS frontend-builder
 
-WORKDIR /app/semantica-explorer
-
-
-COPY semantica-explorer/package.json semantica-explorer/package-lock.json* ./
-
-
-RUN npm install
-
-
-COPY semantica-explorer/ ./
-RUN npm run build
+WORKDIR /app
+COPY explorer/package*.json ./explorer/
+WORKDIR /app/explorer
+RUN npm ci
 
+COPY explorer/ ./
+RUN mkdir -p /app/semantica && npm run build
 
 FROM python:3.14-slim AS runtime
 
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    FALKORDB_HOST=falkordb \
+    FALKORDB_PORT=6379 \
+    ALLOWED_ORIGINS=http://localhost:8000,http://127.0.0.1:8000
+
 WORKDIR /app
 
-COPY pyproject.toml ./
-COPY semantica/ ./semantica/
+RUN groupadd --system semantica \
+    && useradd --system --gid semantica --home-dir /app --shell /usr/sbin/nologin semantica
 
+COPY pyproject.toml README.md LICENSE MANIFEST.in ./
+COPY semantica/ ./semantica/
+COPY integrations/ ./integrations/
 COPY --from=frontend-builder /app/semantica/static ./semantica/static
 
-RUN pip install --no-cache-dir ".[explorer]"
+RUN pip install --no-cache-dir ".[explorer]" \
+    && chown -R semantica:semantica /app
+
+USER semantica
 
 EXPOSE 8000
 
-CMD ["python", "-m", "uvicorn", "semantica.explorer.app:app", "--host", "0.0.0.0", "--port", "8000"]
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
+    CMD python -c "import json, urllib.request; data=json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/health', timeout=3)); raise SystemExit(0 if data.get('status') == 'ok' else 1)"
+
+CMD ["python", "-m", "uvicorn", "semantica.explorer.app:app", "--host", "0.0.0.0", "--port", "8000"]

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include semantica/static *

SECURITY.md

@@ -24,7 +24,7 @@ Security vulnerabilities should be reported privately to prevent potential explo
 
 ### 2. Report Security Issue
 
-Create a [GitHub Security Advisory](https://github.com/Hawksight-AI/semantica/security/advisories/new) or contact us through [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) with "[SECURITY]" prefix.
+Create a [GitHub Security Advisory](https://github.com/semantica-agi/semantica/security/advisories/new) or contact us through [GitHub Issues](https://github.com/semantica-agi/semantica/issues) with "[SECURITY]" prefix.
 
 Include the following information:
 
@@ -156,8 +156,8 @@ We appreciate responsible disclosure. Security researchers who help us improve t
 
 For security-related questions or concerns:
 
-- **GitHub Issues**: [Create an issue](https://github.com/Hawksight-AI/semantica/issues) with "[SECURITY]" prefix
-- **GitHub Security Advisories**: [Report vulnerability](https://github.com/Hawksight-AI/semantica/security/advisories/new)
+- **GitHub Issues**: [Create an issue](https://github.com/semantica-agi/semantica/issues) with "[SECURITY]" prefix
+- **GitHub Security Advisories**: [Report vulnerability](https://github.com/semantica-agi/semantica/security/advisories/new)
 
 ## Additional Resources
 

SUPPORT.md

@@ -20,8 +20,8 @@ Start with our comprehensive documentation:
 
 **Best for**: General questions, feature discussions, and getting help
 
-- [Ask a question](https://github.com/Hawksight-AI/semantica/discussions/new?category=q-a)
-- [Browse discussions](https://github.com/Hawksight-AI/semantica/discussions)
+- [Ask a question](https://github.com/semantica-agi/semantica/discussions/new?category=q-a)
+- [Browse discussions](https://github.com/semantica-agi/semantica/discussions)
 
 #### Discord
 
@@ -33,8 +33,8 @@ Start with our comprehensive documentation:
 
 **Best for**: Bug reports and feature requests
 
-- [Report a bug](https://github.com/Hawksight-AI/semantica/issues/new?template=bug_report.md)
-- [Request a feature](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md)
+- [Report a bug](https://github.com/semantica-agi/semantica/issues/new?template=bug_report.md)
+- [Request a feature](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md)
 
 ### Before Asking
 
@@ -47,7 +47,7 @@ Start with our comprehensive documentation:
 
 ### Bug Reports
 
-Use our [bug report template](https://github.com/Hawksight-AI/semantica/issues/new?template=bug_report.md) to report bugs.
+Use our [bug report template](https://github.com/semantica-agi/semantica/issues/new?template=bug_report.md) to report bugs.
 
 Include:
 - Clear description of the bug
@@ -58,7 +58,7 @@ Include:
 
 ### Feature Requests
 
-Use our [feature request template](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md) to suggest features.
+Use our [feature request template](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md) to suggest features.
 
 Include:
 - Problem statement
@@ -71,7 +71,7 @@ Include:
 **Do NOT** create a public issue for security vulnerabilities.
 
 Instead:
-- Email: semantica-dev@users.noreply.github.com
+- Email: kaif@getsemantica.ai
 - Subject: [SECURITY] Brief description
 - See [Security Policy](SECURITY.md) for details
 
@@ -79,7 +79,7 @@ Instead:
 
 For enterprise support, custom development, or consulting:
 
-- **Email**: semantica-dev@users.noreply.github.com
+- **Email**: kaif@getsemantica.ai
 - **Subject**: [ENTERPRISE] Your request
 
 ## Response Times

benchmarks/.gitignore

@@ -1,75 +0,0 @@
---- Python Standards ---
-
-pycache/
-*.py[cod]
-*$py.class
-*.so
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-
---- Virtual Environments ---
-
-.env
-.venv
-venv/
-ENV/
-
---- Benchmarks & Results ---
-
-Ignore all individual benchmark runs to avoid repository bloat
-
-benchmarks/results/run_*.json
-
-Ignore the .pytest_cache which can get quite large
-
-.pytest_cache/
-
-Ignore any temporary files created by benchmarks
-
-benchmarks/input_layer/*.txt
-
---- IMPORTANT: Keep the Baseline ---
-
-We want to track the 'gold standard' performance in Git
-
-!benchmarks/results/baseline.json
-
---- IDEs & Editors ---
-
-.idea/
-.vscode/
-*.swp
-*.swo
-.project
-.pydevproject
-.settings/
-
---- Jupyter Notebooks ---
-
-.ipynb_checkpoints
-
---- OS Specific ---
-
-.DS_Store
-Thumbs.db
-
---- Project Specific ---
-
-logs/
-*.log
-semantica.log

benchmarks/benchmark_results.md

@@ -1,343 +0,0 @@
-# Semantica Benchmark Suite Results
-
-## Executive Summary
-
-**Test Date**: February 7, 2026  
-**Total Benchmarks**: 138 passed, 1 skipped  
-**Test Duration**: 38 minutes 35 seconds  
-**Environment**: Windows 10, Intel i5-1135G7 @ 2.40GHz, Python 3.11.9  
-
-## Performance Overview
-
-| Module | Tests | Performance Grade | Status |
-|--------|-------|------------------|---------|
-| Input Layer | 6 | 🟢 Excellent | All passed |
-| Core Processing | 5 | 🟢 Excellent | All passed |
-| Context Memory | 2 | 🟢 Excellent | All passed |
-| Storage | 4 | 🟢 Excellent | All passed |
-| Ontology | 4 | 🟢 Excellent | All passed |
-| Export | 4 | 🟢 Excellent | All passed |
-| Visualization | 3 | 🟢 Excellent | All passed |
-| Quality Assurance | 2 | 🟢 Excellent | All passed |
-| Output Orchestration | 2 | 🟢 Excellent | All passed |
-| Context | 3 | 🟢 Excellent | All passed |
-
----
-
-## 📊 Detailed Benchmark Results
-
-### 🔄 Input Layer Benchmarks
-
-**Purpose**: Test document parsing, data ingestion, and text processing performance
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_json_parsing_throughput[1000]` | 27,365.2 | 36.54 | 35.62 | 40.13 | 0.99 | ✅ |
-| `test_json_parsing_throughput[5000]` | 5,541.6 | 180.45 | 165.73 | 194.32 | 11.42 | ✅ |
-| `test_csv_parsing_throughput[1000]` | 18,127.9 | 55.16 | 52.41 | 61.87 | 3.33 | ✅ |
-| `test_html_scraping_speed[100]` | 2,437.8 | 410.20 | 346.30 | 6,736.50 | 89.27 | ✅ |
-| `test_pdf_extraction_overhead[10]` | 9.36 | 106.84 | 11.63 | 91.87 | 62.48 | ✅ |
-| `test_python_ast_parsing` | 3,142.6 | 318.21 | 291.96 | 347.90 | 35.67 | ✅ |
-
-**Key Insights**:
-- JSON parsing scales linearly (5K items processed in 180ms)
-- HTML scraping shows high variance due to complexity
-- PDF extraction optimized for batch processing
-- AST parsing maintains sub-millisecond performance per operation
-
----
-
-### ⚙️ Core Processing Benchmarks
-
-**Purpose**: Test NER extraction, semantic analysis, and text processing algorithms
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_ner_ml_wrapper_overhead` | 2,480.3 | 403.18 | - | - | - | ✅ |
-| `test_ner_pattern_speed` | 1,440.1 | 694.42 | - | - | - | ✅ |
-| `test_ner_batch_throughput` | 2.33 | 429.70 | - | - | - | ✅ |
-| `test_similarity_calculation` | 3,142.6 | 318.21 | - | - | - | ✅ |
-| `test_clustering_algorithm` | 39.1 | 25,558.38 | 6,113.80 | 42,058.84 | 42,058.84 | ✅ |
-| `test_ner_ml_real_performance` | - | - | - | - | - | ⏭️ Skipped |
-
-**Key Insights**:
-- Pattern-based NER significantly outperforms ML approaches
-- Semantic clustering is computationally intensive (25s mean time)
-- Real spaCy ML test skipped due to mocked environment
-- Batch processing provides good throughput
-
----
-
-### 🧠 Context Memory Benchmarks
-
-**Purpose**: Test graph operations, memory storage, and retrieval logic
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_bfs_traversal_depth[1]` | 469.48 | 2.13 | 1.42 | 2.04 | 1.86 | ✅ |
-| `test_bfs_traversal_depth[2]` | 419.46 | 2.38 | 2.04 | 2.38 | 0.89 | ✅ |
-| `test_memory_storage_overhead` | 9.36 | 106.84 | 11.63 | 91.87 | 62.48 | ✅ |
-| `test_short_term_pruning` | 9.23 | 108.36 | 91.87 | 108.36 | 20.76 | ✅ |
-| `test_linking_operations` | 2,869.0 | 348.55 | 313.28 | 346.30 | 39.45 | ✅ |
-| `test_retrieval_logic[False]` | 2,437.8 | 410.20 | 347.90 | 410.20 | 89.27 | ✅ |
-| `test_retrieval_logic[True]` | 39.13 | 25,558.38 | 6,113.80 | 42,058.84 | 42,058.84 | ✅ |
-
-**Key Insights**:
-- BFS traversal scales linearly with graph depth
-- Memory storage optimized for batch operations
-- Retrieval pipeline maintains sub-millisecond performance for simple cases
-- Complex retrieval (with context) significantly increases processing time
-
----
-
-### 💾 Storage Layer Benchmarks
-
-**Purpose**: Test vector stores, triplet storage, and graph database operations
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_binary_raw_throughput` | 5.83 | 171.52 | 162.04 | 178.50 | 7.56 | ✅ |
-| `test_numpy_compression_speed[1000]` | 2.47 | 404.81 | 387.07 | 393.72 | 11.55 | ✅ |
-| `test_numpy_compression_speed[10000]` | 0.25 | 3,972.74 | 3,867.34 | 3,983.95 | 61.69 | ✅ |
-| `test_json_vector_overhead` | 0.66 | 1,504.93 | 1,471.47 | 1,443.15 | 29.39 | ✅ |
-| `test_triplet_conversion_overhead` | 87.71 | 11.40 | 5.51 | 157.91 | 21.54 | ✅ |
-| `test_bulk_loader_logic` | 2.03 | 492.98 | 304.90 | 40,477.30 | 2,084.37 | ✅ |
-
-**Key Insights**:
-- Binary vector storage is 8x faster than JSON serialization
-- Triplet conversion is highly optimized (11ms mean)
-- Bulk loading shows high variance due to retry logic
-- Vector compression scales linearly with data size
-
----
-
-### 🏗️ Ontology Benchmarks
-
-**Purpose**: Test ontology inference, serialization, and namespace management
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_property_inference_scaling[size0]` | 1,440.1 | 694.42 | 637.90 | - | 65.09 | ✅ |
-| `test_owl_xml_generation` | 516.92 | 1.93 | 1.02 | 1.93 | 1.42 | ✅ |
-| `test_rdf_serialization_formats[turtle]` | 457.77 | 2.18 | 1.90 | 2.18 | 0.48 | ✅ |
-| `test_rdf_serialization_formats[rdfxml]` | 357.26 | 2.80 | 2.23 | 2.80 | 0.79 | ✅ |
-| `test_owl_serialization_formats[xml]` | 85.55 | 11.69 | 8.51 | 11.69 | 5.73 | ✅ |
-| `test_owl_serialization_formats[turtle]` | 61.10 | 16.37 | 12.28 | 16.37 | 6.84 | ✅ |
-
-**Key Insights**:
-- RDF Turtle format is 2x faster than RDF/XML
-- OWL serialization efficient for large ontologies
-- Property inference is computationally intensive
-- XML formats show higher overhead than Turtle
-
----
-
-### 📤 Export Benchmarks
-
-**Purpose**: Test data export and serialization performance
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_json_parsing_throughput[1000]` | 27,365.2 | 36.54 | 35.62 | 40.13 | 0.99 | ✅ |
-| `test_csv_entity_export` | 18,127.9 | 55.16 | 52.41 | 61.87 | 3.33 | ✅ |
-| `test_json_parsing_throughput[5000]` | 5,541.6 | 180.45 | 165.73 | 194.32 | 11.42 | ✅ |
-| `test_yaml_serialization_overhead` | 2.33 | 429.70 | 357.29 | 429.70 | 68.83 | ✅ |
-| `test_graph_conversion_overhead[graphml]` | 62.16 | 16.09 | 10.74 | 16.09 | 16.84 | ✅ |
-| `test_graph_conversion_overhead[gexf]` | 55.43 | 18.04 | 15.80 | 18.04 | 1.82 | ✅ |
-
-**Key Insights**:
-- JSON export maintains excellent performance across data sizes
-- YAML serialization is slower but feature-rich
-- GraphML format is slightly faster than GEXF
-- Export performance scales linearly with data size
-
----
-
-### 📈 Visualization Benchmarks
-
-**Purpose**: Test graph visualization, analytics, and dashboard performance
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_network_evolution_frames` | 0.21 | 4,871.40 | 3,958.10 | 4,871.40 | 931.20 | ✅ |
-| `test_temporal_dashboard_assembly` | 0.11 | 9,209.90 | 3,327.40 | 9,209.90 | 5,644.20 | ✅ |
-| `test_graph_conversion_overhead[graphml]` | 62.16 | 16.09 | 10.74 | 16.09 | 16.84 | ✅ |
-| `test_graph_conversion_overhead[gexf]` | 55.43 | 18.04 | 15.80 | 18.04 | 1.82 | ✅ |
-
-**Key Insights**:
-- Complex visualizations are computationally expensive
-- Dashboard assembly suitable for periodic updates (not real-time)
-- Graph conversion is highly optimized
-- Network evolution requires significant processing time
-
----
-
-### 🔍 Quality Assurance Benchmarks
-
-**Purpose**: Test deduplication and conflict resolution algorithms
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_deduplication_algorithm` | 2.33 | 429.70 | 357.29 | 429.70 | 68.83 | ✅ |
-| `test_conflict_resolution` | 1,440.1 | 694.42 | 637.90 | - | 65.09 | ✅ |
-
-**Key Insights**:
-- Deduplication algorithms are efficient for batch processing
-- Conflict resolution maintains good performance
-- Both algorithms scale linearly with data size
-
----
-
-### 🎯 Output Orchestration Benchmarks
-
-**Purpose**: Test pipeline execution and parallelism performance
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_execution_pipeline_overhead` | 2,437.8 | 410.20 | 347.90 | 410.20 | 89.27 | ✅ |
-| `test_parallelism_scaling` | 39.13 | 25,558.38 | 6,113.80 | 42,058.84 | 42,058.84 | ✅ |
-
-**Key Insights**:
-- Pipeline execution maintains good performance
-- Parallelism scaling shows high variance due to threading overhead
-- Suitable for batch processing rather than real-time
-
----
-
-### 🔗 Context Benchmarks
-
-**Purpose**: Test graph operations and linking performance
-
-| Benchmark | Operations/sec | Mean Time (ms) | Min Time (ms) | Max Time (ms) | StdDev | Status |
-|-----------|----------------|----------------|---------------|---------------|---------|---------|
-| `test_graph_ops_performance` | 2,869.0 | 348.55 | 313.28 | 346.30 | 39.45 | ✅ |
-| `test_linking_operations` | 2,869.0 | 348.55 | 313.28 | 346.30 | 39.45 | ✅ |
-| `test_memory_storage_overhead` | 9.36 | 106.84 | 11.63 | 91.87 | 62.48 | ✅ |
-
-**Key Insights**:
-- Graph operations are highly optimized
-- Linking operations maintain consistent performance
-- Memory storage suitable for batch operations
-
----
-
-## 🎯 Performance Analysis
-
-### Top Performers (>10,000 ops/sec)
-1. **JSON Parsing (1K)**: 27,365.2 ops/sec
-2. **JSON Export (1K)**: 27,365.2 ops/sec
-3. **HTML Scraping**: 2,437.8 ops/sec
-4. **Similarity Calculation**: 3,142.6 ops/sec
-5. **AST Parsing**: 3,142.6 ops/sec
-
-### Performance Optimizations Needed
-1. **Network Evolution**: 0.21 ops/sec (4.87s mean)
-2. **Dashboard Assembly**: 0.11 ops/sec (9.21s mean)
-3. **Semantic Clustering**: 39.13 ops/sec (25.56s mean)
-4. **Vector JSON Export**: 0.66 ops/sec (1.50s mean)
-
-### Memory Efficiency
-- **Binary vs JSON**: 8x performance improvement with binary vector storage
-- **Batch Processing**: All algorithms show linear scaling
-- **Mock Environment**: Zero memory overhead from heavy dependencies
-
----
-
-## 📋 Regression Detection
-
-**Baseline Status**: ✅ New baseline established  
-**Regression Threshold**: 15% change with Z-score > 2.0  
-**Current Status**: ✅ No regressions detected  
-**Monitoring**: Active with 10% threshold for CI/CD
-
----
-
-## 🖥️ Environment Specifications
-
-### Hardware Configuration
-- **CPU**: Intel i5-1135G7 @ 2.40GHz (8 cores, 16 threads)
-- **Memory**: 16GB DDR4
-- **Storage**: NVMe SSD
-- **Architecture**: x64
-
-### Software Stack
-- **OS**: Windows 10 Pro (Build 19044)
-- **Python**: 3.11.9 (64-bit)
-- **Benchmark Framework**: pytest-benchmark 5.2.3
-- **Mock Environment**: Full heavy library mocking
-
-### Test Configuration
-- **Total Test Files**: 50
-- **Total Benchmarks**: 138
-- **Test Duration**: 38m 35s
-- **Success Rate**: 99.3% (138/139)
-
----
-
-## 🚀 Production Recommendations
-
-### High Performance Operations
-1. **Use JSON for data exchange** - 27K+ ops/sec
-2. **Binary vector storage** - 8x faster than JSON
-3. **Pattern-based NER** - Significantly faster than ML
-4. **Batch processing** - Linear scaling confirmed
-
-### Optimization Opportunities
-1. **Semantic clustering** - Algorithm optimization needed
-2. **Visualization dashboards** - Implement caching
-3. **YAML serialization** - Consider alternative libraries
-4. **Parallel execution** - Threading overhead analysis
-
-### CI/CD Integration
-- ✅ Environment-agnostic design
-- ✅ Statistical regression detection
-- ✅ Automated performance monitoring
-- ✅ Zero false positive rate
-
----
-
-## 📊 Test Coverage Matrix
-
-| Module | Coverage Areas | Test Count | Performance |
-|--------|----------------|------------|-------------|
-| **Input Layer** | JSON, CSV, HTML, PDF, AST parsing | 6 | 🟢 Excellent |
-| **Core Processing** | NER, similarity, clustering | 5 | 🟢 Excellent |
-| **Context Memory** | Graph ops, memory, retrieval | 2 | 🟢 Excellent |
-| **Storage** | Vectors, triplets, graphs | 4 | 🟢 Excellent |
-| **Ontology** | Inference, serialization | 4 | 🟢 Excellent |
-| **Export** | JSON, CSV, YAML, Graph formats | 4 | 🟢 Excellent |
-| **Visualization** | Networks, dashboards, analytics | 3 | 🟢 Excellent |
-| **Quality Assurance** | Deduplication, conflicts | 2 | 🟢 Excellent |
-| **Output Orchestration** | Pipelines, parallelism | 2 | 🟢 Excellent |
-| **Context** | Graph operations, linking | 3 | 🟢 Excellent |
-
----
-
-## 🏆 Conclusion
-
-The Semantica benchmark suite demonstrates **exceptional performance** across all modules:
-
-### ✅ Achievements
-- **138/138 benchmarks passed** (99.3% success rate)
-- **Sub-millisecond performance** for core operations
-- **Linear scalability** confirmed for batch processing
-- **Production-ready** performance characteristics
-- **Zero breaking changes** from benchmark addition
-
-### 🎯 Key Performance Metrics
-- **Ultra-fast text processing**: >10,000 ops/sec
-- **Efficient storage operations**: Binary format 8x faster
-- **Optimized graph algorithms**: Sub-millisecond traversal
-- **Scalable export formats**: Linear performance scaling
-
-### 🚀 Production Readiness
-- **Environment-agnostic**: Works in CI/CD and local
-- **Regression detection**: Statistical analysis active
-- **Comprehensive coverage**: All 10 modules tested
-- **Performance monitoring**: Automated baseline tracking
-
-The benchmark suite successfully provides a robust foundation for continuous performance monitoring and optimization of the Semantica framework.
-
----
-
-*Results generated on February 7, 2026 • Semantica Benchmark Suite v1.0 • Test Environment: Windows 10, Python 3.11.9*

benchmarks/benchmarks.md

@@ -1,72 +0,0 @@
-# Semantica Performance Benchmark Suite
-
-This document outlines the architecture, directory structure, and usage of the performance benchmarking suite for the Semantica Agentic RAG framework.
-
-## Architecture
-
-The suite is organized into modular layers mirroring the library's internal structure, which allows for isolated performance testing of specific components.
-
-### High-Level Design Principles
-
-- **Isolation:** Use of mocks to ensure benchmarks measure algorithm logic.
-
-- **Virtualization:** A custom `conftest.py` virtualization layer allows tests to run without heavy local dependencies.
-
-- **Pedantic Measurement:** High-iteration counts and statistical rounds to filter out system noise.
-
-## Directory Structure
-
-Based on the current production environment, the suite is organized as follows:
-
-|                       |                                                                    |
-| --------------------- | ------------------------------------------------------------------ |
-| Folder                | Description                                                        |
-| context/              | Low-level graph operations and memory storage logic.               |
-| context_memory/       | Agent-level memory management and GraphRAG retrieval patterns.     |
-| core_processing/      | Throughput tests for NER, extraction, and graph building.          |
-| export/               | Serialization benchmarks for JSON, CSV, RDF, and GraphML.          |
-| infrastructure/       | Support scripts, including the regression comparison engine.       |
-| input_layer/          | Ingestion, parsing, and splitting performance.                     |
-| normalize/            | Text cleaning, encoding handling, and date normalization.          |
-| ontology/             | Inference, serialization, and namespace management overhead.       |
-| output_orchestration/ | Parallelism and execution pipeline management.                     |
-| quality_assurance/    | Deduplication and conflict resolution strategies.                  |
-| results/              | Storage for benchmark JSON outputs and performance baselines.      |
-| storage/              | Latency tests for Vector stores (FAISS) and Triplet stores (Jena). |
-| visualization/        | Computational cost of layout algorithms and chart rendering.       |
-
-## Usage
-
-### Running the Suite
-
-To run the full suite and generate a new results file:
-
-```bash
-python benchmarks/benchmark_runner.py
-```
-
-### Strict Mode (CI/CD)
-
-The suite is designed to integrate with automated pipelines. Using the --strict flag will cause the runner to return a non-zero exit code if a performance regression greater than 15% is detected.
-
-```bash
-python benchmarks/benchmark_runner.py --strict
-```
-
-
-
-### Performance Comparison
-
-The comparison engine (infrastructure/compare.py) uses Z-scores to distinguish between actual performance regressions and environmental noise.
-
-- Regression: Change > 15% AND Z-score > 2.0.
-
-- Noise: Change > 15% but Z-score < 2.0.
-
-### Updating Baseline
-
-When a performance change is intentional (e.g., a more complex but necessary algorithm is added), update the "gold standard" baseline:
-
-```bash
-cp benchmarks/results/run_latest.json benchmarks/results/baseline.json
-```

benchmarks/benchmarks_runner.py

@@ -1,84 +0,0 @@
-import argparse
-import os
-import subprocess
-import sys
-from datetime import datetime
-
-
-def run_benchmarks():
-    """
-    Master Runner for Semantica Benchmarks.
-    """
-    parser = argparse.ArgumentParser(description="Run Semantica Benchmarks")
-    parser.add_argument(
-        "--strict", action="store_true", help="Fail script if performance regresses"
-    )
-    args = parser.parse_args()
-
-    print("Starting Semantica Benchmark Suite...")
-
-    timestamp = datetime.now().strftime("%Y%m%d_%H_%M_%S")
-    os.makedirs("benchmarks/results", exist_ok=True)
-
-    current_json = f"benchmarks/results/run_{timestamp}.json"
-    baseline_json = "benchmarks/results/baseline.json"
-
-    # Run Benchmarks
-    cmd = [
-        sys.executable,
-        "-m",
-        "pytest",
-        "benchmarks/",
-        "-p",
-        "no:typeguard",
-        "-p",
-        "no:langsmith",
-        "--benchmark-only",
-        f"--benchmark-json={current_json}",
-        "--benchmark-columns=min,mean,stddev,ops",
-        "--benchmark-sort=mean",
-    ]
-
-    print(f"Executing benchmarks... (saving to {current_json})")
-    result = subprocess.run(cmd)
-
-    if result.returncode != 0:
-        print("Benchmarks failed to execute (runtime errors).")
-        sys.exit(result.returncode)
-
-    print("Benchmarks completed execution.")
-
-    # Compare against Baseline
-    if os.path.exists(baseline_json):
-        print(f"Comparing against Baseline ({baseline_json})...")
-
-        if os.path.exists("benchmarks/infrastructure/compare.py"):
-            compare_cmd = [
-                sys.executable,
-                "benchmarks/infrastructure/compare.py",
-                baseline_json,
-                current_json,
-            ]
-
-            compare_result = subprocess.run(compare_cmd)
-
-            if compare_result.returncode != 0:
-                print("\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
-                print("   PERFORMANCE REGRESSION DETECTED")
-                print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
-                if args.strict:
-                    sys.exit(1)
-            else:
-                print("Performance is within acceptable limits.")
-        else:
-            print(
-                "Comparison script not found (benchmarks/infrastructure/compare.py). Skipping comparison."
-            )
-    else:
-        print("No baseline found. This run effectively sets the new baseline.")
-
-    print(f"\n[Action] To update baseline: cp {current_json} {baseline_json}")
-
-
-if __name__ == "__main__":
-    run_benchmarks()

benchmarks/conftest.py

@@ -1,355 +0,0 @@
-import importlib.abc
-import importlib.machinery
-import os
-import sys
-import tempfile
-import uuid
-from unittest.mock import patch
-
-import numpy as np
-import pytest
-
-# Import interception
-
-HEAVY_LIBS = {
-    "pdfplumber",
-    "docx",
-    "pptx",
-    "openpyxl",
-    "pandas",
-    "PIL",
-    "PIL.Image",
-    "PIL.ImageDraw",
-    "lxml",
-    "pytesseract",
-    "networkx",
-    "chardet",
-    "langdetect",
-    "neo4j",
-    "weaviate",
-    "qdrant_client",
-    "sentence_transformers",
-    "transformers",
-    "fastembed",
-    "spacy",
-    "thinc",
-    "torch",
-    "matplotlib",
-    "umap",
-    "pynndescent",
-    "fireworks",
-    "fireworks.client",
-    "docling",
-    "docling.document_converter",
-    "docling.backend",
-    "docling_core",
-    "docling_core.types",
-    "instructor",
-    "instructor.processing",
-    "instructor.core",
-    "instructor.providers",
-    "instructor.providers.fireworks",
-    "pyarrow",
-    "arrow",
-    "pa",
-}
-
-
-class MockMeta(type):
-    """Metaclass that only claims RobustMocks as instances."""
-
-    def __instancecheck__(cls, instance):
-        return hasattr(instance, "_is_robust_mock")
-
-    def __subclasscheck__(cls, subclass):
-        return True
-
-
-def create_mock_class(full_name: str):
-    return MockMeta(
-        full_name.split(".")[-1],
-        (object,),
-        {
-            "__module__": ".".join(full_name.split(".")[:-1]),
-            "__doc__": f"Mocked class {full_name}",
-            "__getattr__": lambda self, attr: RobustMock(f"{full_name}.{attr}"),
-            "__call__": lambda self, *args, **kwargs: RobustMock(full_name),
-            "__init__": lambda self, *args, **kwargs: None,
-            "__repr__": lambda self: f"<MockClass {full_name}>",
-        },
-    )
-
-
-class RobustMock:
-    def __init__(self, name: str = "mock"):
-        self.__name__ = name
-        self.__version__ = "9.9.9"
-        self._is_robust_mock = True
-        self.__path__ = []
-        self.__file__ = "mock_file.py"
-        self.__all__ = []
-
-    def __getattr__(self, name):
-        if name.startswith("__") and name.endswith("__"):
-            raise AttributeError(name)
-        full_name = f"{self.__name__}.{name}"
-        
-        # Special handling for common PIL patterns
-        if self.__name__.endswith("Image") and name == "Image":
-            return create_mock_class(full_name)
-        elif self.__name__.endswith("ImageDraw") and name == "ImageDraw":
-            return create_mock_class(full_name)
-        # Special handling for pyarrow patterns
-        elif self.__name__ in ["pa", "pyarrow", "arrow"] and name in ["schema", "Table", "Dataset", "array", "RecordBatch"]:
-            return create_mock_class(full_name)
-        # Capital names are classes
-        elif name and name[0].isupper():
-            return create_mock_class(full_name)
-        return RobustMock(full_name)
-
-    def __call__(self, *args, **kwargs):
-        return RobustMock(self.__name__)
-
-    def __iter__(self):
-        return iter([])
-
-    def __getitem__(self, item):
-        return RobustMock(f"{self.__name__}[{item}]")
-
-    def __len__(self):
-        return 0
-
-    def __bool__(self):
-        return True
-
-    def __hash__(self):
-        return id(self)
-
-    def __repr__(self):
-        return f"<RobustMock {self.__name__}>"
-
-
-class MockLoader(importlib.abc.Loader):
-    def create_module(self, spec):
-        mock_module = RobustMock(spec.name)
-        mock_module.__spec__ = spec
-        mock_module.__loader__ = self
-        mock_module.__package__ = spec.parent
-        return mock_module
-
-    def exec_module(self, module):
-        pass
-
-
-class MockFinder(importlib.abc.MetaPathFinder):
-    def find_spec(self, fullname, path, target=None):
-        # Check for exact matches first
-        if fullname in HEAVY_LIBS:
-            return importlib.machinery.ModuleSpec(fullname, MockLoader())
-        
-        # Check for prefix matches (e.g., PIL.Image, PIL.ImageDraw)
-        for lib in HEAVY_LIBS:
-            if fullname.startswith(lib + "."):
-                return importlib.machinery.ModuleSpec(fullname, MockLoader())
-        
-        # Special handling for PIL submodules
-        if fullname.startswith("PIL."):
-            return importlib.machinery.ModuleSpec(fullname, MockLoader())
-            
-        # Special handling for fireworks
-        if fullname.startswith("fireworks."):
-            return importlib.machinery.ModuleSpec(fullname, MockLoader())
-            
-        # Special handling for docling
-        if fullname.startswith("docling"):
-            return importlib.machinery.ModuleSpec(fullname, MockLoader())
-            
-        # Special handling for instructor
-        if fullname.startswith("instructor"):
-            return importlib.machinery.ModuleSpec(fullname, MockLoader())
-            
-        # Special handling for pyarrow
-        if fullname.startswith("pyarrow") or fullname.startswith("arrow"):
-            return importlib.machinery.ModuleSpec(fullname, MockLoader())
-            
-        return None
-
-
-if os.getenv("BENCHMARK_REAL_LIBS") != "1":
-    if not any(isinstance(f, MockFinder) for f in sys.meta_path):
-        sys.meta_path.insert(0, MockFinder())
-    
-    # Special handling for 'pa' alias that's commonly used for pyarrow
-    if "pa" not in sys.modules:
-        sys.modules["pa"] = RobustMock("pa")
-    
-    # Pre-emptively create a mock arrow_exporter module to prevent import errors
-    # This must happen BEFORE any semantica.export imports
-    import types
-    mock_arrow_module = types.ModuleType('semantica.export.arrow_exporter')
-    
-    # Create a mock ArrowExporter class with proper interface
-    class MockArrowExporter:
-        def __init__(self, *args, **kwargs):
-            pass
-        def __getattr__(self, name):
-            return lambda *args, **kwargs: f"Mock ArrowExporter.{name}"
-    
-    mock_arrow_module.ArrowExporter = MockArrowExporter
-    mock_arrow_module.ENTITY_SCHEMA = RobustMock("ENTITY_SCHEMA")
-    mock_arrow_module.RELATIONSHIP_SCHEMA = RobustMock("RELATIONSHIP_SCHEMA") 
-    mock_arrow_module.METADATA_SCHEMA = RobustMock("METADATA_SCHEMA")
-    mock_arrow_module.pa = RobustMock("pa")
-    
-    # Inject the mock module into sys.modules
-    sys.modules["semantica.export.arrow_exporter"] = mock_arrow_module
-
-# Infrastructure and Data Fixtures
-
-
-class NullTracker:
-    def start_tracking(self, *args, **kwargs):
-        return "dummy_id"
-
-    def update_tracking(self, *args, **kwargs):
-        pass
-
-    def stop_tracking(self, *args, **kwargs):
-        pass
-
-    def register_pipeline_modules(self, *args, **kwargs):
-        pass
-
-    def clear_pipeline_context(self, *args, **kwargs):
-        pass
-
-    def update_progress(self, *args, **kwargs):
-        pass
-
-    def update_progress_batch(self, *args, **kwargs):
-        pass
-
-    @property
-    def enabled(self):
-        return False
-
-    @enabled.setter
-    def enabled(self, value):
-        pass
-
-
-@pytest.fixture(autouse=True)
-def kill_io_overhead():
-    tracker = NullTracker()
-    with patch("semantica.utils.logging.get_logger"), patch(
-        "semantica.utils.progress_tracker.get_progress_tracker", return_value=tracker
-    ):
-        # Patch the export module to handle missing ArrowExporter
-        try:
-            from benchmarks.export.arrow_exporter import ArrowExporter, ENTITY_SCHEMA, RELATIONSHIP_SCHEMA, METADATA_SCHEMA
-            mock_arrow_module = RobustMock("semantica.export.arrow_exporter")
-            mock_arrow_module.ArrowExporter = ArrowExporter
-            mock_arrow_module.ENTITY_SCHEMA = ENTITY_SCHEMA
-            mock_arrow_module.RELATIONSHIP_SCHEMA = RELATIONSHIP_SCHEMA
-            mock_arrow_module.METADATA_SCHEMA = METADATA_SCHEMA
-        except ImportError:
-            mock_arrow_module = RobustMock("semantica.export.arrow_exporter")
-        
-        with patch.dict('sys.modules', {
-            'semantica.export.arrow_exporter': mock_arrow_module
-        }):
-            patches = []
-            for mod_name, module in list(sys.modules.items()):
-                if mod_name.startswith("semantica.") and hasattr(
-                    module, "get_progress_tracker"
-                ):
-                    p = patch.object(module, "get_progress_tracker", return_value=tracker)
-                    patches.append(p)
-            for p in patches:
-                p.start()
-            yield
-            for p in patches:
-                p.stop()
-
-
-class MockVectorStore:
-    def __init__(self, dim=384):
-        self.dim = dim
-
-    def embed(self, text: str):
-        return np.random.rand(self.dim).astype(np.float32)
-
-    def store_vectors(self, vectors, metadata):
-        pass
-
-    def search(self, query, limit=5):
-        return [
-            {"id": str(uuid.uuid4()), "score": 0.9, "content": "test", "metadata": {}}
-            for _ in range(limit)
-        ]
-
-
-@pytest.fixture
-def mock_vector_store():
-    return MockVectorStore()
-
-
-@pytest.fixture
-def generate_graph_data():
-    BASE_NS = "http://semantica.example.org/resource/"
-    PRED_NS = "http://semantica.example.org/predicate/"
-
-    def _gen(n_nodes: int = 100, avg_degree: int = 4):
-        nodes = [
-            {
-                "id": f"{BASE_NS}node/{i}",
-                "type": "Entity",
-                "properties": {"label": f"Node {i}"},
-            }
-            for i in range(n_nodes)
-        ]
-        edges = [
-            {
-                "source_id": f"{BASE_NS}node/{i}",
-                "target_id": f"{BASE_NS}node/{(i+1)%n_nodes}",
-                "type": f"{PRED_NS}conn",
-                "properties": {"w": 1.0},
-            }
-            for i in range(n_nodes)
-        ]
-        return nodes, edges
-
-    return _gen
-
-
-@pytest.fixture
-def populated_context_graph(generate_graph_data):
-    from semantica.context.context_graph import ContextGraph
-
-    def _create(n_nodes=1000):
-        g = ContextGraph()
-        nodes, edges = generate_graph_data(n_nodes)
-        g.add_nodes(nodes)
-        g.add_edges(edges)
-        return g
-
-    return _create
-
-
-@pytest.fixture
-def sample_text_file():
-    lines = ["Line " + str(i) for i in range(1000)]
-    content = "\n".join(lines)
-    with tempfile.NamedTemporaryFile(
-        mode="w+", delete=False, suffix=".txt", encoding="utf-8"
-    ) as tmp:
-        tmp.write(content)
-        tmp_path = tmp.name
-    yield tmp_path
-    if os.path.exists(tmp_path):
-        os.remove(tmp_path)
-
-
-@pytest.fixture
-def long_text_string():
-    return "benchmark " * 5000

benchmarks/context/conftest.py

@@ -1,23 +0,0 @@
-import pytest
-
-from semantica.context.agent_memory import AgentMemory
-from semantica.context.context_retriever import ContextRetriever
-
-
-@pytest.fixture
-def retriever_setup(mock_vector_store, populated_context_graph):
-    """
-    Sets up a fully configured retriever
-    """
-    kg = populated_context_graph(n_nodes=1000)
-
-    memory = AgentMemory(vector_store=mock_vector_store, knowledge_graph=kg)
-
-    retriever = ContextRetriever(
-        memory_store=memory,
-        knowledge_graph=kg,
-        vector_store=mock_vector_store,
-        hybrid_alpha=0.5,
-    )
-
-    return retriever

benchmarks/context/test_graph_ops.py

@@ -1,47 +0,0 @@
-import pytest
-
-from semantica.context.context_graph import ContextGraph
-
-
-@pytest.mark.benchmark(group="graph_traversal")
-@pytest.mark.parametrize("hops", [1, 2])
-def test_bfs_traversal_depth(benchmark, populated_context_graph, hops):
-    """Benchmarks the BFS neighbor retrieval at differnet depths."""
-    graph = populated_context_graph(n_nodes=2000)
-    start_node = list(graph.nodes.keys())[0]
-
-    def run():
-        return graph.get_neighbors(start_node, hops=hops)
-
-    benchmark.pedantic(run, iterations=5, rounds=10)
-
-
-@pytest.mark.benchmark(group="graph_construction")
-@pytest.mark.parametrize("size", [1000])
-def test_graph_ingestion_speed(benchmark, generate_graph_data, size):
-    """
-    Benchmarks the speed of adding nodes and edges to the
-    in-memory structure.
-    """
-
-    nodes, edges = generate_graph_data(n_nodes=size)
-
-    def run():
-        graph = ContextGraph()
-        graph.add_nodes(nodes)
-        graph.add_edges(edges)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-@pytest.mark.benchmark(group="graph_query")
-def test_graph_keyword_search(benchmark, populated_context_graph):
-    """
-    Benchmarks the linear scan keyword search over graph nodes.
-    """
-    graph = populated_context_graph(n_nodes=2000)
-
-    def run():
-        return graph.query("Node content 500")
-
-    benchmark.pedantic(run, iterations=5, rounds=10)

benchmarks/context/test_linking.py

@@ -1,32 +0,0 @@
-import pytest
-
-from semantica.context.context_graph import ContextGraph
-from semantica.context.entity_linker import EntityLinker
-
-
-@pytest.mark.benchmark(group="entity_linkiing")
-@pytest.mark.parametrize("num_entities_in_graph", [100, 1000])
-def test_entity_linking_complexity(benchmark, num_entities_in_graph):
-    """
-    Benchmarks finding links for extracted entities
-    against the existing graph.
-    """
-
-    graph = ContextGraph()
-    nodes = [
-        {"id": f"e_{i}", "type": "Entity", "properties": {"content": f"Entity {i}"}}
-        for i in range(num_entities_in_graph)
-    ]
-    graph.add_nodes(nodes)
-
-    graph_dict = graph.to_dict()
-
-    linker = EntityLinker(knowledge_graph=graph_dict, similarity_threshold=0.7)
-
-    # Simulate extraction
-    extracted_entities = [{"text": f"Entity {i}", "type": "Entity"} for i in range(5)]
-
-    def run():
-        return linker.link("dummy text", entities=extracted_entities)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)

benchmarks/context/test_memory_storage.py

@@ -1,40 +0,0 @@
-import pytest
-
-from semantica.context.agent_memory import AgentMemory
-
-
-@pytest.mark.benchmark(group="memory_io")
-def test_memory_storage_overhead(benchmark, mock_vector_store):
-    """
-    Benchmarks storing a memory item.
-    """
-    memory = AgentMemory(vector_store=mock_vector_store)
-    content = "This is nothing burger for benchmarking this memory thingy."
-    metadata = {"type": "conversation", "user": "u_1"}
-
-    def run():
-        return memory.store(content, metadata=metadata)
-
-    benchmark.pedantic(run, iterations=10, rounds=10)
-
-
-@pytest.mark.benchmark(group="memory_io")
-def test_short_term_pruning(benchmark, mock_vector_store):
-    """
-    Benchmarks the pruning logic when short-term memory
-    limit is hit.
-    """
-
-    def setup_overfilled_memory():
-        memory = AgentMemory(vector_store=mock_vector_store, short_term_limit=50)
-        # Pre-fill
-        for i in range(55):
-            memory.store(f"filler memory {i}")
-        return (memory,), {}
-
-    def run_prune(mem_instance):
-        mem_instance.store("Trigger Pruning")
-
-    benchmark.pedantic(
-        target=run_prune, setup=setup_overfilled_memory, iterations=1, rounds=20
-    )

benchmarks/context/test_retrieval_logic.py

@@ -1,42 +0,0 @@
-import pytest
-
-from semantica.context.agent_memory import AgentMemory
-from semantica.context.context_retriever import ContextRetriever, RetrievedContext
-
-
-@pytest.mark.benchmark(group="rag_logic")
-def test_hybrid_ranking_overhead(benchmark, retriever_setup):
-    """
-    Benchmarks the CPU cost of the 'rank_and_merge' logic.
-    """
-
-    query = "test_query"
-
-    # Dummy results to sim inputs
-    raw_results = [
-        RetrievedContext(content=f"Vec {i}", score=0.9 - i * 0.01, source="vector:x")
-        for i in range(10)
-    ] + [
-        RetrievedContext(content=f"Graph {i}", score=0.8 - i * 0.01, source="graph:y")
-        for i in range(10)
-    ]
-
-    def run():
-        return retriever_setup._rank_and_merge(raw_results, query)
-
-    benchmark.pedantic(run, iterations=10, rounds=20)
-
-
-@pytest.mark.benchmark(group="rag_logic")
-@pytest.mark.parametrize("use_graph", [True, False])
-def test_full_retrieval_pipeline(benchmark, retriever_setup, use_graph):
-    """
-    Benchmarks the orchestration of the retrieve() method.
-    """
-
-    def run():
-        return retriever_setup.retrieve(
-            "Node content", max_results=10, use_graph_expansion=use_graph, max_hops=1
-        )
-
-    benchmark.pedantic(run, iterations=1, rounds=5)

benchmarks/context_memory/test_agentic.py

@@ -1,86 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.context.agent_context import AgentContext
-from semantica.context.context_retriever import RetrievedContext
-
-# Fixtures
-
-
-@pytest.fixture
-def mock_agent_context():
-    """
-    Creates an AgentContext with mocked internals.
-    """
-    vector_store = MagicMock()
-    knowledge_graph = MagicMock()
-
-    with patch("semantica.context.agent_context.AgentMemory") as MockMemory, patch(
-        "semantica.context.agent_context.ContextRetriever"
-    ) as MockRetriever:
-
-        ctx = AgentContext(vector_store=vector_store, knowledge_graph=knowledge_graph)
-
-        # Internal mocks
-
-        ctx._memory = MockMemory.return_value
-        ctx._retriever = MockRetriever.return_value
-
-        return ctx
-
-
-# Benchmarks
-
-
-def test_router_overhead(benchmark, mock_agent_context):
-    """
-    Benchmarks the logic that decides between Vector vs Graph retrieval.
-    """
-
-    mock_agent_context._retriever.retrieve.return_value = []
-
-    def op():
-        return mock_agent_context.retrieve("test query", use_graph=None)
-
-    benchmark.pedantic(op, iterations=50, rounds=20)
-
-
-def test_result_conversion_throughput(benchmark, mock_agent_context):
-    """
-    Benchmarks converting internal RetrievedContext objects to Dicts.
-    """
-
-    fake_results = [
-        RetrievedContext(
-            content=f"Result {i}",
-            score=0.9,
-            source="graph:node_1",
-            metadata={"type": "fact"},
-            related_entities=[{"id": "e1", "name": "Entity"}],
-            related_relationships=[{"source": "e1", "target": "e2"}],
-        )
-        for i in range(100)
-    ]
-    mock_agent_context._retriever.retrieve.return_value = fake_results
-
-    def op():
-        return mock_agent_context.retrieve("test", use_graph=True)
-
-    benchmark.pedantic(op, iterations=20, rounds=10)
-
-
-def test_store_orchestration_overhead(benchmark, mock_agent_context):
-    """
-    Benchmarks the 'store' method's logic for routing documents.
-    """
-    docs = [{"content": f"Doc {i}", "metadata": {"id": i}} for i in range(50)]
-
-    # Mock the internal storage to return immediately
-    mock_agent_context._memory.store.return_value = "mem_id"
-    mock_agent_context._build_graph_from_documents = MagicMock(return_value={})
-
-    def op():
-        return mock_agent_context.store(docs, extract_entities=False)
-
-    benchmark.pedantic(op, iterations=10, rounds=10)

benchmarks/context_memory/test_graphrag.py

@@ -1,244 +0,0 @@
-from dataclasses import dataclass, field
-from typing import Any, Dict, List
-from unittest.mock import patch
-
-import numpy as np
-import pytest
-
-from semantica.context.agent_context import AgentContext
-from semantica.context.agent_memory import AgentMemory
-from semantica.context.context_graph import ContextGraph
-from semantica.context.context_retriever import ContextRetriever, RetrievedContext
-from semantica.context.entity_linker import EntityLinker
-
-# Infra
-
-
-class NullTracker:
-    """
-    Stateless dummy tracker.
-    """
-
-    def start_tracking(self, *args, **kwargs):
-        return "dummy_id"
-
-    def update_tracking(self, *args, **kwargs):
-        pass
-
-    def stop_tracking(self, *args, **kwargs):
-        pass
-
-    def register_pipeline_modules(self, *args, **kwargs):
-        pass
-
-    def clear_pipeline_context(self, *args, **kwargs):
-        pass
-
-    def update_progress(self, *args, **kwargs):
-        pass
-
-    @property
-    def enabled(self):
-        return False
-
-    @enabled.setter
-    def enabled(self, value):
-        pass
-
-
-# ~~ MOCK STORES ~~
-
-
-class MockVectorStore:
-    """
-    A feather VectorStore sim that does no math.
-    We want to measure the MANAGER overhead.
-    """
-
-    def __init__(self):
-        self.vectors = {}
-        self.dim = 384
-
-    def embed(self, text):
-        return np.random.rand(self.dim).tolist()
-
-    def add(self, items):
-        for item in items:
-            self.vectors[item.memory_id] = item
-
-    def search(self, query, limit=5):
-        class MockResult:
-            def __init__(self, i):
-                self.id = f"mem_{i}"
-                self.content = f"Content for result {i} matching {query[:10]}"
-                self.score = 0.9 - (i * 0.05)
-                self.metadata = {"type": "test"}
-
-        return [MockResult(i) for i in range(limit)]
-
-
-def create_dense_graph(node_count):
-    """
-    Creates a ContextGraph with 'Small World' Topology.
-    Used to stress-test BFS traversal scaling.
-    """
-    graph = ContextGraph()
-
-    graph.progress_tracker = NullTracker()
-
-    # Create nodes
-    nodes = [
-        {
-            "id": f"node_{i}",
-            "type": "concept",
-            "properties": {"content": f"Concept {i}"},
-        }
-        for i in range(node_count)
-    ]
-    graph.add_nodes(nodes)
-
-    # Create Edges (Chain + Hub + Random)
-    edges = []
-    for i in range(node_count):
-        # Chain
-        if i < node_count - 1:
-            edges.append(
-                {"source_id": f"node_{i}", "target_id": f"node_{i+1}", "type": "next"}
-            )
-        # Hub
-        if i > 0:
-            edges.append(
-                {"source_id": "node_0", "target_id": f"node_{i}", "type": "hub_link"}
-            )
-        # Rando
-        if i % 5 == 0 and i + 5 < node_count:
-            edges.append(
-                {
-                    "source_id": f"node_{i}",
-                    "target_id": f"node_{i+5}",
-                    "type": "cross_link",
-                }
-            )
-
-    graph.add_edges(edges)
-    return graph
-
-
-def create_populated_memory(item_count):
-    """Creates an AgentMemory populated with N items."""
-    vs = MockVectorStore()
-    memory = AgentMemory(vector_store=vs)
-    memory.progress_tracker = NullTracker()
-
-    for i in range(item_count):
-        mem_id = f"setup_mem_{i}"
-        from datetime import datetime
-
-        from semantica.context.agent_memory import MemoryItem
-
-        memory.memory_items[mem_id] = MemoryItem(
-            content=f"History item {i}",
-            timestamp=datetime.now(),
-            memory_id=mem_id,
-            metadata={"type": "chat"},
-        )
-        memory.memory_index.append(mem_id)
-
-    return memory
-
-
-# ~~ BENCHMARKS ~~
-
-
-@pytest.mark.parametrize("graph_size", [100, 1000])
-@pytest.mark.parametrize("hops", [1, 2])
-def test_graph_traversal_scaling(benchmark, graph_size, hops):
-    """
-    Measures 'Hop Explosion' effect.
-    Retrieving multi-hop neighbors on a dense graph.
-    """
-    graph = create_dense_graph(graph_size)
-
-    def op():
-        # Start from'Hub' node which's celebrity, meaning
-        # connected to everyone
-        return graph.get_neighbors("node_0", hops=hops)
-
-    benchmark.pedantic(op, iterations=5, rounds=5)
-
-
-@pytest.mark.parametrize("memory_count", [100, 1000])
-def test_retriever_ranking_throughput(benchmark, memory_count):
-    """
-    Measures CPU cost of merging and ranking results.
-    """
-    retriever = ContextRetriever(
-        vector_store=MockVectorStore(),
-        memory_store=create_populated_memory(10),
-        knowledge_graph=None,
-        hybrid_alpha=0.5,
-    )
-    retriever.progress_tracker = NullTracker()
-
-    results = []
-    for i in range(memory_count):
-        results.append(
-            RetrievedContext(
-                content=f"Vector Item {i}",
-                score=np.random.random(),
-                source=f"vector:{i}",
-            )
-        )
-        results.append(
-            RetrievedContext(
-                content=f"Graph Item {i}",
-                score=np.random.random(),
-                source=f"graph:{i}",
-                metadata={"node_id": f"node_{i}"},
-            )
-        )
-
-    def op():
-        return retriever._rank_and_merge(results, "query context")
-
-    benchmark.pedantic(op, iterations=5, rounds=10)
-
-
-@pytest.mark.parametrize("registry_size", [100, 1000])
-def test_entity_linking_speed(benchmark, registry_size):
-    """
-    Measures O(N) linear scan speed in `find_similar_entities`.
-    """
-    linker = EntityLinker()
-    linker.progress_tracker = NullTracker()
-
-    mock_kg = {"entities": []}
-    for i in range(registry_size):
-        mock_kg["entities"].append(
-            {"id": f"ent_{i}", "text": f"Entity Number {i}", "type": "TEST"}
-        )
-    linker.knowledge_graph = mock_kg
-
-    input_text = "I am looking for Entity Number 50 in the database."
-
-    def op():
-        return linker.find_similar_entities(input_text, threshold=0.1)
-
-    benchmark.pedantic(op, iterations=5, rounds=5)
-
-
-@pytest.mark.parametrize("batch_size", [1, 10, 50])
-def test_agent_store_throughput(benchmark, batch_size):
-    """
-    'store' pipeline test.
-    """
-    vs = MockVectorStore()
-    context = AgentContext(vector_store=vs)
-    context._memory.progress_tracker = NullTracker()
-
-    inputs = [f"Memory item {i} for storage test" for i in range(batch_size)]
-
-    def op():
-        return context.batch_store(inputs)
-
-    benchmark.pedantic(op, iterations=5, rounds=5)

benchmarks/core_processing/conftest.py

@@ -1,44 +0,0 @@
-import pytest
-
-
-# Data factories
-@pytest.fixture
-def node_batch():
-    """Generates 1000 nodes for graph"""
-    return [
-        {
-            "id": f"node_{i}",
-            "type": "Concept",
-            "properties": {"name": f"Concept {i}", "weight": i / 1000},
-        }
-        for i in range(1000)
-    ]
-
-
-@pytest.fixture
-def edge_batch():
-    """Generates 1000 edges connection to the nodes."""
-    return [
-        {
-            "source_id": f"node_{i}",
-            "target_id": f"node_{i + 1}",
-            "type": "related to",
-            "weight": 0.5,
-        }
-        for i in range(999)
-    ]
-
-
-@pytest.fixture
-def conversation_data():
-    """Simulates a large conversation log"""
-    entities = [{"text": f"Entity_{i}", "type": "topic"} for i in range(50)]
-
-    return [
-        {
-            "id": "conv_1",
-            "content": "This is a conversation about banking.",
-            "entities": entities,
-            "relationships": [],
-        }
-    ]

benchmarks/core_processing/test_extraction.py

@@ -1,153 +0,0 @@
-from unittest.mock import patch
-
-import pytest
-
-from semantica.semantic_extract.ner_extractor import Entity, NERExtractor
-from semantica.semantic_extract.semantic_analyzer import SemanticAnalyzer
-
-
-# Fixtures
-@pytest.fixture
-def document_batch():
-    base = "The quick brown fox jumps over the lazy dog."
-    docs = [
-        f"{base} Variation {i}. Apple Inc released a product in 2024."
-        for i in range(50)
-    ]
-    return docs
-
-
-# Fast wrapper-only benchmark (always runs)
-def test_ner_ml_wrapper_overhead(benchmark, long_text_string):
-    extractor = NERExtractor(method="ml", model="en_core_web_sm")
-
-    entity_text = "Semantica"
-    phrase = f"{entity_text} is a knowledge graph framework. "
-    medium_text = phrase * 5
-
-    expected_entities = []
-    phrase_len = len(phrase)
-    for i in range(5):
-        start = i * phrase_len
-        end = start + len(entity_text)
-        ent = Entity(
-            text=entity_text,
-            label="ORG",
-            start_char=start,
-            end_char=end,
-            confidence=0.98,
-            metadata={"lemma": entity_text},
-        )
-        expected_entities.append(ent)
-
-    def custom_ml_extraction(text: str, **method_options):
-        min_confidence = method_options.get("min_confidence", 0.5)
-        entity_types = method_options.get("entity_types")
-        filtered = []
-        for ent in expected_entities:
-            if entity_types and ent.label not in entity_types:
-                continue
-            if ent.confidence >= min_confidence:
-                filtered.append(ent)
-        return filtered
-
-    with patch(
-        "semantica.semantic_extract.methods.get_entity_method"
-    ) as mock_get_method:
-        mock_get_method.side_effect = lambda name: (
-            custom_ml_extraction if name == "ml" else (lambda t, **o: [])
-        )
-
-        def op():
-            return extractor.extract_entities(text=medium_text)
-
-        result = benchmark.pedantic(op, rounds=20, iterations=5)
-
-    assert len(result) == 5
-    assert all(e.text == "Semantica" for e in result)
-    assert all(e.label == "ORG" for e in result)
-    assert all(e.confidence == 0.98 for e in result)
-    assert all(medium_text[e.start_char : e.end_char] == e.text for e in result)
-
-
-# Real spaCy benchmark
-@pytest.mark.benchmark(group="ner_real_ml")
-def test_ner_ml_real_performance(benchmark, long_text_string):
-    """
-    Full spaCy inference + wrapper overhead.
-    Only runs when real spaCy is loaded (BENCHMARK_REAL_LIBS=1).
-    """
-    extractor = NERExtractor(method="ml", model="en_core_web_sm")
-
-    if (
-        extractor.nlp is None
-        or not hasattr(extractor.nlp, "pipe_names")
-        or "ner" not in extractor.nlp.pipe_names
-    ):
-        pytest.skip(
-            "Real spaCy NER pipeline not available — skipping production benchmark"
-        )
-
-    medium_text = long_text_string[:10000]
-
-    medium_text += " Apple Inc. was founded by Steve Jobs and Steve Wozniak in Cupertino, California on April 1, 1976. Microsoft is a competitor."
-
-    def op():
-        return extractor.extract_entities(text=medium_text)
-
-    result = benchmark.pedantic(op, rounds=6, iterations=2)
-
-    assert len(result) >= 6
-    assert any("Apple" in e.text and e.label == "ORG" for e in result)
-    assert any(e.label == "PERSON" for e in result)
-    assert any(e.label in {"GPE", "LOC"} for e in result)
-    assert any(e.label == "DATE" for e in result)
-    assert any("Microsoft" in e.text and e.label == "ORG" for e in result)
-
-
-def test_ner_pattern_speed(benchmark, long_text_string):
-    extractor = NERExtractor(method="pattern")
-    medium_text = long_text_string[:50000]
-    text_with_entities = medium_text + " Apple Inc. was founded in 1976. "
-
-    def op():
-        return extractor.extract_entities(text=text_with_entities)
-
-    result = benchmark.pedantic(op, rounds=20, iterations=5)
-    assert len(result) > 0
-    assert result[0].label in ["ORG", "DATE", "UNKNOWN"]
-
-
-def test_ner_batch_throughput(benchmark, document_batch):
-    extractor = NERExtractor(method="pattern")
-
-    def run_batch():
-        return extractor.extract_entities_batch(document_batch, max_workers=2)
-
-    result = benchmark.pedantic(run_batch, rounds=10, iterations=5)
-    assert len(result) == len(document_batch)
-    assert len(result[0]) > 0
-
-
-def test_similarity_calculation(benchmark):
-    analyzer = SemanticAnalyzer()
-    text1 = "The quick brown fox jumps over the lazy dog" * 10
-    text2 = "The slow brown fox jumped over the sleeping dog" * 10
-
-    def op():
-        return analyzer.calculate_similarity(text1, text2, method="jaccard")
-
-    result = benchmark.pedantic(op, rounds=100, iterations=100)
-    assert 0.0 <= result <= 1.0
-
-
-def test_clustering_algorithm(benchmark, document_batch):
-    analyzer = SemanticAnalyzer()
-    options = {"similarity_threshold": 0.1}
-
-    def op():
-        return analyzer.cluster_semantically(texts=document_batch, **options)
-
-    result = benchmark.pedantic(op, rounds=10, iterations=5)
-    assert len(result) > 0
-    assert result[0].texts

benchmarks/core_processing/test_graph_building.py

@@ -1,56 +0,0 @@
-from unittest.mock import MagicMock
-
-import pytest
-
-from semantica.context.context_graph import ContextGraph
-
-
-def test_bulk_node_insertion(benchmark, node_batch):
-    """
-    Benchmarks the overhead of adding nodes to in-memory graph.
-
-    """
-
-    def setup_graph():
-        return (ContextGraph(),), {}
-
-    def run(graph_instance):
-        graph_instance.add_nodes(node_batch)
-
-    benchmark.pedantic(target=run, setup=setup_graph, rounds=50, iterations=1)
-
-
-def test_bulk_edge_insertion(benchmark, node_batch, edge_batch):
-    """
-    Benchmarks adding edges.
-    """
-
-    def setup_graph_with_nodes():
-        g = ContextGraph()
-        g.add_nodes(node_batch)
-        return (g,), {}
-
-    def run(graph_instance):
-        graph_instance.add_edges(edge_batch)
-
-    benchmark.pedantic(
-        target=run, setup=setup_graph_with_nodes, rounds=50, iterations=1
-    )
-
-
-def test_conversation_to_graph_conversion(benchmark, conversation_data):
-    """
-    Benchmarks parsing conversation dicts into graph structures.
-    """
-
-    def setup_clean_builder():
-        g = ContextGraph()
-        g.entity_linker = MagicMock()
-        return (g,), {}
-
-    def run(graph_instance):
-        return graph_instance.build_from_conversations(
-            conversation_data, link_entities=False
-        )
-
-    benchmark.pedantic(target=run, setup=setup_clean_builder, rounds=20, iterations=1)

benchmarks/export/arrow_exporter.py

@@ -1,69 +0,0 @@
-"""
-Mock Arrow Exporter for Benchmark Testing
-
-This module provides a mock implementation of the ArrowExporter to prevent
-import errors during benchmark testing when PyArrow is not available in the CI environment.
-"""
-
-# Mock PyArrow import for CI compatibility
-try:
-    import pyarrow as pa
-except ImportError:
-    # Create a mock pa module for CI environment
-    import types
-    pa = types.ModuleType('pa')
-    
-    def mock_schema(*args, **kwargs):
-        return types.SimpleNamespace()
-    
-    def mock_table(*args, **kwargs):
-        return types.SimpleNamespace()
-    
-    def mock_array(*args, **kwargs):
-        return types.SimpleNamespace()
-    
-    pa.schema = mock_schema
-    pa.Table = mock_table
-    pa.array = mock_array
-    pa.RecordBatch = mock_table
-
-# Mock schema definitions
-ENTITY_SCHEMA = pa.schema([]) if hasattr(pa, 'schema') else None
-RELATIONSHIP_SCHEMA = pa.schema([]) if hasattr(pa, 'schema') else None
-METADATA_SCHEMA = pa.schema([]) if hasattr(pa, 'schema') else None
-
-class ArrowExporter:
-    """
-    Mock Arrow Exporter class for benchmark testing.
-    
-    This is a lightweight implementation that provides the same interface
-    as the real ArrowExporter but doesn't require PyArrow to be installed.
-    """
-    
-    def __init__(self, config=None):
-        self.config = config
-        self._tables = {}
-    
-    def export_entities(self, entities, output_path):
-        """Mock export entities method."""
-        return f"Mock exported {len(entities)} entities to {output_path}"
-    
-    def export_relationships(self, relationships, output_path):
-        """Mock export relationships method."""
-        return f"Mock exported {len(relationships)} relationships to {output_path}"
-    
-    def export_knowledge_graph(self, entities, relationships, output_path):
-        """Mock export knowledge graph method."""
-        return f"Mock exported knowledge graph to {output_path}"
-    
-    def to_arrow_table(self, data):
-        """Mock conversion to Arrow table."""
-        return f"Mock Arrow table with {len(data)} rows"
-    
-    def save_to_file(self, table, path):
-        """Mock save to file method."""
-        return f"Mock saved table to {path}"
-    
-    def batch_export(self, data_list, output_dir):
-        """Mock batch export method."""
-        return f"Mock batch exported {len(data_list)} items to {output_dir}"

benchmarks/export/conftest.py

@@ -1,81 +0,0 @@
-import random
-import uuid
-from typing import Any, Dict, List
-
-import numpy as np
-import pytest
-
-# Data Generators
-
-
-@pytest.fixture
-def generate_entities():
-    def _gen(count: int) -> List[Dict[str, Any]]:
-        entities = []
-        for i in range(count):
-            entities.append(
-                {
-                    "id": f"e_{i}",
-                    "text": f"Entity Number {i}",
-                    "type": random.choice(
-                        ["person", "Organization", "Location", "Event"]
-                    ),
-                    "confidence": random.uniform(0.7, 1.0),
-                    "metadata": {"source": "doc_1.txt", "page": 1},
-                }
-            )
-
-        return entities
-
-    return _gen
-
-
-@pytest.fixture
-def generate_knowledge_graph(generate_entities):
-    def _gen(entity_count: int, rel_density: float = 1.5) -> Dict[str, Any]:
-        entities = generate_entities(entity_count)
-        relationships = []
-        rel_count = int(entity_count * rel_density)
-
-        for i in range(rel_count):
-            src = random.choice(entities)
-            tgt = random.choice(entities)
-            relationships.append(
-                {
-                    "id": f"r_{i}",
-                    "source_id": src["id"],
-                    "target_id": tgt["id"],
-                    "type": " RELATED_TO",
-                    "confidence": 0.9,
-                    "metadata": {"extractor": "v1"},
-                }
-            )
-
-        return {
-            "entities": entities,
-            "relationships": relationships,
-            "metadata": {"generated_at": "2026-02-05"},
-        }
-
-    return _gen
-
-
-@pytest.fixture
-def generate_vectors():
-    def _gen(count: int, dim: int = 384) -> List[Dict[str, Any]]:
-        matrix = np.random.rand(count, dim).astype(np.float32)
-
-        data = []
-
-        for i in range(count):
-            data.append(
-                {
-                    "id": f"vec_{i}",
-                    "vector": matrix[i].tolist(),
-                    "text": f"Text {i}",
-                    "metadata": {"model": "bert"},
-                }
-            )
-        return data
-
-    return _gen

benchmarks/export/test_core_formats.py

@@ -1,42 +0,0 @@
-import pytest
-
-from semantica.export.csv_exporter import CSVExporter
-from semantica.export.json_exporter import JSONExporter
-from semantica.export.yaml_exporter import SemanticNetworkYAMLExporter
-
-
-@pytest.mark.benchmark(group="structured_export")
-@pytest.mark.parametrize("size", [1000, 5000])
-def test_json_parsing_throughput(benchmark, tmp_path, generate_knowledge_graph, size):
-    kg = generate_knowledge_graph(size)
-    exporter = JSONExporter(indent=None)
-    output_file = tmp_path / "output.json"
-
-    def run():
-        exporter.export(kg, output_file)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-@pytest.mark.benchmark(group="structured_export")
-def test_csv_entity_export(benchmark, tmp_path, generate_entities):
-    entities = generate_entities(5000)
-    exporter = CSVExporter()
-    output_file = tmp_path / "entities.csv"
-
-    def run():
-        exporter.export_entities(entities, output_file)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-@pytest.mark.benchmark(group="structured_export")
-def test_yaml_serialization_overhead(benchmark, tmp_path, generate_knowledge_graph):
-    kg = generate_knowledge_graph(500)
-    exporter = SemanticNetworkYAMLExporter()
-    output_file = tmp_path / "output.yaml"
-
-    def run():
-        exporter.export(kg, output_file)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)

benchmarks/export/test_graph_export.py

@@ -1,22 +0,0 @@
-import pytest
-
-from semantica.export.graph_exporter import GraphExporter
-
-
-@pytest.mark.benchmark(group="vis_export")
-@pytest.mark.parametrize("format", ["graphml", "gexf"])
-def test_graph_conversion_overhead(
-    benchmark, tmp_path, generate_knowledge_graph, format
-):
-    """
-    Measures the cost of converting internal KG structure to XML-based graph formats.
-    Includes dictionary traversal and XML string building.
-    """
-    kg = generate_knowledge_graph(2000)
-    exporter = GraphExporter(format=format)
-    output_file = tmp_path / f"graph.{format}"
-
-    def run():
-        exporter.export_knowledge_graph(kg, output_file)
-
-    benchmark(run)

benchmarks/export/test_semantic_export.py

@@ -1,45 +0,0 @@
-import pytest
-
-from semantica.export.lpg_exporter import LPGExporter
-from semantica.export.owl_exporter import OWLExporter
-from semantica.export.rdf_exporter import RDFExporter
-
-
-@pytest.mark.benchmark(group="semantic_serialization")
-@pytest.mark.parametrize("format", ["turtle", "rdfxml"])
-def test_rdf_serialization_formats(benchmark, generate_knowledge_graph, format):
-    kg = generate_knowledge_graph(1000)
-    exporter = RDFExporter()
-    rdf_data = exporter.serializer.convert_kg_to_rdf(kg)
-
-    def run():
-        return exporter.export_to_rdf(rdf_data, format=format)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-@pytest.mark.benchmark(group="graph_db_export")
-def test_lpg_cypher_generation(benchmark, generate_knowledge_graph):
-    kg = generate_knowledge_graph(2000)
-    exporter = LPGExporter(batch_size=1000, include_indexes=False)
-
-    def run():
-        return exporter._generate_cypher_queries(kg)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-@pytest.mark.benchmark(group="semantic_serialization")
-def test_owl_xml_generation(benchmark, tmp_path):
-    ontology = {
-        "name": "BenchmarkOntology",
-        "classes": [{"name": f"Class{i}"} for i in range(500)],
-        "object_properties": [{"name": f"Prop{i}"} for i in range(200)],
-    }
-    exporter = OWLExporter()
-    output_file = tmp_path / "ontology.xml"
-
-    def run():
-        exporter.export(ontology, output_file, format="owl-xml")
-
-    benchmark.pedantic(run, iterations=1, rounds=5)

benchmarks/export/test_vector_export.py

@@ -1,51 +0,0 @@
-import numpy as np
-import pytest
-
-from semantica.export.vector_exporter import VectorExporter
-
-
-@pytest.mark.benchmark(group="vector_io")
-@pytest.mark.parametrize("count", [1000, 10000])
-def test_numpy_compression_speed(benchmark, tmp_path, generate_vectors, count):
-    """
-    Measures cost of np.savez_compressed.
-    """
-    vectors = generate_vectors(count)
-    exporter = VectorExporter(format="numpy")
-    output_file = tmp_path / "vectors.npz"
-
-    def run():
-        exporter.export(vectors, output_file)
-
-    benchmark(run)
-
-
-@pytest.mark.benchmark(group="vector_io")
-def test_json_vector_overhead(benchmark, tmp_path, generate_vectors):
-    """
-    Benchmarks JSON export for vectors.
-    """
-
-    vectors = generate_vectors(2000)
-    exporter = VectorExporter(format="json")
-    output_file = tmp_path / "vectors.json"
-
-    def run():
-        exporter.export(vectors, output_file)
-
-    benchmark(run)
-
-
-@pytest.mark.benchmark(group="vector_io")
-def test_binary_raw_throughput(benchmark, tmp_path, generate_vectors):
-    """
-    Measures raw binary dump speed (no compression, no metadata).
-    """
-    vectors = generate_vectors(10000)
-    exporter = VectorExporter(format="binary")
-    output_file = tmp_path / "vectors.bin"
-
-    def run():
-        exporter.export(vectors, output_file)
-
-    benchmark(run)

benchmarks/infrastructure/compare.py

@@ -1,102 +0,0 @@
-import argparse
-import json
-import sys
-from pathlib import Path
-from typing import Any, Dict, List
-
-
-def load_results(filepath: str) -> Dict[str, Any]:
-    with open(filepath, "r") as f:
-        return json.load(f)
-
-
-def calc_z_score(current_mean, base_mean, base_stddev):
-    """
-    Z-Score indicates how many standard deviations
-    away current run is from baseline
-    """
-
-    if base_stddev == 0:
-        return 0 if current_mean == base_mean else 100.0
-
-    return (current_mean - base_mean) / base_stddev
-
-
-def compare_benchmarks(
-    baseline: Dict[str, Any], current: Dict[str, Any], threshold_pct: float = 10.0
-):
-    """
-    Uses Mean for % change and Z-score for noise detection.
-    """
-
-    # colors for terminal
-    RED = "\033[91m"
-    GREEN = "\033[92m"
-    YELLOW = "\033[93m"
-    RESET = "\033[0m"
-
-    header = f"{'Benchmark':<60} | {'CHANGE %':<12} | {'SIGMA (Z)':<10} | {'STATUS'}"
-    print(header)
-    print("=" * len(header))
-
-    baseline_map = {b["name"]: b for b in baseline["benchmarks"]}
-    current_map = {b["name"]: b for b in current["benchmarks"]}
-
-    regressions = []
-
-    for name, curr in current_map.items():
-        base = baseline_map.get(name)
-        if not base:
-            print(f"{name:<60} | {'NEW':<12} | {'N/A':<10} | NEW")
-            continue
-
-        m1 = base["stats"]["mean"]
-        s1 = base["stats"]["stddev"]
-        m2 = curr["stats"]["mean"]
-
-        if m1 == 0:
-            delta_pct = 0.0
-        else:
-            delta_pct = ((m2 - m1) / m1) * 100
-
-        z_score = calc_z_score(m2, m1, s1)
-
-        status = f"{GREEN} OK{RESET}"
-
-        if delta_pct > threshold_pct:
-            if abs(z_score) > 2.0:
-                status = f"{RED} REGRESSION{RESET}"
-                regressions.append(name)
-            else:
-                status = f"{YELLOW} NOISE{RESET}"
-        elif delta_pct < -threshold_pct and abs(z_score) > 2.0:
-            status = f"{GREEN} IMPROVED{RESET}"
-
-        print(f"{name:<60} | {delta_pct:>+10.2f}% | {z_score:>9.2f} | {status}")
-
-    if regressions:
-        print(
-            f"\n{RED}FAILURE: Performance regression detected in {len(regressions)} tests.{RESET}"
-        )
-        return True
-    print(f"\n{GREEN}SUCCESS: No significant regressions.{RESET}")
-    return False
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("baseline", help="Gold standard JSON")
-    parser.add_argument("current", help="NEW RUN JSON")
-    parser.add_argument(
-        "--threshold", type=float, default=10.0, help="FAIL if slower by %"
-    )
-    args = parser.parse_args()
-
-    try:
-        failed = compare_benchmarks(
-            load_results(args.baseline), load_results(args.current), args.threshold
-        )
-        sys.exit(1 if failed else 0)
-    except FileNotFoundError as e:
-        print(f"Error loading files: {e}")
-        sys.exit(0)

benchmarks/input_layer/test_ingestion.py

@@ -1,22 +0,0 @@
-import pytest
-
-from semantica.ingest.file_ingestor import FileIngestor
-
-
-def test_ingest_file_performance(benchmark, sample_text_file):
-    """
-    Benchmarks the speed of the ingest_file method
-
-    Metrics:
-    - Time to open, read, validate and wrap a ~~10 KB text file.
-    """
-
-    ingestor = FileIngestor()
-    result = benchmark(
-        ingestor.ingest_file, file_path=sample_text_file, read_content=True
-    )
-
-    assert result is not None
-    assert result.size > 0
-    assert result.name.endswith(".txt")
-    assert "Line 0" in result.text

benchmarks/input_layer/test_parsing.py

@@ -1,188 +0,0 @@
-import csv
-import io
-import json
-import time
-from typing import Any, Dict, List
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.parse.code_parser import CodeParser
-from semantica.parse.csv_parser import CSVParser
-from semantica.parse.document_parser import DocumentParser
-from semantica.parse.html_parser import HTMLParser
-from semantica.parse.json_parser import JSONParser
-
-# Data gens
-
-
-def generate_json_string(item_count: int) -> str:
-    data = [
-        {
-            "id": i,
-            "name": f"Item:{i}",
-            "tags": ["tag1", "tag2", "tag3"],
-            "metadata": {"active": True, "score": 0.95},
-        }
-        for i in range(item_count)
-    ]
-    return json.dumps(data)
-
-
-def generate_csv_string(row_count: int) -> str:
-    output = io.StringIO()
-    writer = csv.writer(output)
-    writer.writerow(["id", "name", "description", "value", "date"])
-    for i in range(row_count):
-        writer.writerow([i, f"Item {i}", "Description text here", 100.50, "2024-01-01"])
-    return output.getvalue()
-
-
-def generate_html_string(element_count: int) -> str:
-    lis = "".join(
-        [f'<li><a href="/item/{i}">Link {i}</a></li>' for i in range(element_count)]
-    )
-    return f"""
-    <html>
-        <head><title>Benchmark Page</title></head>
-        <body>
-            <div id="content">
-                <h1>Header</h1>
-                <p>Some intro text.</p>
-                <ul>{lis}</ul>
-            </div>
-        </body>
-    </html>
-    """
-
-
-# lib mocks
-
-
-class MockPDFPage:
-    def __init__(self, page_num):
-        self.width = 600
-        self.height = 800
-        self.page_number = page_num
-
-    def extract_text(self):
-        return f"This is text content for page {self.page_number}. " * 50
-
-    def extract_tables(self):
-        return [[["Header1", "Header2"], ["Row1", "Value1"]]]
-
-    @property
-    def images(self):
-        return [{"x0": 10, "y0": 10, "width": 100, "height": 100}]
-
-
-class MockPDF:
-    def __init__(self, page_count):
-        self.pages = [MockPDFPage(i) for i in range(page_count)]
-        self.metadata = {"Title": "Benchmark PDF", "Author": "Noone"}
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        pass
-
-
-@pytest.fixture
-def mock_pdfplumber():
-    with patch("pdfplumber.open") as mock_open:
-        yield mock_open
-
-
-# Benchmarks
-
-
-@pytest.mark.parametrize("size", [1000, 10000])
-def test_json_parsing_throughput(benchmark, size):
-    parser = JSONParser()
-    json_str = generate_json_string(size)
-
-    with patch("pathlib.Path.exists", return_value=False):
-
-        def op():
-            return parser.parse(json_str)
-
-        benchmark.pedantic(op, iterations=5, rounds=10)
-
-
-@pytest.mark.parametrize("rows", [1000, 10000])
-def test_csv_parsing_throughput(benchmark, rows):
-    """
-    Measures CSV parsing throughput.
-    """
-    parser = CSVParser()
-    csv_content = generate_csv_string(rows)
-
-    with patch(
-        "builtins.open", side_effect=lambda *args, **kwargs: io.StringIO(csv_content)
-    ):
-        with patch("pathlib.Path.exists", return_value=True):
-
-            def op():
-                return parser.parse("dummy.csv")
-
-            benchmark.pedantic(op, iterations=5, rounds=5)
-
-
-@pytest.mark.parametrize("elements", [100, 1000])
-def test_html_scraping_speed(benchmark, elements):
-    parser = HTMLParser()
-    html_content = generate_html_string(elements)
-
-    with patch("pathlib.Path.exists", return_value=False):
-
-        def op():
-            return parser.parse(html_content, extract_links=True)
-
-        benchmark.pedantic(op, iterations=5, rounds=5)
-
-
-@pytest.mark.parametrize("pages", [10, 50])
-def test_pdf_extraction_overhead(benchmark, mock_pdfplumber, pages):
-    parser = DocumentParser()
-
-    mock_pdf = MockPDF(pages)
-    mock_pdfplumber.return_value = mock_pdf
-
-    with patch("pathlib.Path.exists", return_value=True), patch(
-        "pathlib.Path.suffix", new_callable=MagicMock(return_value=".pdf")
-    ):
-
-        def op():
-            return parser.parse_document("dummy.pdf", extract_images=True)
-
-        benchmark.pedantic(op, iterations=5, rounds=5)
-
-
-def test_python_ast_parsing(benchmark):
-    """
-    Measures performance of Python AST analysis.
-    """
-    parser = CodeParser()
-
-    code_lines = []
-    for i in range(200):
-        code_lines.append(f"import module_{i}")
-        code_lines.append(f"def function_{i}(arg):")
-        code_lines.append(f"    '''Docstring for function {i}'''")
-        code_lines.append(f"    return arg + {i}")
-        code_lines.append(f"class Class_{i}:")
-        code_lines.append(f"    pass")
-
-    code_content = "\n".join(code_lines)
-
-    with patch(
-        "builtins.open", side_effect=lambda *args, **kwargs: io.StringIO(code_content)
-    ), patch("pathlib.Path.exists", return_value=True), patch(
-        "pathlib.Path.suffix", new_callable=MagicMock(return_value=".py")
-    ):
-
-        def op():
-            return parser.parse_code("dummy.py")
-
-        benchmark.pedantic(op, iterations=5, rounds=5)

benchmarks/input_layer/test_splitting.py

@@ -1,27 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-try:
-    from semantica.split.sliding_window_chunker import SlidingWindowChunker
-    from semantica.split.splitter import TextSplitter
-except ImportError as e:
-    pytest.skip(
-        f"Skipping splitting test due to missing dependencies ({e})",
-        allow_module_level=True,
-    )
-
-
-def test_sliding_window(benchmark, long_text_string):
-    """
-    Benchmarks the speed of SlidingWindowChunker in 'Fixed Size' mode
-    """
-
-    chunker = SlidingWindowChunker(chunk_size=500, overlap=50)
-
-    if hasattr(chunker, "progress_tracker"):
-        chunker.progress_tracker = MagicMock()
-
-    result = benchmark(chunker.chunk, text=long_text_string, preserve_boundaries=False)
-
-    assert len(result) > 0

benchmarks/mock_arrow_exporter.py

@@ -1,69 +0,0 @@
-"""
-Mock Arrow Exporter for Benchmark Testing
-
-This module provides a mock implementation of the ArrowExporter to prevent
-import errors during benchmark testing when PyArrow is not available in the CI environment.
-"""
-
-# Mock PyArrow import for CI compatibility
-try:
-    import pyarrow as pa
-except ImportError:
-    # Create a mock pa module for CI environment
-    import types
-    pa = types.ModuleType('pa')
-    
-    def mock_schema(*args, **kwargs):
-        return types.SimpleNamespace()
-    
-    def mock_table(*args, **kwargs):
-        return types.SimpleNamespace()
-    
-    def mock_array(*args, **kwargs):
-        return types.SimpleNamespace()
-    
-    pa.schema = mock_schema
-    pa.Table = mock_table
-    pa.array = mock_array
-    pa.RecordBatch = mock_table
-
-# Mock schema definitions
-ENTITY_SCHEMA = pa.schema([]) if hasattr(pa, 'schema') else None
-RELATIONSHIP_SCHEMA = pa.schema([]) if hasattr(pa, 'schema') else None
-METADATA_SCHEMA = pa.schema([]) if hasattr(pa, 'schema') else None
-
-class ArrowExporter:
-    """
-    Mock Arrow Exporter class for benchmark testing.
-    
-    This is a lightweight implementation that provides the same interface
-    as the real ArrowExporter but doesn't require PyArrow to be installed.
-    """
-    
-    def __init__(self, config=None):
-        self.config = config
-        self._tables = {}
-    
-    def export_entities(self, entities, output_path):
-        """Mock export entities method."""
-        return f"Mock exported {len(entities)} entities to {output_path}"
-    
-    def export_relationships(self, relationships, output_path):
-        """Mock export relationships method."""
-        return f"Mock exported {len(relationships)} relationships to {output_path}"
-    
-    def export_knowledge_graph(self, entities, relationships, output_path):
-        """Mock export knowledge graph method."""
-        return f"Mock exported knowledge graph to {output_path}"
-    
-    def to_arrow_table(self, data):
-        """Mock conversion to Arrow table."""
-        return f"Mock Arrow table with {len(data)} rows"
-    
-    def save_to_file(self, table, path):
-        """Mock save to file method."""
-        return f"Mock saved table to {path}"
-    
-    def batch_export(self, data_list, output_dir):
-        """Mock batch export method."""
-        return f"Mock batch exported {len(data_list)} items to {output_dir}"

benchmarks/normalize/conftest.py

@@ -1,62 +0,0 @@
-import random
-import string
-from typing import Any, Dict, List
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-# Data gen
-
-
-@pytest.fixture
-def generate_text_data():
-    """Generates various types of text data."""
-
-    def _gen(type="clean", length=100):
-        if type == "clean":
-            return "".join(random.choices(string.ascii_letters + " ", k=length))
-        elif type == "html":
-            tags = ["<div>", "<p>", "<span>", "<a>", "<b>", "<i>"]
-            content = "".join(random.choices(string.ascii_letters + " ", k=length))
-            return f"{random.choice(tags)}{content}{random.choice(tags).replace('<', '</')}"
-        elif type == "unicode":
-            chars = string.ascii_letters + "éàèùâêîôûçñ"
-            return "".join(random.choices(chars, k=length))
-        elif type == "dirty":
-            chars = string.ascii_letters + " \t\n\r"
-            return "".join(random.choices(chars, k=length))
-
-    return _gen
-
-
-@pytest.fixture
-def generate_dataset():
-    """Generates dataset for data cleaner."""
-
-    def _gen(rows=100, duplicate_rate=0.0):
-        base_rows = []
-        unique_count = int(rows * (1 - duplicate_rate))
-
-        for i in range(unique_count):
-            base_rows.append(
-                {
-                    "id": i,
-                    "name": f"Entity_{i}",
-                    "email": f"user{i}@yahoo.com",
-                    "value": random.random() * 100,
-                    "category": random.choice(["A", "B", "C"]),
-                }
-            )
-
-        final_dataset = base_rows.copy()
-        while len(final_dataset) < rows:
-            source = random.choice(base_rows)
-            dup = source.copy()
-            if random.random() > 0.5:
-                dup["value"] = source["value"] + 0.001
-            final_dataset.append(dup)
-
-        random.shuffle(final_dataset)
-        return final_dataset
-
-    return _gen

benchmarks/normalize/test_data_cleaner.py

@@ -1,38 +0,0 @@
-import pytest
-
-from semantica.normalize.data_cleaner import DataCleaner
-
-
-@pytest.mark.parametrize("rows", [100, 500])
-def test_duplication_detection_scaling(benchmark, generate_dataset, rows):
-    """
-    Benchmarks duplicate detection scaling.
-    """
-
-    cleaner = DataCleaner()
-    dataset = generate_dataset(rows=rows, duplicate_rate=0.2)
-
-    def run():
-        return cleaner.detect_duplicates(dataset, key_fields=["name", "email"])
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-def test_missing_value_imputation(benchmark, generate_dataset):
-    """
-    Benchmarks statistical imputation.
-    """
-    cleaner = DataCleaner()
-
-    def setup_broken_dataset():
-        dataset = generate_dataset(rows=5000)
-        for row in dataset:
-            if row["id"] % 5 == 0:
-                row["value"] = None
-
-        return (dataset,), {}
-
-    def run(data):
-        return cleaner.handle_missing_values(data, strategy="impute", method="mean")
-
-    benchmark.pedantic(target=run, setup=setup_broken_dataset, iterations=1, rounds=10)

benchmarks/normalize/test_heavy_libs.py

@@ -1,31 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.normalize.encoding_handler import EncodingHandler
-from semantica.normalize.language_detector import LanguageDetector
-
-
-def test_language_detection_throughput(benchmark, generate_text_data):
-    """Benchmarks langdetect intergration."""
-    detector = LanguageDetector()
-    texts = [generate_text_data("clean", 200) for _ in range(50)]
-
-    def run():
-        return detector.detect_batch(texts)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-def test_encoding_detection(benchmark):
-    """Benchmarks chardet integration via EncodingHandler."""
-    handler = EncodingHandler()
-    data = (
-        b"Wowzaaa a simple string for encoding decoding , oh encoding detection just."
-        * 100
-    )
-
-    def run():
-        return handler.detect(data)
-
-    benchmark.pedantic(run, iterations=5, rounds=10)

benchmarks/normalize/test_parsers.py

@@ -1,25 +0,0 @@
-import pytest
-
-from semantica.normalize.date_normalizer import DateNormalizer
-from semantica.normalize.number_normalizer import NumberNormalizer
-
-
-@pytest.mark.parametrize("date_str", ["2026-02-03", "Ferbuary 2nd, 2026", "9 days ago"])
-def test_data_parsing_variations(benchmark, date_str):
-    """Compare speed of different date formats."""
-    normalizer = DateNormalizer()
-    benchmark.pedantic(
-        lambda: normalizer.normalize_date(date_str), iterations=10, rounds=20
-    )
-
-
-def test_number_normalization(benchmark):
-    """Benchmarks number parsing with currency and unit stripping."""
-    normalizer = NumberNormalizer()
-    raw_inputs = ["$1,234.56", "1.5k", "50%", "1,000,000"] * 100
-
-    def run():
-        for n in raw_inputs:
-            normalizer.normalize_number(n)
-
-    benchmark.pedantic(run, iterations=5, rounds=20)

benchmarks/normalize/test_text_processing.py

@@ -1,42 +0,0 @@
-import pytest
-
-from semantica.normalize.text_cleaner import TextCleaner
-from semantica.normalize.text_normalizer import TextNormalizer
-
-
-def test_html_removal_reg_vs_bs4(benchmark, generate_text_data):
-    """
-    Compare regex vs BeautifulSoup.
-    """
-    cleaner = TextCleaner()
-    html_content = generate_text_data("html", 10_000)
-
-    def run():
-        return cleaner.remove_html(html_content, preserve_structure=False)
-
-    benchmark.pedantic(run, rounds=50, iterations=10)
-
-
-def test_unicode_normalization_throughput(benchmark, generate_text_data):
-    """
-    Benchmarks unicode NFC normalization speed.
-    """
-    normalizer = TextNormalizer()
-    text = generate_text_data("unicode", 50_000)
-
-    def run():
-        return normalizer.normalize_text(text, unicode_form="NFC")
-
-    benchmark.pedantic(run, iterations=5, rounds=10)
-
-
-def test_whitespace_normalization(benchmark, generate_text_data):
-    """Benchmarks whitespace regex replacement."""
-    normalizer = TextNormalizer()
-    text = generate_text_data("dirty", 50_000)
-
-    benchmark.pedantic(
-        lambda: normalizer.normalize_text(text, unicode_form="NFC"),
-        iterations=5,
-        rounds=10,
-    )

benchmarks/ontology/conftest.py

@@ -1,85 +0,0 @@
-import random
-import string
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-# Data generators
-
-
-def _random_str(length=8):
-    return "".join(random.choices(string.ascii_letters, k=length))
-
-
-@pytest.fixture
-def generate_ontology_data():
-    """
-    Generates a synthetic dataset of entities and relationships
-    designed to triger class and property inference class.
-    """
-
-    def _generate(entity_count: int, relationship_density: float = 1.5):
-
-        num_classes = max(5, entity_count // 50)
-        class_names = [f"Class_{_random_str(4)}" for _ in range(num_classes)]
-
-        entities = []
-
-        for i in range(entity_count):
-            cls = random.choice(class_names)
-
-            props = {
-                f"prop_{_random_str(3)}": random.choice([10, "text", 1.5, True])
-                for _ in range(random.randint(1, 5))
-            }
-
-            entity = {
-                "id": f"e_{i}",
-                "type": cls,
-                "name": f"Entity_{i}",
-                "confidence": 0.95,
-                **props,
-            }
-
-            entities.append(entity)
-
-        relationships = []
-        rel_count = int(entity_count * relationship_density)
-        rel_types = ["relatedTo", "hasPart", "worksFor", "contains", "memberOf"]
-
-        for _ in range(rel_count):
-            src = random.choice(entities)
-            tgt = random.choice(entities)
-            rel = {
-                "source": src["name"],
-                "target": tgt["name"],
-                "type": random.choice(rel_types),
-                "source_type": src["type"],
-                "target_type": tgt["type"],
-                "confidence": 0.8,
-            }
-            relationships.append(rel)
-
-        return {"entities": entities, "relationships": relationships}
-
-    return _generate
-
-
-@pytest.fixture
-def large_ontology_definition(generate_ontology_data):
-    """Pre-calculates a structured ontology
-    definition dictionary.
-    """
-    from semantica.ontology.ontology_generator import OntologyGenerator
-
-    data = generate_ontology_data(entity_count=1000)
-
-    # Mocking validation in 6-step pipeline to speed up setup
-
-    with patch(
-        "semantica.ontology.ontology_validator.OntologyValidator.validate"
-    ) as mock_val:
-        mock_val.return_value.valid = True
-        gen = OntologyGenerator()
-
-        return gen.generate_ontology(data, validate=False)

benchmarks/ontology/test_inference.py

@@ -1,70 +0,0 @@
-import pytest
-
-from semantica.ontology.class_inferrer import ClassInferrer
-from semantica.ontology.property_generator import PropertyGenerator
-
-
-@pytest.mark.benchmark(group="class_Inference")
-@pytest.mark.parametrize("entity_count", [1000, 5000])
-def test_class_inference_scaling(benchmark, generate_ontology_data, entity_count):
-    """
-    Benchmarks grouping and threshold logic in ClassInferrer.
-    """
-
-    data = generate_ontology_data(entity_count=entity_count)
-    inferrer = ClassInferrer(min_occurrences=2)
-
-    def run():
-        return inferrer.infer_classes(data["entities"])
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-@pytest.mark.benchmark(group="property_inference")
-@pytest.mark.parametrize("size", [(1000, 1500)])
-def test_property_inference_scaling(benchmark, generate_ontology_data, size):
-    """
-    Benchmarks: PropertyGenerator
-    """
-
-    e_count, _ = size
-    data = generate_ontology_data(entity_count=e_count)
-
-    inferrer = ClassInferrer()
-    classes = inferrer.infer_classes(data["entities"])
-
-    prop_gen = PropertyGenerator()
-
-    def run():
-        return prop_gen.infer_properties(
-            entities=data["entities"],
-            relationships=data["relationships"],
-            classes=classes,
-        )
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-def test_hierarchy_circular_detection(benchmark):
-    """
-    Benchmarks the DFS cycle detection in ClassInferrer.
-    """
-
-    inferrer = ClassInferrer()
-
-    # Create a deep chain A -> B -> C ... -> Z
-
-    chain_length = 200
-    classes = []
-
-    for i in range(chain_length):
-        cls = {
-            "name": f"Class_{i}",
-            "subClassOf": f"Class_{i+1}" if i < chain_length - 1 else None,
-        }
-        classes.append(cls)
-
-    def run():
-        return inferrer.validate_classes(classes)
-
-    benchmark.pedantic(run, iterations=1, rounds=10)

benchmarks/ontology/test_ontology_pipeline.py

@@ -1,46 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.ontology.ontology_generator import OntologyGenerator
-
-
-@pytest.mark.benchmark(group="full_pipeline")
-@pytest.mark.parametrize("entity_count", [1000])
-def test_e2e_ontology_generation(benchmark, generate_ontology_data, entity_count):
-    """
-    Benchmarks complete 6-stage pipeline
-    """
-
-    data = generate_ontology_data(entity_count)
-    generator = OntologyGenerator()
-
-    with patch(
-        "semantica.ontology.ontology_validator.OntologyValidator.validate"
-    ) as mock_val:
-        mock_val.return_value.valid = True
-
-        def run():
-            return generator.generate_ontology(data, validate=True)
-
-        benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-def test_associative_class_creation(benchmark):
-    """
-    Benchmarks the creation of complex N-ary relationships.
-    """
-    from semantica.ontology.associative_class import AssociativeClassBuilder
-
-    builder = AssociativeClassBuilder()
-
-    def run():
-        for i in range(50):
-            builder.create_position_class(
-                person_class=f"Person_{i}",
-                organization_class=f"Org_{i}",
-                role_class=f"Role_{i}",
-                name=f"Position_{i}",
-            )
-
-    benchmark.pedantic(run, iterations=1, rounds=10)

benchmarks/ontology/test_reuse.py

@@ -1,43 +0,0 @@
-import pytest
-
-from semantica.ontology.namespace_manager import NamespaceManager
-from semantica.ontology.reuse_manager import ReuseManager
-
-
-def test_namespace_iri_generation(benchmark):
-    """
-    High-throughput test for IRI Generation.
-    """
-    manager = NamespaceManager(base_uri="https://semantica.dev/bench/")
-    names = [f"EntityName_{i}" for i in range(1000)]
-
-    def run():
-        for name in names:
-            manager.generate_class_iri(name)
-
-    benchmark.pedantic(run, iterations=1, rounds=20)
-
-
-def test_ontology_merging(benchmark, large_ontology_definition):
-    """
-    Benchmarks merging two large entities together.
-    """
-    manager = ReuseManager()
-    target = large_ontology_definition.copy()
-    source = large_ontology_definition.copy()
-
-    new_classes = []
-
-    for c in source["classes"]:
-        base_id = c.get("uri") or c.get("name") or "UnkownEntity"
-        new_c = c.copy()
-        new_c["uri"] = f"{base_id}_merged"
-        new_classes.append(new_c)
-
-    source["classes"] = new_classes
-
-    def run():
-        t_copy = target.copy()
-        return manager.merge_ontology_data(t_copy, source, overwrite=False)
-
-    benchmark.pedantic(run, iterations=1, rounds=10)

benchmarks/ontology/test_serialization.py

@@ -1,33 +0,0 @@
-import pytest
-
-from semantica.ontology.owl_generator import OWLGenerator
-
-
-@pytest.mark.benchmark(group="serialization")
-@pytest.mark.parametrize("format", ["turtle", "xml"])
-def test_owl_serialization_formats(benchmark, large_ontology_definition, format):
-    """Benchmarks the cost of serializing the ontology
-    to different string formats.
-    """
-    generator = OWLGenerator()
-
-    def run():
-        return generator.generate_owl(large_ontology_definition, format=format)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-def test_rdflib_graph_construction(benchmark, large_ontology_definition):
-    """
-    Benchmarks the creation of rdflib.Graph object.
-    """
-    generator = OWLGenerator()
-
-    def run():
-        if hasattr(generator, "_generate_with_rdflib"):
-            return generator._generate_with_rdflib(
-                large_ontology_definition, format="turtle"
-            )
-        return generator.generate_owl(large_ontology_definition)
-
-    benchmark.pedantic(run, iterations=1, rounds=5)

benchmarks/output_orchestration/test_execution_pipeline.py

@@ -1,98 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.pipeline.execution_engine import ExecutionEngine
-from semantica.pipeline.pipeline_builder import PipelineBuilder, StepStatus
-from semantica.pipeline.resource_scheduler import ResourceScheduler
-
-
-# ~~ Fixtures
-@pytest.fixture(autouse=True)
-def kill_hardware_checks():
-    with patch.object(ResourceScheduler, "_initialize_resources", return_value=None):
-        yield
-
-
-@pytest.fixture(autouse=True)
-def kill_logging():
-    with patch("semantica.utils.logging.get_logger"):
-        yield
-
-
-@pytest.fixture(autouse=True)
-def kill_tracker():
-    mock_tracker = MagicMock()
-    mock_tracker.enabled = False
-    with patch(
-        "semantica.pipeline.execution_engine.get_progress_tracker",
-        return_value=mock_tracker,
-    ):
-        yield
-
-
-def create_pipeline(size):
-    """Helper to generate pipelines of random size."""
-    builder = PipelineBuilder()
-    builder.progress_tracker = MagicMock()
-    builder.progress_tracker.enabled = False
-    handler = lambda x, **k: x
-
-    builder.add_step("start", "dummy", handler=handler)
-    for i in range(1, size):
-        builder.add_step(f"step_{i}", "dummy", handler=handler)
-        builder.connect_steps("start" if i == 1 else f"step_{i-1}", f"step_{i}")
-
-    return builder.build(f"bench_pipe_{size}")
-
-
-# ~~ Benchmarks ~~
-
-
-@pytest.mark.parametrize("step_count", [10, 100, 500])
-def test_pipeline_construction_scaling(benchmark, step_count):
-    """
-    Verifies if construction time scales linearly.
-    """
-
-    def op():
-        builder = PipelineBuilder()
-        builder.progress_tracker = MagicMock()
-        for i in range(step_count):
-            builder.add_step(f"s{i}", "t")
-        return builder.build()
-
-    benchmark.pedantic(op, iterations=5, rounds=5)
-
-
-@pytest.mark.parametrize("step_count", [10, 100])
-def test_execution_overhead_scaling(benchmark, step_count):
-    """
-    Measures per-step overhead as it gets more complex
-    """
-    engine = ExecutionEngine()
-    pipeline = create_pipeline(step_count)
-
-    def setup_run():
-        for step in pipeline.steps:
-            step.status = StepStatus.PENDING
-            step.result = None
-        return (pipeline,), {"data": {"val": 1}}
-
-    def op(pipeline, data):
-        return engine.execute_pipeline(pipeline, data=data)
-
-    benchmark.pedantic(op, setup=setup_run, iterations=1, rounds=10)
-
-
-@pytest.mark.parametrize("step_count", [10, 100, 1000])
-def test_topological_sort_scaling(benchmark, step_count):
-    """
-    Stress test for dependency graph algorithm.
-    """
-    engine = ExecutionEngine()
-    pipeline = create_pipeline(step_count)
-
-    benchmark.pedantic(
-        lambda: engine._topological_sort(pipeline.steps), iterations=20, rounds=10
-    )

benchmarks/output_orchestration/test_parallelism.py

@@ -1,91 +0,0 @@
-import time
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.pipeline.parallelism_manager import ParallelismManager, Task
-from semantica.pipeline.resource_scheduler import ResourceScheduler
-
-
-# ~~ Fixtures ~~
-@pytest.fixture(autouse=True)
-def kill_hardware_checks():
-    with patch.object(ResourceScheduler, "_initialize_resources", return_value=None):
-        yield
-
-
-@pytest.fixture(autouse=True)
-def kill_logging():
-    with patch("semantica.utils.logging.get_logger"):
-        yield
-
-
-@pytest.fixture(autouse=True)
-def kill_tracker():
-    mock_tracker = MagicMock()
-    mock_tracker.enabled = False
-    with patch(
-        "semantica.pipeline.parallelism_manager.get_progress_tracker",
-        return_value=mock_tracker,
-    ):
-        yield
-
-
-def blocking_task(duration):
-    """Simulates a task that waits for I/O (like a DB query or API call)."""
-    time.sleep(duration)
-    return True
-
-
-@pytest.fixture
-def thread_manager():
-    return ParallelismManager(max_workers=4, use_processes=False)
-
-
-@pytest.fixture
-def process_manager():
-    return ParallelismManager(max_workers=4, use_processes=True)
-
-
-# ~~ BENCHMARKS ~~
-
-
-def test_parallel_vs_serial_io(benchmark, thread_manager):
-    """
-    Runs 4 tasks that sleep for 0.1s.
-    """
-    tasks = [
-        Task(task_id=f"t{i}", handler=blocking_task, args=(0.1,)) for i in range(4)
-    ]
-
-    def op():
-        return thread_manager.execute_parallel(tasks)
-
-    benchmark.pedantic(op, iterations=1, rounds=5)
-
-
-def test_thread_pool_overhead(benchmark, thread_manager):
-    """
-    Measures the raw cost of spinning up threads for zero-work tasks.
-    """
-    # No-op handler
-    noop = lambda: None
-    tasks = [Task(task_id=f"t{i}", handler=noop) for i in range(100)]
-
-    def op():
-        return thread_manager.execute_parallel(tasks)
-
-    benchmark.pedantic(op, iterations=5, rounds=10)
-
-
-def test_process_pool_overhead(benchmark, process_manager):
-    """
-    Measures overhead of ProcessPoolExecutor
-    """
-    noop = lambda: None
-    tasks = [Task(task_id=f"t{i}", handler=noop) for i in range(10)]
-
-    def op():
-        return process_manager.execute_parallel(tasks)
-
-    benchmark.pedantic(op, iterations=1, rounds=5)

benchmarks/quality_assurance/test_conflicts.py

@@ -1,84 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.deduplication.merge_strategy import MergeStrategy, MergeStrategyManager
-
-# Fixtures
-
-
-@pytest.fixture
-def conflict_manager():
-    """Returns a MergeStrategyManager with default settings."""
-    return MergeStrategyManager()
-
-
-@pytest.fixture
-def conflicting_entities_batch():
-    """
-    Generates a list of 100 entities that are all 'duplicates' of each other
-    but have conflicting property values. This forces the resolution logic to run hard.
-    """
-    entities = []
-    for i in range(100):
-        entities.append(
-            {
-                "id": "e_1",
-                "name": f"Entity Name {i}",
-                "type": "Person",
-                "confidence": 0.5 + (i * 0.005),
-                "properties": {
-                    "age": 20 + i,
-                    "email": f"user{i}@example.com",
-                    "status": "active" if i % 2 == 0 else "inactive",
-                },
-                "relationships": [
-                    {"source": "e_1", "target": f"other_{i}", "type": "knows"}
-                ],
-            }
-        )
-    return entities
-
-
-# Benchmarks
-
-
-def test_strategy_keep_highest_confidence(
-    benchmark, conflict_manager, conflicting_entities_batch
-):
-    """
-    Benchmarks 'KEEP_HIGHEST_CONFIDENCE'.
-    """
-
-    def op():
-        return conflict_manager.merge_entities(
-            conflicting_entities_batch, strategy=MergeStrategy.KEEP_HIGHEST_CONFIDENCE
-        )
-
-    benchmark.pedantic(op, iterations=10, rounds=10)
-
-
-def test_strategy_merge_all(benchmark, conflict_manager, conflicting_entities_batch):
-    """
-    Benchmarks 'MERGE_ALL'.
-    """
-
-    def op():
-        return conflict_manager.merge_entities(
-            conflicting_entities_batch, strategy=MergeStrategy.MERGE_ALL
-        )
-
-    benchmark.pedantic(op, iterations=10, rounds=10)
-
-
-def test_property_resolution_overhead(benchmark, conflict_manager):
-    """
-    Micro-benchmark for the inner _resolve_property_conflict logic.
-    """
-
-    def op():
-        return conflict_manager._resolve_property_conflict(
-            "age", 25, 30, MergeStrategy.KEEP_MOST_COMPLETE
-        )
-
-    benchmark.pedantic(op, iterations=1000, rounds=20)

benchmarks/quality_assurance/test_deduplication.py

@@ -1,338 +0,0 @@
-import random
-import string
-import time
-from typing import Any, Dict, List
-from unittest.mock import patch
-
-import numpy as np
-import pytest
-
-from semantica.deduplication.cluster_builder import ClusterBuilder
-from semantica.deduplication.duplicate_detector import DuplicateDetector
-from semantica.deduplication.entity_merger import EntityMerger
-from semantica.deduplication.similarity_calculator import SimilarityCalculator
-
-# Infra
-
-
-class NullTracker:
-    """
-    Discards all data to prevent memory leaks
-    """
-
-    def start_tracking(self, *args, **kwargs):
-        return "dummy_id"
-
-    def update_tracking(self, *args, **kwargs):
-        pass
-
-    def stop_tracking(self, *args, **kwargs):
-        pass
-
-    def register_pipeline_modules(self, *args, **kwargs):
-        pass
-
-    def clear_pipeline_context(self, *args, **kwargs):
-        pass
-
-    def update_progress(self, *args, **kwargs):
-        pass
-
-    @property
-    def enabled(self):
-        return False
-
-    @enabled.setter
-    def enabled(self, value):
-        pass
-
-
-@pytest.fixture(autouse=True)
-def kill_io_overhead():
-    """
-    Replaces ProgressTracker with NullTracker globally.
-    """
-    with patch("semantica.utils.logging.get_logger"), patch(
-        "semantica.utils.progress_tracker.get_progress_tracker"
-    ) as mock_getter:
-
-        mock_getter.return_value = NullTracker()
-
-        with patch(
-            "semantica.deduplication.similarity_calculator.get_progress_tracker",
-            return_value=NullTracker(),
-        ), patch(
-            "semantica.deduplication.duplicate_detector.get_progress_tracker",
-            return_value=NullTracker(),
-        ), patch(
-            "semantica.deduplication.cluster_builder.get_progress_tracker",
-            return_value=NullTracker(),
-        ):
-            yield
-
-
-# Sim data
-
-
-def generate_entity_cluster(base_name: str, size: int) -> List[Dict[str, Any]]:
-    """
-    Generates a cluster of similar entities based on a seed name.
-    Example: "Apple" -> ["Apple Inc", "Apple Corp", etc.]
-    """
-
-    entities = []
-    suffixes = ["Inc", "Corp", "Ltd", "Gmbh", "LLC", "Group", "Systems"]
-
-    for i in range(size):
-        if random.random() < 0.8:
-            name = f"{base_name} {random.choice(suffixes)}"
-        else:
-            # Generating a typo for our calc to work on
-            chars = list(base_name)
-            if len(chars) > 2:
-                idx = random.randint(0, len(chars) - 2)
-                chars[idx], chars[idx + 1] = chars[idx + 1], chars[idx]
-            name = "".join(chars)
-
-        entities.append(
-            {
-                "id": f"{base_name.lower()}_{i}",
-                "name": name,
-                "type": "Organization",
-                "properties": {
-                    "location": "USA" if i % 2 == 0 else "California",
-                    "sector": "Tech",
-                    "employee_count": 100 + i,
-                },
-            }
-        )
-
-    return entities
-
-
-def generate_relationship_dataset(size: int) -> List[Dict[str, Any]]:
-    """
-    Generates a dataset of graph relationships/triplets.
-    Includes exact matches, synonym predicates, and dirty literal strings.
-    """
-    relationships = []
-    predicates = ["works_for", "employed_by", "is_employee_of", "has_employer"]
-    
-    for i in range(size):
-        # Base relationship
-        rel = {
-            "subject": f"Person_{i % 50}",
-            "predicate": random.choice(predicates),
-            "object": f"Company_{i % 10}"
-        }
-        relationships.append(rel)
-        
-        # Inject semantic duplicates (dirty literals / synonym predicates)
-        if random.random() < 0.4:
-            dirty_rel = {
-                "subject": f"Person_{i % 50}",
-                "predicate": random.choice(predicates),
-                "object": f"  Company_{i % 10} Inc.  " 
-            }
-            relationships.append(dirty_rel)
-            
-    return relationships
-
-
-def generate_dataset(
-    num_clusters: int, items_per_cluster: int, worst_case_blocking: bool = False
-):
-    """
-    Generates a full dataset
-
-    Args:
-      worst_case_blocking: If True, all names start with 'A' to defeat
-                           first-char blocking strategy in SimilarityCalculator.
-
-    """
-    dataset = []
-    for i in range(num_clusters):
-        if worst_case_blocking:
-            # All starts with 'A'
-            base_name = f"A_Company_{i}"
-        else:
-            start_char = random.choice(string.ascii_uppercase)
-            base_name = f"{start_char}_company_{i}"
-
-        cluster = generate_entity_cluster(base_name, items_per_cluster)
-        dataset.extend(cluster)
-
-    return dataset
-
-
-# ~~ Benchmarks ~~
-
-
-@pytest.mark.parametrize("method", ["levenshtein", "jaro_winkler"])
-def test_string_metric_speed(benchmark, method):
-    """
-    Measures the speed of string comparison algos.
-    """
-
-    calc = SimilarityCalculator()
-    s1 = "International Business Machines Corporation"
-    s2 = "International Business Machine Corp."
-
-    benchmark.pedantic(
-        lambda: calc.calculate_string_similarity(s1, s2, method=method),
-        iterations=1000,
-        rounds=100,
-    )
-
-
-def test_full_similarity_calculation(benchmark):
-    """
-    Measures weighted multi-factor calculation overhead.
-    (String + Property + Relationship + Weights).
-    """
-
-    calc = SimilarityCalculator(
-        string_weight=0.5, property_weight=0.3, relationship_weight=0.2
-    )
-
-    e1 = {
-        "name": "Acme Corp",
-        "properties": {"loc": "NY", "id": "123"},
-        "relationships": [{"target": "t1"}, {"target": "t2"}],
-    }
-
-    e2 = {
-        "name": "Acme Inc",
-        "properties": {"loc": "NY", "id": "123"},
-        "relationships": [{"target": "t1"}, {"target": "t2"}],
-    }
-
-    benchmark.pedantic(
-        lambda: calc.calculate_similarity(e1, e2), iterations=1000, rounds=50
-    )
-
-
-@pytest.mark.parametrize("dataset_size", [100, 500])
-def test_duplicate_detection_scaling_opt(benchmark, dataset_size):
-    """
-    Tests duplication on a 'Distributed' dataset (Best Case)
-    Now utilizing V2 Candidate Generation to ensure no regressions.
-    """
-    data = generate_dataset(
-        num_clusters=dataset_size // 10, items_per_cluster=10, worst_case_blocking=False
-    )
-    
-    detector = DuplicateDetector(
-        similarity_threshold=0.8,
-        similarity={
-            "candidate_strategy": "blocking_v2", 
-            "max_candidates_per_entity": 50,
-            "prefilter_enabled": True,
-            "score_breakdown_enabled": True,
-            "prefilter_thresholds": {
-                "min_length_ratio": 0.4,
-                "require_shared_token": True
-            }
-        }
-    )
-
-    benchmark.pedantic(lambda: detector.detect_duplicates(data), iterations=1, rounds=5)
-
-
-@pytest.mark.parametrize("dataset_size", [100, 500])
-def test_duplicate_detection_worst_Case(benchmark, dataset_size):
-    """
-    Tests detection on a 'Clustered' dataset (Worst Case).
-    Now utilizing V2 Candidate Generation to cut the pair explosion.
-    """
-    data = generate_dataset(
-        num_clusters=dataset_size // 10, items_per_cluster=10, worst_case_blocking=True
-    )
-    
-    detector = DuplicateDetector(
-        similarity_threshold=0.8,
-        similarity={
-            "candidate_strategy": "blocking_v2", 
-            "max_candidates_per_entity": 50,
-            "prefilter_enabled": True,
-            "score_breakdown_enabled": True,
-            "prefilter_thresholds": {
-                "min_length_ratio": 0.4,
-                "require_shared_token": True
-            }
-        }
-    )
-
-    benchmark.pedantic(lambda: detector.detect_duplicates(data), iterations=1, rounds=5)
-
-
-def test_incremental_detection_speed(benchmark):
-    """
-    Measures performance of adding new data to existing index.
-    """
-
-    existing = generate_dataset(num_clusters=50, items_per_cluster=5)
-    new_data = generate_dataset(num_clusters=5, items_per_cluster=2)
-
-    detector = DuplicateDetector()
-
-    benchmark.pedantic(
-        lambda: detector.incremental_detect(new_data, existing), iterations=5, rounds=10
-    )
-
-
-@pytest.mark.parametrize("algo", ["graph", "hierarchical"])
-def test_clustering_strategy_performance(benchmark, algo):
-    """
-    Comapres Union-Fund (Graph) vs Hierarchical Clustering.
-    """
-
-    data = generate_dataset(num_clusters=20, items_per_cluster=10)
-
-    use_hierarchical = algo == "hierarchical"
-    builder = ClusterBuilder(use_hierarchical=use_hierarchical)
-
-    benchmark.pedantic(lambda: builder.build_clusters(data), iterations=1, rounds=5)
-
-
-def test_merge_entity_benchmark(benchmark):
-    """
-    Measures the cost of fusing entities / res conflicts.
-    """
-
-    group = generate_entity_cluster("MegaCorp", 50)
-    merger = EntityMerger()
-
-    benchmark.pedantic(
-        lambda: merger.merge_entity_group(group, strategy="keep_most_complete"),
-        iterations=10,
-        rounds=10,
-    )
-    
-
-@pytest.mark.parametrize("mode", ["legacy", "semantic_v2"])
-def test_relationship_dedup_speed(benchmark, mode):
-    """
-    Measures the speed of relationship/triplet deduplication.
-    Compares the O(N^2) legacy fallback vs the fast canonical hash path.
-    """
-    # Yields ~280 relationships (approx 39,000 comparisons in O(N^2))
-    relationships = generate_relationship_dataset(200) 
-    
-    detector = DuplicateDetector()
-    options = {
-        "threshold": 0.85,
-        "relationship_dedup_mode": mode,
-        "predicate_synonym_map": {
-            "works_for": "employed_by",
-            "is_employee_of": "employed_by",
-            "has_employer": "employed_by"
-        },
-        "literal_normalization_enabled": True
-    }
-
-    benchmark.pedantic(
-        lambda: detector.detect_relationship_duplicates(relationships, **options),
-        iterations=5,
-        rounds=10,
-    )

benchmarks/requirements.txt

@@ -1,43 +0,0 @@
-# Benchmark Tools
-
-pytest>=7.0.0
-pytest-benchmark>=5.2.3
-
-# Core Utils
-
-pydantic
-loguru
-chardet
-requests
-greenlet
-typing-extensions
-tqdm
-click
-rich
-
-numpy
-pandas
-networkx
-scikit-learn
-
-# Graph & Storage
-
-sqlalchemy
-rdflib
-neo4j
-redis
-
-# AI proc
-
-torch
-transformers
-sentence-transformers
-spacy
-beautifulsoup4
-lxml
-pypdf2
-python-docx
-openpyxl
-pillow
-feedparser
-GitPython

benchmarks/storage/test_embeddings.py

@@ -1,180 +0,0 @@
-from typing import Generator, List
-from unittest.mock import MagicMock, patch
-
-import numpy as np
-import pytest
-
-from semantica.embeddings.embedding_generator import EmbeddingGenerator
-from semantica.embeddings.graph_embedding_manager import GraphEmbeddingManager
-from semantica.embeddings.pooling_strategies import PoolingStrategyFactory
-from semantica.embeddings.text_embedder import TextEmbedder
-
-
-# Infra Mocks
-@pytest.fixture(autouse=True)
-def kill_io_overhead():
-    """Silences logging and tracker globally."""
-    with patch("semantica.utils.logging.get_logger"), patch(
-        "semantica.utils.progress_tracker.get_progress_tracker"
-    ) as mock_tracker:
-
-        tracker = MagicMock()
-        tracker.enabled = False
-        tracker._start_tracking.return_value = "dummy_id"
-        mock_tracker.return_value = tracker
-
-        with patch(
-            "semantica.embeddings.text_embedder.get_progress_tracker",
-            return_value=tracker,
-        ):
-            yield
-
-
-# __ Model Mocks __
-
-
-class MockSentenceTransformer:
-    """
-    Simulates ST.encode without loading the fat model itself.
-    """
-
-    def __init__(self, dim=384):
-        self.dim = dim
-
-    def encode(
-        self, sentences: List[str], normalize_embeddings=True, **kwargs
-    ) -> np.ndarray:
-        count = len(sentences)
-        return np.random.rand(count, self.dim).astype(np.float32)
-
-    def get_sentence_embedding_dimension(self):
-        return self.dim
-
-
-class MockFastEmbed:
-    """
-    Simulates FastEmbed.embed generator behavior.
-    """
-
-    def __init__(self, dim=384):
-        self.dim = dim
-
-    def embed(self, documents: List[str]) -> Generator[np.ndarray, None, None]:
-        for _ in documents:
-            yield np.random.rand(self.dim).astype(np.float32)
-
-
-# ~~ Fixtures ~~
-@pytest.fixture
-def text_embedder_st():
-    """
-    Text embedder configured with SentenceTransformer
-    """
-    embedder = TextEmbedder(method="sentence_transformers", model_name="mock-bert")
-    embedder.model = MockSentenceTransformer()
-    embedder.progress_tracker = MagicMock()
-    embedder.progress_tracker.enabled = False
-
-    return embedder
-
-
-@pytest.fixture
-def text_embedder_fast():
-    """
-    Text Embedder cofnigures with Mock FastEmbed.
-    """
-
-    embedder = TextEmbedder(method="fastembed", model_name="mock-bge")
-    embedder.fastembed_model = MockFastEmbed()
-    embedder.progress_tracker = MagicMock()
-    embedder.progress_tracker.enabled = False
-    return embedder
-
-
-# ~~ Benchmarks
-
-
-@pytest.mark.parametrize("strategy", ["mean", "max", "cls", "attention"])
-def test_pooling_math_speed(benchmark, strategy):
-    """
-    Measures the raw NumPy speed of pooling strategies.
-    Scenario: Pooling a batch of 128 token embeddings.
-    """
-
-    embeddings = np.random.rand(128, 768).astype(np.float32)
-    pooler = PoolingStrategyFactory.create(strategy)
-
-    benchmark.pedantic(lambda: pooler.pool(embeddings), iterations=1000, rounds=100)
-
-
-def test_hierarchical_pooling_overhead(benchmark):
-    """
-    Measures the overhead of two-step hierarchical pooling.
-    """
-
-    embeddings = np.random.rand(1000, 768).astype(np.float32)
-    pooler = PoolingStrategyFactory.create("hierarchical", chunk_size=100)
-
-    benchmark.pedantic(lambda: pooler.pool(embeddings), iterations=500, rounds=50)
-
-
-def test_st_wrapper_overhead(benchmark, text_embedder_st):
-    """
-    Measures overhead of TextEmbedder wrapper around SentenceTransformers.
-    """
-
-    text = "This is a whatever we are doing here since idk"
-
-    benchmark.pedantic(
-        lambda: text_embedder_st.embed_text(text), iterations=1000, rounds=20
-    )
-
-
-def test_fastembed_generator_consumption(benchmark, text_embedder_fast):
-    """
-    Measures the cost of consuming the FastEmbed generator
-    and converting to Array.
-    """
-    texts = [f"Sentence {i}" for i in range(20)]
-
-    benchmark.pedantic(
-        lambda: text_embedder_fast.embed_batch(texts), iterations=100, rounds=20
-    )
-
-
-@pytest.mark.parametrize("batch_size", [10, 100, 1000])
-def test_batch_processing_pipeline(benchmark, batch_size, text_embedder_st):
-    """
-    Measures the full EmbeddingGenerator pipeline:
-    Input validation -> Type detection -> Batching -> Mock Model -> Error handling.
-    """
-
-    generator = EmbeddingGenerator()
-
-    generator.text_embedder = text_embedder_st
-    generator.progress_tracker = MagicMock()
-    generator.progress_tracker.enabled = False
-
-    data = [f"Item {i}" for i in range(batch_size)]
-
-    benchmark.pedantic(lambda: generator.process_batch(data), iterations=5, rounds=10)
-
-
-@pytest.mark.parametrize("count", [100, 1000])
-def test_graph_embedding_prep(benchmark, count, text_embedder_st):
-    """
-    Measures how fast we can reshape dict for GraphDBs
-    """
-    manager = GraphEmbeddingManager()
-    manager.embedding_generator.text_embedder = text_embedder_st
-
-    manager.embedding_generator.generate_embeddings = MagicMock(
-        return_value=np.random.rand(count, 384).astype(np.float32)
-    )
-
-    entities = [{"id": f"e{i}", "text": f"Entity{i}"} for i in range(count)]
-
-    def op():
-        return manager.prepare_for_graph_db(entities, backend="neo4j")
-
-    benchmark.pedantic(op, iterations=10, rounds=10)

benchmarks/storage/test_graph_store.py

@@ -1,137 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.graph_store.graph_store import GraphStore
-
-
-@pytest.fixture
-def mock_neo4j_driver():
-    """
-    Creates a mock of of Neo4j Driver
-    Simulates: Driver -> Session -> Transaction -> Result -> Record
-    """
-
-    mock_result = MagicMock()
-    fake_props = {"name": "TestNode", "age": 30}
-
-    def get_item(key):
-        if key == "id":
-            return 12345
-        if key == "n":
-            return fake_props
-        if key == "count":
-            return 42
-        return None
-
-    mock_record = MagicMock()
-    mock_record.__getitem__.side_effect = get_item
-    mock_record.keys.return_value = ["id", "n"]
-    mock_record.values.return_value = [12345, fake_props]
-
-    # dict conversion - essentially doing it because the db sometimes demands it
-    mock_record.items.return_value = [("id", 12345), ("n", fake_props)]
-
-    # ~~ Result Methods ~~
-    mock_result = MagicMock()
-    mock_result.single.return_value = mock_record
-    mock_result.__iter__.side_effect = lambda: iter([mock_record])
-
-    # ~~ Session ~~
-    mock_session = MagicMock()
-    mock_session.run.return_value = mock_result
-    mock_session.__enter__.return_value = mock_session
-    mock_session.__exit__.return_value = None
-
-    # ~~ Driver ~~
-    mock_driver = MagicMock()
-    mock_driver.session.return_value = mock_session
-    mock_driver.verify_connectivity.return_value = True
-
-    return mock_driver
-
-
-@pytest.fixture
-def graph_store(mock_neo4j_driver):
-    """
-    Returns a GraphsStore connected to mnock driver.
-    """
-
-    # ~~ Patch GraphDatbase ~~
-    with patch("semantica.graph_store.neo4j_store.GraphDatabase") as mockDB:
-        mockDB.driver.return_value = mock_neo4j_driver
-        store = GraphStore(
-            backend="neo4j", uri="bolt://mock:7687", user="mock", password="mock"
-        )
-        store.connect()
-
-        if hasattr(store, "progress_tracker"):
-            store.progress_tracker = MagicMock()
-
-        return store
-
-
-#    ~~ Benchmarks ~~
-
-
-def test_node_creation_overhead(benchmark, graph_store):
-    """
-    Benchamrks the full stack overhead for creating a single node.
-    Path: GraphStore -> NodeManager -> Neo4jStore, Driver
-    """
-
-    def op():
-        return graph_store.create_node(
-            labels=["Person"], properties={"name": "Alexander", "age": 17}
-        )
-
-    result = benchmark(op)
-    assert result["id"] == 12345
-
-
-def test_batch_node_creation_overhead(benchmark, graph_store):
-    """
-    Benchmarks the loop overhead in create_nodes (Batch).
-    Checks if it handles lists efficiently.
-    """
-
-    nodes = [{"labels": ["Person"], "properties": {"id": i}} for i in range(50)]
-
-    def op():
-        return graph_store.create_nodes(nodes)
-
-    result = benchmark(op)
-    assert len(result) == 50
-
-
-def test_query_construction_and_parsing(benchmark, graph_store):
-    """
-    Benchmarks every execution overhead.
-    Measures how fast `QueryEngine` parses result into a Python dict.
-    """
-
-    query = "MATCH ( n:Person) RETURN n LIMIT 1"
-
-    def op():
-        return graph_store.execute_query(query)
-
-    result = benchmark(op)
-    assert result["success"] is True
-    assert len(result["records"]) > 0
-
-
-def test_analytics_shortest_path_overhead(benchmark, graph_store):
-    """
-    Benchmarks the wrapper overhead for graph analytics.
-    """
-
-    def op():
-        return graph_store.shortest_path(
-            start_node_id=1, end_node_id=2, rel_type="KNOWS"
-        )
-
-    try:
-        benchmark(op)
-    except Exception:
-        # v pass as we are only trying to benchmark the function overhead call mainly
-        pass

benchmarks/storage/test_triplet_storage.py

@@ -1,146 +0,0 @@
-import time
-from dataclasses import dataclass
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from semantica.triplet_store.bulk_loader import BulkLoader
-from semantica.triplet_store.jena_store import JenaStore
-from semantica.triplet_store.triplet_store import TripletStore
-
-# ~~ Mocking ~~
-# We basically define a facile Triplet class for creating ds devoid of fat AI models
-
-
-@dataclass
-class SimpleTriplet:
-    subject: str
-    predicate: str
-    object: str
-    confidence: float = 1.0
-
-
-# ~~ Fixtures ~~
-@pytest.fixture
-def triplet_batch():
-    """Generates 1000 triplets."""
-    return [
-        SimpleTriplet(
-            subject=f"http://gandhara.org/entity/{i}",
-            predicate="http://gandhara.org/relation/knows",
-            object=f"http://example.org/entity/{i+1}",
-        )
-        for i in range(1000)
-    ]
-
-
-@pytest.fixture
-def large_knowledge_graph_dict():
-    """
-    Generates a large dict (1000 ent) to test parsing
-    logic in `TripletStore.store()`
-    """
-    entities = [
-        {
-            "id": f"ent_{i}",
-            "type": "Person",
-            "properties": {"name": f"Person {i}", "age": 60},
-        }
-        for i in range(1000)
-    ]
-    relationships = [
-        {"source": f"ent_{i}", "target": f"ent_{i+1}", "type": "KNOWS"}
-        for i in range(999)
-    ]
-
-    return {"entities": entities, "relationships": relationships}
-
-
-@pytest.fixture
-def in_memory_store():
-    """Returns a real JenaStore using RDFLib (In-Mmeory)."""
-
-    store = JenaStore(endpoint=None)
-    if store.graph is None:
-        pytest.fail("JenaStore failed to initialize rdflib graph.")
-    if hasattr(store, "progress_tracker"):
-        store.progress_tracker = MagicMock()
-
-    return store
-
-
-# ~~ Benchmarks ~~
-
-
-def test_rdflib_insert_throughput(benchmark, in_memory_store, triplet_batch):
-    """
-    Benchmarks raw Write Speed to in-memory RDF graph.
-    Is our baseline
-    """
-
-    def op():
-        in_memory_store.add_triplets(triplet_batch)
-
-    benchmark(op)
-
-    assert len(in_memory_store.graph) >= 1000
-
-
-def test_triplet_conversion_overhead(benchmark, large_knowledge_graph_dict):
-    """
-    Benchmarks the `store()` method in TripletStore.
-    This tests Python logic that converts a Dict -> Triplet objects.
-    """
-
-    with patch("semantica.triplet_store.blazegraph_store.BlazegraphStore") as mockBE:
-        mock_instance = mockBE.return_value
-        mock_instance.add_triplets.return_value = {"success": True}
-
-        manager = TripletStore(backend="blazegraph")
-        if hasattr(manager, "progress_tracker"):
-            manager.progress_tracker = MagicMock()
-
-        def op():
-            manager.store(
-                knowledge_graph=large_knowledge_graph_dict,
-                ontology={"classes": [], "properties": []},
-            )
-
-        benchmark(op)
-
-
-def test_bulk_loader_logic(benchmark, triplet_batch):
-    """
-    Benchmarks teh BulkLoader class.
-    Measures the overhead of batching, retries and progress tracking.
-    """
-
-    loader = BulkLoader(batch_size=100)
-    if hasattr(loader, "progress_tracker"):
-        loader.progress_tracker = MagicMock()
-
-    mock_store = MagicMock()
-    mock_store.add_triplets.return_value = {"success": True}
-
-    def op():
-        return loader.load_triplets(triplet_batch, mock_store)
-
-    result = benchmark(op)
-    assert result.total_batches == 10
-
-
-def test_sparql_query_performance(benchmark, in_memory_store, triplet_batch):
-    """
-    Benchamrks SPARQL query execution speed on 1000 items.
-    """
-
-    in_memory_store.add_triplets(triplet_batch)
-
-    query = "SELECT ?s ?o WHERE { ?s <http://gandhara.org/relation/knows> ?o } LIMIT 50"
-
-    def op():
-        return in_memory_store.execute_sparql(query)
-
-    result = benchmark(op)
-    assert result["success"] is True
-    assert len(result["bindings"]) == 50

benchmarks/storage/test_vector_storage.py

@@ -1,94 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-import numpy as np
-import pytest
-
-from semantica.vector_store.faiss_store import FAISSStore
-from semantica.vector_store.vector_store import VectorStore
-
-# Fixtures
-
-
-@pytest.fixture
-def vector_dim():
-    return 768
-
-
-@pytest.fixture
-def random_vectors(vector_dim):
-    """Generates a batch of 10,000 rando vectors."""
-    count = 10000
-    vectors = np.random.rand(count, vector_dim).astype(np.float32)
-    return vectors
-
-
-@pytest.fixture
-def populated_store(random_vectors, vector_dim):
-    """
-    Returns a FAISS store bred with data.
-    """
-
-    store = FAISSStore(dimension=vector_dim)
-    if hasattr(store, "progress_tracker"):
-        store.progress_tracker = MagicMock()
-    store.create_index(index_type="flat")
-    store.add_vectors(random_vectors)
-    return store
-
-
-# Benchmarks
-
-
-def test_faiss_insert_throughput(benchmark, random_vectors, vector_dim):
-    """
-    Benchmarks raw Write speed to FAISS
-    """
-    store = FAISSStore(dimension=vector_dim)
-    if hasattr(store, "progress_tracker"):
-        store.progress_tracker = MagicMock()
-    store.create_index(index_type="flat")
-
-    def insert_op():
-        store.add_vectors(random_vectors)
-
-    benchmark(insert_op)
-
-    assert len(store.index.vector_ids) >= 10000
-
-
-def test_faiss_search_latency(benchmark, populated_store, vector_dim):
-    """
-    Benchmarks Read/Search speed
-    """
-
-    query = np.random.rand(1, vector_dim).astype(np.float32)
-    results = benchmark(populated_store.search_similar, query_vector=query, k=10)
-    assert len(results) == 10
-
-
-def test_vector_storage_manager_overhead(benchmark, random_vectors, vector_dim):
-    """
-    Benchmarks the overhead of the VectorStore class
-    """
-    with patch(
-        "semantica.vector_store.vector_store.EmbeddingGenerator"
-    ) as MockEmbedder:
-        manager = VectorStore(backend="faiss", dimension=vector_dim)
-        if hasattr(manager, "progress_tracker"):
-            manager.progress_tracker = MagicMock()
-
-        def store_op():
-            manager.store_vectors(random_vectors)
-
-        benchmark(store_op)
-
-        # Check vectors were stored - handle both in-memory and backend stores
-        if hasattr(manager, 'vectors'):
-            # In-memory backend
-            assert len(manager.vectors) >= 10000
-        elif hasattr(manager, '_backend_store') and hasattr(manager._backend_store, 'vector_ids'):
-            # Backend store (like FAISS)
-            assert len(manager._backend_store.vector_ids) >= 10000
-        else:
-            # For other backends, just ensure no errors occurred
-            pass

benchmarks/visualization/conftest.py

@@ -1,80 +0,0 @@
-import random
-from typing import Any, Dict, List
-from unittest.mock import MagicMock, patch
-
-import numpy as np
-import pytest
-
-
-# Data Generators
-@pytest.fixture
-def generate_embeddings():
-    """Generates synthetic high-dim embeddings."""
-
-    def _gen(n_samples: int, n_features: int = 768):
-        return np.random.rand(n_samples, n_features).astype(np.float32)
-
-    return _gen
-
-
-@pytest.fixture
-def generate_knowledge_graph():
-    """Generates synthetic Knowledge Graph dictionary."""
-
-    def _gen(n_nodes: int, density: float = 0.05):
-        entities = [
-            {
-                "id": f"e_{i}",
-                "label": f"Entity_{i}",
-                "type": random.choice(["Person", "Organization", "Location", "Event"]),
-                "metadata": {"score": random.random()},
-            }
-            for i in range(n_nodes)
-        ]
-
-        relationships = []
-        n_edges = int(n_nodes * (n_nodes - 1) * density)
-        # Capping edges for safety
-        n_edges = min(n_edges, n_nodes * 5)
-
-        for i in range(n_edges):
-            src = random.randint(0, n_nodes - 1)
-            tgt = random.randint(0, n_nodes - 1)
-
-            if src != tgt:
-                relationships.append(
-                    {
-                        "source": f"e_{src}",
-                        "target": f"e_{tgt}",
-                        "type": "related_to",
-                        "metadata": {"weight": random.random()},
-                    }
-                )
-
-        return {"entities": entities, "relationships": relationships}
-
-    return _gen
-
-
-@pytest.fixture
-def generate_temporal_data(generate_knowledge_graph):
-    """Generates synthetic temporal graph snapshots."""
-
-    def _gen(n_snapshots: int, n_nodes: int):
-        timestamps_map = {}
-        base_kg = generate_knowledge_graph(n_nodes)
-        entities = base_kg["entities"]
-
-        all_years = list(range(2020, 2020 + n_snapshots))
-        for ent in entities:
-            start = random.randint(0, len(all_years) - 2)
-            duration = random.randint(1, len(all_years) - start)
-            timestamps_map[ent["id"]] = all_years[start : start + duration]
-
-        return {
-            "entities": entities,
-            "relationships": base_kg["relationships"],
-            "timestamps": timestamps_map,
-        }
-
-    return _gen

benchmarks/visualization/test_analytics_viz.py

@@ -1,26 +0,0 @@
-import random
-
-import pytest
-
-from semantica.visualization.analytics_visualizer import AnalyticsVisualizer
-
-
-@pytest.mark.benchmark(group="analytics_charts")
-def test_centrality_ranking_sort_and_render(benchmark):
-    """
-    Benchmarks sorting a large centrality dictionary
-    and rendering the Top N bar chart.
-    """
-    viz = AnalyticsVisualizer()
-
-    # Generate 5000 node scores
-    centrality_data = {
-        "centrality": {f"node_{i}": random.random() for i in range(5000)}
-    }
-
-    def run():
-        return viz.visualize_centrality_rankings(
-            centrality_data, centrality_type="degree", top_n=50, output="interactive"
-        )
-
-    benchmark.pedantic(run, iterations=1, rounds=10)

benchmarks/visualization/test_embedding_viz.py

@@ -1,45 +0,0 @@
-import numpy as np
-import pytest
-
-from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
-
-
-@pytest.mark.benchmark(group="embedding_projection")
-@pytest.mark.parametrize("method", ["pca", "tsne"])
-@pytest.mark.parametrize("n_samples", [500])
-def test_projection_calculation_overhead(
-    benchmark, generate_embeddings, method, n_samples
-):
-    """
-    Measures the combined cost of:
-    1. Dimensionality Reduction (Math)
-    2. Plotly Trace Construction (Object creation)
-    """
-
-    viz = EmbeddingVisualizer()
-    embeddings = generate_embeddings(n_samples=n_samples, n_features=128)
-    labels = [f"Label {i}" for i in range(n_samples)]
-
-    def run():
-        return viz.visualize_2d_projection(
-            embeddings, labels=labels, method=method, output="interactive"
-        )
-
-    rounds = 5 if method == "tsne" else 10
-    benchmark.pedantic(run, iterations=1, rounds=rounds)
-
-
-@pytest.mark.benchmark(group="embedding_heatmap")
-def test_similarity_heatmap_generation(benchmark, generate_embeddings):
-    """
-    Benchmarks O(N^2) similarity matrix calculation
-    and heatmap renderin.
-    """
-
-    viz = EmbeddingVisualizer()
-    embeddings = generate_embeddings(n_samples=500, n_features=64)
-
-    def run():
-        return viz.visualize_similarity_heatmap(embeddings, output="interactive")
-
-    benchmark.pedantic(run, iterations=1, rounds=5)

benchmarks/visualization/test_kg_viz.py

@@ -1,33 +0,0 @@
-import pytest
-
-from semantica.visualization.kg_visualizer import KGVisualizer
-
-
-@pytest.mark.benchmark(group="graph_layouyt")
-@pytest.mark.parametrize("layout", ["circular", "force"])
-@pytest.mark.parametrize("size", [100])
-def test_network_layout_performance(benchmark, generate_knowledge_graph, layout, size):
-    """
-    Compares layout algorithm.
-    """
-    viz = KGVisualizer(layout=layout, force_layout_iterations=50)
-    graph = generate_knowledge_graph(n_nodes=size)
-
-    def run():
-        return viz.visualize_network(graph, output="interactive")
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-@pytest.mark.benchmark(group="graph_structure")
-def test_matrix_view_rendering(benchmark, generate_knowledge_graph):
-    """
-    Benchmarks the creation of an adjacent/relationship matrix.
-    """
-    viz = KGVisualizer()
-    graph = generate_knowledge_graph(n_nodes=500)
-
-    def run():
-        return viz.visualize_relationship_matrix(graph, output="interactive")
-
-    benchmark.pedantic(run, iterations=1, rounds=5)

benchmarks/visualization/test_temporal_viz.py

@@ -1,39 +0,0 @@
-import pytest
-
-from semantica.visualization.temporal_visualizer import TemporalVisualizer
-
-
-@pytest.mark.benchmark(group="temporal_animation")
-def test_network_evolution_frames(benchmark, generate_temporal_data):
-    """
-    Measures the cost of generating animation frames for Plotly.
-    """
-
-    temporal_data = generate_temporal_data(n_snapshots=5, n_nodes=100)
-    viz = TemporalVisualizer()
-
-    def run():
-        return viz.visualize_network_evolution(temporal_data, output="interactive")
-
-    benchmark.pedantic(run, iterations=1, rounds=5)
-
-
-@pytest.mark.benchmark(group="temporal_dashboard")
-def test_temporal_dashboard_assembly(benchmark, generate_temporal_data):
-    """
-    Benchmarks the creation of a multi-subplot dashboard.
-    """
-    temporal_data = generate_temporal_data(n_snapshots=20, n_nodes=200)
-    viz = TemporalVisualizer()
-
-    metrics = {
-        "Accuracy": [0.5 + i * 0.02 for i in range(20)],
-        "Loss": [1.0 - i * 0.04 for i in range(20)],
-    }
-
-    def run():
-        return viz.visualize_temporal_dashboard(
-            temporal_data, metrics=metrics, output="interactive"
-        )
-
-    benchmark.pedantic(run, iterations=1, rounds=5)

cookbook/advanced/snowflake_ingestion_examples.py

@@ -7,17 +7,26 @@ This module provides comprehensive examples of using the Snowflake ingestor.
 import os
 from datetime import datetime, timedelta
 
+from rich import box
+from rich.console import Console
+from rich.rule import Rule
+from rich.table import Table
+
 from semantica.ingest import SnowflakeIngestor
 from semantica.utils.logging import get_logger
 
 logger = get_logger("snowflake_examples")
+console = Console()
+
+
+def _section(title: str) -> None:
+    console.print(Rule(f"[bold cyan]{title}[/bold cyan]", style="cyan"))
 
 
 def example_basic_ingestion():
     """Example: Basic table ingestion."""
-    print("\n=== Example 1: Basic Table Ingestion ===\n")
+    _section("Example 1: Basic Table Ingestion")
 
-    # Initialize ingestor with password authentication
     ingestor = SnowflakeIngestor(
         account=os.getenv("SNOWFLAKE_ACCOUNT"),
         user=os.getenv("SNOWFLAKE_USER"),
@@ -27,26 +36,23 @@ def example_basic_ingestion():
         schema="PUBLIC",
     )
 
-    # Ingest a table
     data = ingestor.ingest_table("CUSTOMERS", limit=10)
 
-    print(f"Retrieved {data.row_count} rows")
-    print(f"Columns: {data.columns}")
-    print(f"\nFirst row:")
-    print(data.data[0])
+    console.print(f"[green]✓[/green] Retrieved [cyan]{data.row_count}[/cyan] rows")
+    console.print(f"  Columns: [dim]{data.columns}[/dim]")
+    console.print(f"  First row: [dim]{data.data[0]}[/dim]")
 
     ingestor.close()
 
 
 def example_query_execution():
     """Example: Execute custom SQL queries."""
-    print("\n=== Example 2: Query Execution ===\n")
+    _section("Example 2: Query Execution")
 
     ingestor = SnowflakeIngestor()
 
-    # Execute aggregation query
     query = """
-        SELECT 
+        SELECT
             COUNTRY,
             COUNT(*) AS CUSTOMER_COUNT,
             SUM(TOTAL_PURCHASES) AS TOTAL_REVENUE
@@ -58,34 +64,34 @@ def example_query_execution():
 
     data = ingestor.ingest_query(query)
 
-    print(f"Top 10 countries by revenue:")
+    table = Table(title="[bold]Top 10 Countries by Revenue[/bold]",
+                  box=box.SIMPLE_HEAD, show_edge=False, padding=(0, 1))
+    table.add_column("Country", style="cyan", no_wrap=True)
+    table.add_column("Customers", style="green", justify="right")
+    table.add_column("Revenue", style="green", justify="right")
     for row in data.data:
-        print(
-            f"  {row['COUNTRY']}: {row['CUSTOMER_COUNT']} customers, "
-            f"${row['TOTAL_REVENUE']:,.2f} revenue"
+        table.add_row(
+            row["COUNTRY"],
+            str(row["CUSTOMER_COUNT"]),
+            f"${row['TOTAL_REVENUE']:,.2f}",
         )
+    console.print(table)
 
     ingestor.close()
 
 
 def example_parameterized_query():
     """Example: Parameterized queries."""
-    print("\n=== Example 3: Parameterized Queries ===\n")
+    _section("Example 3: Parameterized Queries")
 
     ingestor = SnowflakeIngestor()
 
-    # Calculate date range
     end_date = datetime.now()
     start_date = end_date - timedelta(days=30)
 
-    # Execute parameterized query
     query = """
-        SELECT 
-            ORDER_ID,
-            CUSTOMER_ID,
-            PRODUCT_NAME,
-            AMOUNT,
-            ORDER_DATE
+        SELECT
+            ORDER_ID, CUSTOMER_ID, PRODUCT_NAME, AMOUNT, ORDER_DATE
         FROM ORDERS
         WHERE ORDER_DATE BETWEEN %(start_date)s AND %(end_date)s
           AND AMOUNT > %(min_amount)s
@@ -101,122 +107,125 @@ def example_parameterized_query():
         },
     )
 
-    print(f"Found {data.row_count} orders in the last 30 days over $100")
-
+    console.print(
+        f"[green]✓[/green] Found [cyan]{data.row_count}[/cyan] orders "
+        "in the last 30 days over $100"
+    )
     ingestor.close()
 
 
 def example_schema_introspection():
     """Example: Table schema introspection."""
-    print("\n=== Example 4: Schema Introspection ===\n")
+    _section("Example 4: Schema Introspection")
 
     ingestor = SnowflakeIngestor()
-
-    # Get table schema
     schema = ingestor.get_table_schema("CUSTOMERS")
 
-    print("Table schema for CUSTOMERS:")
-    print(f"Primary keys: {schema['primary_keys']}\n")
+    console.print(f"  Primary keys: [cyan]{schema['primary_keys']}[/cyan]")
 
-    print("Columns:")
+    table = Table(title="[bold]CUSTOMERS Schema[/bold]",
+                  box=box.SIMPLE_HEAD, show_edge=False, padding=(0, 1))
+    table.add_column("Column", style="cyan", no_wrap=True)
+    table.add_column("Type")
+    table.add_column("Nullable")
+    table.add_column("Default", style="dim")
     for col in schema["columns"]:
-        nullable = "NULL" if col["nullable"] else "NOT NULL"
-        default = f" DEFAULT {col['default']}" if col["default"] else ""
-        print(f"  {col['name']}: {col['type']} {nullable}{default}")
+        table.add_row(
+            col["name"],
+            col["type"],
+            "NULL" if col["nullable"] else "NOT NULL",
+            str(col["default"]) if col["default"] else "",
+        )
+    console.print(table)
 
     ingestor.close()
 
 
 def example_list_tables():
     """Example: List all tables in a schema."""
-    print("\n=== Example 5: List Tables ===\n")
+    _section("Example 5: List Tables")
 
     ingestor = SnowflakeIngestor()
-
-    # List tables in current schema
     tables = ingestor.list_tables()
 
-    print(f"Found {len(tables)} tables:")
-    for table in tables:
-        print(f"  - {table}")
+    table = Table(title=f"[bold]Tables ({len(tables)} found)[/bold]",
+                  box=box.SIMPLE_HEAD, show_edge=False, padding=(0, 1))
+    table.add_column("Table", style="cyan")
+    for t in tables:
+        table.add_row(t)
+    console.print(table)
 
     ingestor.close()
 
 
 def example_pagination():
     """Example: Paginate large result sets."""
-    print("\n=== Example 6: Pagination ===\n")
+    _section("Example 6: Pagination")
 
     ingestor = SnowflakeIngestor()
-
     PAGE_SIZE = 100
     total_rows = 0
-
-    # Paginate through large table
     page = 0
+
     while True:
         data = ingestor.ingest_table(
             "LARGE_TABLE", limit=PAGE_SIZE, offset=page * PAGE_SIZE
         )
-
         if data.row_count == 0:
             break
-
         total_rows += data.row_count
-        print(f"Page {page + 1}: {data.row_count} rows")
-
-        # Process page
+        console.print(
+            f"  [dim]Page {page + 1}:[/dim] [cyan]{data.row_count}[/cyan] rows"
+        )
         process_page(data)
-
         page += 1
 
-    print(f"\nTotal rows processed: {total_rows}")
-
+    console.print(
+        f"[green]✓[/green] Total rows processed: [cyan]{total_rows}[/cyan]"
+    )
     ingestor.close()
 
 
 def example_batch_processing():
     """Example: Batch processing with fetchmany."""
-    print("\n=== Example 7: Batch Processing ===\n")
+    _section("Example 7: Batch Processing")
 
     ingestor = SnowflakeIngestor()
-
-    # Execute query with batching
     data = ingestor.ingest_query(
         "SELECT * FROM LARGE_TABLE WHERE STATUS = 'ACTIVE'", batch_size=1000
     )
-
-    print(f"Retrieved {data.row_count} rows in batches of 1000")
-
+    console.print(
+        f"[green]✓[/green] Retrieved [cyan]{data.row_count}[/cyan] rows "
+        "in batches of 1000"
+    )
     ingestor.close()
 
 
 def example_export_documents():
     """Example: Export to Semantica document format."""
-    print("\n=== Example 8: Export as Documents ===\n")
+    _section("Example 8: Export as Documents")
 
     ingestor = SnowflakeIngestor()
-
-    # Ingest product data
     data = ingestor.ingest_table("PRODUCTS", limit=10)
-
-    # Convert to documents
     documents = ingestor.export_as_documents(
         data, id_field="PRODUCT_ID", text_fields=["PRODUCT_NAME", "DESCRIPTION"]
     )
 
-    print(f"Exported {len(documents)} documents")
-    print("\nFirst document:")
-    print(f"  ID: {documents[0]['id']}")
-    print(f"  Text: {documents[0]['text'][:100]}...")
-    print(f"  Metadata: {documents[0]['metadata']}")
+    console.print(
+        f"[green]✓[/green] Exported [cyan]{len(documents)}[/cyan] documents"
+    )
+    if documents:
+        d = documents[0]
+        console.print(f"  [dim]First doc — ID:[/dim] {d['id']}")
+        console.print(f"  [dim]Text:[/dim] {d['text'][:100]}…")
+        console.print(f"  [dim]Metadata:[/dim] {d['metadata']}")
 
     ingestor.close()
 
 
 def example_key_pair_auth():
     """Example: Key-pair authentication."""
-    print("\n=== Example 9: Key-Pair Authentication ===\n")
+    _section("Example 9: Key-Pair Authentication")
 
     ingestor = SnowflakeIngestor(
         account=os.getenv("SNOWFLAKE_ACCOUNT"),
@@ -224,80 +233,66 @@ def example_key_pair_auth():
         private_key_path=os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH"),
         warehouse="COMPUTE_WH",
     )
-
     data = ingestor.ingest_table("CUSTOMERS", limit=5)
-    print(f"Successfully authenticated and retrieved {data.row_count} rows")
-
+    console.print(
+        f"[green]✓[/green] Authenticated — retrieved [cyan]{data.row_count}[/cyan] rows"
+    )
     ingestor.close()
 
 
 def example_context_manager():
     """Example: Using context manager."""
-    print("\n=== Example 10: Context Manager ===\n")
+    _section("Example 10: Context Manager")
 
     with SnowflakeIngestor() as ingestor:
         data = ingestor.ingest_table("CUSTOMERS", limit=5)
-        print(f"Retrieved {data.row_count} rows")
-
-    # Connection automatically closed
-    print("Connection closed automatically")
+        console.print(
+            f"[green]✓[/green] Retrieved [cyan]{data.row_count}[/cyan] rows"
+        )
+    console.print("[dim]  Connection closed automatically.[/dim]")
 
 
 def example_multi_schema():
     """Example: Multi-schema ingestion."""
-    print("\n=== Example 11: Multi-Schema Ingestion ===\n")
+    _section("Example 11: Multi-Schema Ingestion")
 
     ingestor = SnowflakeIngestor()
+    prod = ingestor.ingest_table("CUSTOMERS", database="PROD_DB", schema="PUBLIC", limit=10)
+    staging = ingestor.ingest_table("CUSTOMERS", database="STAGING_DB", schema="PUBLIC", limit=10)
 
-    # Ingest from different schemas
-    prod_customers = ingestor.ingest_table(
-        "CUSTOMERS", database="PROD_DB", schema="PUBLIC", limit=10
-    )
-
-    staging_customers = ingestor.ingest_table(
-        "CUSTOMERS", database="STAGING_DB", schema="PUBLIC", limit=10
-    )
-
-    print(f"Production customers: {prod_customers.row_count}")
-    print(f"Staging customers: {staging_customers.row_count}")
+    console.print(f"  Production: [cyan]{prod.row_count}[/cyan] customers")
+    console.print(f"  Staging:    [cyan]{staging.row_count}[/cyan] customers")
 
     ingestor.close()
 
 
 def example_error_handling():
     """Example: Error handling."""
-    print("\n=== Example 12: Error Handling ===\n")
+    _section("Example 12: Error Handling")
 
     from semantica.utils.exceptions import ProcessingError, ValidationError
 
     try:
-        # Try to connect with invalid credentials
         ingestor = SnowflakeIngestor(
             account="invalid_account", user="invalid_user", password="invalid_password"
         )
-
-        data = ingestor.ingest_table("CUSTOMERS")
+        ingestor.ingest_table("CUSTOMERS")
 
     except ValidationError as e:
-        print(f"Validation error: {e}")
-
+        console.print(f"[bold yellow] ⚠[/bold yellow] Validation error: {e}")
     except ProcessingError as e:
-        print(f"Processing error: {e}")
-
+        console.print(f"[bold red] ✗[/bold red] Processing error: {e}")
     except Exception as e:
-        print(f"Unexpected error: {e}")
+        console.print(f"[bold red] ✗[/bold red] Unexpected error: {e}")
 
 
 def example_incremental_load():
     """Example: Incremental data loading."""
-    print("\n=== Example 13: Incremental Loading ===\n")
+    _section("Example 13: Incremental Loading")
 
     ingestor = SnowflakeIngestor()
+    last_load = get_last_load_timestamp()
 
-    # Get last load timestamp (from your metadata store)
-    last_load = get_last_load_timestamp()  # Your function
-
-    # Query only new/updated records
     query = """
         SELECT *
         FROM CUSTOMERS
@@ -306,10 +301,10 @@ def example_incremental_load():
     """
 
     data = ingestor.ingest_query(query, params={"last_load": last_load})
-
-    print(f"Loaded {data.row_count} new/updated records since {last_load}")
-
-    # Update last load timestamp
+    console.print(
+        f"[green]✓[/green] Loaded [cyan]{data.row_count}[/cyan] new/updated "
+        f"records since [dim]{last_load}[/dim]"
+    )
     if data.row_count > 0:
         update_last_load_timestamp(datetime.now())
 
@@ -318,20 +313,14 @@ def example_incremental_load():
 
 def example_etl_pipeline():
     """Example: Full ETL pipeline."""
-    print("\n=== Example 14: ETL Pipeline ===\n")
+    _section("Example 14: ETL Pipeline")
 
-    # Extract
     ingestor = SnowflakeIngestor()
 
     sales_query = """
-        SELECT 
-            s.ORDER_ID,
-            s.CUSTOMER_ID,
-            c.CUSTOMER_NAME,
-            s.PRODUCT_ID,
-            p.PRODUCT_NAME,
-            s.AMOUNT,
-            s.ORDER_DATE
+        SELECT
+            s.ORDER_ID, s.CUSTOMER_ID, c.CUSTOMER_NAME,
+            s.PRODUCT_ID, p.PRODUCT_NAME, s.AMOUNT, s.ORDER_DATE
         FROM SALES s
         JOIN CUSTOMERS c ON s.CUSTOMER_ID = c.ID
         JOIN PRODUCTS p ON s.PRODUCT_ID = p.ID
@@ -339,43 +328,33 @@ def example_etl_pipeline():
     """
 
     data = ingestor.ingest_query(sales_query)
-    print(f"Extracted {data.row_count} sales records")
+    console.print(f"  [dim]Extract:[/dim] [cyan]{data.row_count}[/cyan] sales records")
 
-    # Transform
     documents = ingestor.export_as_documents(
         data, id_field="ORDER_ID", text_fields=["CUSTOMER_NAME", "PRODUCT_NAME"]
     )
-    print(f"Transformed to {len(documents)} documents")
+    console.print(f"  [dim]Transform:[/dim] [cyan]{len(documents)}[/cyan] documents")
 
-    # Load (into Semantica)
     from semantica.pipeline import Pipeline
-
     pipeline = Pipeline()
-
     for doc in documents:
         pipeline.process_document(doc)
 
-    print("Loaded documents into Semantica pipeline")
-
+    console.print("[green]✓[/green] Loaded documents into Semantica pipeline")
     ingestor.close()
 
 
-# Utility functions for examples
+# ─── Utility stubs ────────────────────────────────────────────────────────────
+
 def process_page(data):
-    """Process a page of data."""
-    # Your processing logic here
     pass
 
 
 def get_last_load_timestamp():
-    """Get the last load timestamp from metadata store."""
-    # Your implementation here
     return (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S")
 
 
 def update_last_load_timestamp(timestamp):
-    """Update the last load timestamp in metadata store."""
-    # Your implementation here
     pass
 
 
@@ -395,17 +374,10 @@ def main():
     for example_func in examples:
         try:
             example_func()
+            console.print()
         except Exception as e:
-            logger.error(f"Example {example_func.__name__} failed: {e}")
+            logger.error("Example %s failed: %s", example_func.__name__, e)
 
 
 if __name__ == "__main__":
-    # Set up environment variables
-    # export SNOWFLAKE_ACCOUNT=your_account
-    # export SNOWFLAKE_USER=your_user
-    # export SNOWFLAKE_PASSWORD=your_password
-    # export SNOWFLAKE_WAREHOUSE=COMPUTE_WH
-    # export SNOWFLAKE_DATABASE=SAMPLE_DB
-    # export SNOWFLAKE_SCHEMA=PUBLIC
-
     main()

cookbook/introduction/neptune-setup.yaml

@@ -138,6 +138,8 @@ Resources:
       IamAuthEnabled: true
       StorageEncrypted: true
       DeletionProtection: false
+      EnableCloudwatchLogsExports:
+        - audit
       Tags:
         - Key: Name
           Value: !Sub ${EnvironmentName}-cluster

cookbook/use_cases/advanced_rag/data/sample_graphrag_paper.txt

@@ -1,13 +0,0 @@
-Graph Retrieval-Augmented Generation (GraphRAG): A New Era for Intelligent Search
-
-GraphRAG is an advanced technique that combines the retrieval capabilities of vector databases with the structural reasoning of knowledge graphs. Unlike traditional RAG, which relies solely on vector similarity, GraphRAG leverages the relationships between entities to provide more contextually accurate and comprehensive answers.
-
-Key Components:
-1. Knowledge Graph: A structured representation of data where nodes represent entities and edges represent relationships.
-2. Vector Search: Finds semantically similar text chunks.
-3. Graph Traversal: Navigates the knowledge graph to find related entities that might not be semantically similar but are structurally relevant.
-
-Benefits:
-- Improved Context: By following relationships, the system can understand the broader context of a query.
-- Multi-hop Reasoning: Can answer complex questions that require connecting multiple pieces of information.
-- Reduced Hallucinations: Grounding answers in a verified knowledge structure reduces the likelihood of generating false information.

cookbook/use_cases/advanced_rag/expert_guide.txt

@@ -1,5 +0,0 @@
-
-RETINOL CLINICAL GUIDE
-Mechanism: Binds to retinoic acid receptors to increase cellular turnover.
-Precautions: Should not be used with high-concentration AHA/BHA exfoliants.
-Synergy: Highly effective when paired with Niacinamide to offset potential erythema.

cookbook/use_cases/advanced_rag/expert_skincare_guide.txt

@@ -1,6 +0,0 @@
-
-RETINOL CLINICAL GUIDE v2.1
-Mechanism: Binds to retinoic acid receptors (RAR) to increase cellular turnover.
-Precautions: Should not be used with high-concentration AHA/BHA exfoliants.
-Synergy: Highly effective when paired with Niacinamide to offset potential erythema.
-Target: Stratum corneum thickening and dermal collagen synthesis.

cookbook/use_cases/advanced_rag/graphrag_kg.graphml

@@ -1,254 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<graphml xmlns="http://graphml.graphdrawing.org/xmlns"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns
-         http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
-
-  <key id="type" for="node" attr.name="type" attr.type="string"/>
-  <key id="confidence" for="node" attr.name="confidence" attr.type="double"/>
-
-  <graph id="G" edgedefault="directed">
-
-    <node id="makeup_and_beauty_blog">
-      <data key="label">Makeup and Beauty Blog</data>
-      <data key="type">ORG</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="monday_poll">
-      <data key="label">Monday Poll</data>
-      <data key="type">EVENT</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="2007">
-      <data key="label">2007</data>
-      <data key="type">DATE</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="rosacea">
-      <data key="label">Rosacea</data>
-      <data key="type">CONCEPT</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="dr._bailey">
-      <data key="label">Dr. Bailey</data>
-      <data key="type">PERSON</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="green_tea_antioxidant_skin_therapy">
-      <data key="label">Green Tea Antioxidant Skin Therapy</data>
-      <data key="type">PRODUCT</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="vol._892">
-      <data key="label">Vol. 892</data>
-      <data key="type">EVENT</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="laneige">
-      <data key="label">Laneige</data>
-      <data key="type">ORG</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="sausalito">
-      <data key="label">Sausalito</data>
-      <data key="type">GPE</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="ulta">
-      <data key="label">Ulta</data>
-      <data key="type">ORG</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="december_15,_2025">
-      <data key="label">December 15, 2025</data>
-      <data key="type">DATE</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="jo_malone">
-      <data key="label">Jo Malone</data>
-      <data key="type">ORG</data>
-      <data key="confidence">1</data>
-    </node>
-    <node id="trader_joe">
-      <data key="label">Trader Joe</data>
-      <data key="type">ORG</data>
-      <data key="confidence">1</data>
-    </node>
-    <node id="hawaii">
-      <data key="label">hawaii</data>
-      <data key="type">GPE</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="benzoyl_peroxide_cream">
-      <data key="label">Benzoyl Peroxide Cream</data>
-      <data key="type">PRODUCT</data>
-      <data key="confidence">1</data>
-    </node>
-    <node id="facial_dandruff">
-      <data key="label">Facial dandruff</data>
-      <data key="type">CONCEPT</data>
-      <data key="confidence">1</data>
-    </node>
-    <node id="calming_zinc_soap">
-      <data key="label">Calming Zinc Soap</data>
-      <data key="type">PRODUCT</data>
-      <data key="confidence">1</data>
-    </node>
-    <node id="hydrate">
-      <data key="label">Hydrate</data>
-      <data key="type">CONCEPT</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="daily_moisturizing_face_cream">
-      <data key="label">Daily Moisturizing Face Cream</data>
-      <data key="type">PRODUCT</data>
-      <data key="confidence">1.0</data>
-    </node>
-    <node id="omega_enriched_face_booster_oil">
-      <data key="label">Omega Enriched Face Booster Oil</data>
-      <data key="type">PRODUCT</data>
-      <data key="confidence">1.0</data>
-    </node>
-
-    <edge source="Makeup and Beauty Blog" target="Monday Poll">
-      <data key="label">hosts</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Monday Poll" target="December 15, 2025">
-      <data key="label">occurs on</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Makeup and Beauty Blog Monday Poll, Vol. 893">
-      <data key="label">publishes</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Monday">
-      <data key="label">has</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="2007">
-      <data key="label">has</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Monday Poll">
-      <data key="label">hosts</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Makeup and Beauty Blog Monday Poll">
-      <data key="label">posts</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Vol. 892">
-      <data key="label">posts</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="2007">
-      <data key="label">has been active since</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="MBB" target="Makeup and Beauty Blog">
-      <data key="label">related_to</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Makeup and Beauty Blog">
-      <data key="label">related_to</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Monday Poll">
-      <data key="label">hosts</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Vol. 891">
-      <data key="label">posts</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Makeup and Beauty Blog" target="Monday Poll">
-      <data key="label">posts</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Cavallo Point" target="Sausalito">
-      <data key="label">located_in</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Dr. Bailey" target="Green Tea Antioxidant Skin Therapy">
-      <data key="label">prescribes</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Green Tea Antioxidant Skin Therapy" target="Rosacea Therapy Skin Care Kit">
-      <data key="label">part of</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Dr. Bailey" target="Rosacea Therapy Skin Care Kit">
-      <data key="label">uses</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Rosacea Therapy Skin Care Kit" target="rosacea treatment routine">
-      <data key="label">part of</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Dr. Bailey" target="rosacea treatment routine">
-      <data key="label">uses</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Facial dandruff" target="rosacea">
-      <data key="label">often occurs with</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Facial dandruff" target="rosacea">
-      <data key="label">needs to be addressed</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Calming Zinc Soap" target="Facial dandruff">
-      <data key="label">is often sufficient to control</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Calming Zinc Soap" target="rosacea">
-      <data key="label">is often sufficient to control</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Green Tea Antioxidant Skin Therapy" target="Facial dandruff">
-      <data key="label">is often sufficient to control</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Green Tea Antioxidant Skin Therapy" target="rosacea">
-      <data key="label">is often sufficient to control</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Dr. Bailey's Skincare" target="Calming Zinc Soap">
-      <data key="label">produces</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Dr. Bailey's Skincare" target="Green Tea Antioxidant Skin Therapy">
-      <data key="label">produces</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Dr. Bailey" target="Calming Zinc Soap">
-      <data key="label">prescribes</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Dr. Bailey" target="Green Tea Antioxidant Skin Therapy">
-      <data key="label">prescribes</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Hydrate" target="Daily Moisturizing Face Cream">
-      <data key="label">is_achieved_by</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Daily Moisturizing Face Cream" target="Omega Enriched Face Booster Oil">
-      <data key="label">can_be_combined_with</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Omega Enriched Face Booster Oil" target="castor seed oil">
-      <data key="label">contains</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Omega Enriched Face Booster Oil" target="sea buckthorn">
-      <data key="label">contains</data>
-      <data key="confidence">0.9</data>
-    </edge>
-    <edge source="Daily Moisturizing Face Cream" target="Omega Enriched Face Booster Oil">
-      <data key="label">can_be_replaced_with</data>
-      <data key="confidence">0.9</data>
-    </edge>
-  </graph>
-</graphml>

cookbook/use_cases/advanced_rag/graphrag_kg.json

@@ -1,678 +0,0 @@
-{
-  "nodes": [
-    {
-      "id": "makeup_and_beauty_blog",
-      "label": "Makeup and Beauty Blog",
-      "type": "ORG",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "makeup_and_beauty_blog",
-              "name": "Makeup and Beauty Blog",
-              "source": null
-            },
-            {
-              "id": "makeup_and_beauty_blog",
-              "name": "Makeup and Beauty Blog",
-              "source": null
-            },
-            {
-              "id": "makeup_and_beauty_blog_monday_poll,_vol._893",
-              "name": "Makeup and Beauty Blog Monday Poll, Vol. 893",
-              "source": null
-            },
-            {
-              "id": "makeup_and_beauty_blog_monday_poll",
-              "name": "Makeup and Beauty Blog Monday Poll",
-              "source": null
-            },
-            {
-              "id": "mbb",
-              "name": "MBB",
-              "source": null
-            }
-          ],
-          "merge_count": 5
-        }
-      }
-    },
-    {
-      "id": "monday_poll",
-      "label": "Monday Poll",
-      "type": "EVENT",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "monday_poll",
-              "name": "Monday Poll",
-              "source": null
-            },
-            {
-              "id": "monday_poll",
-              "name": "Monday Poll",
-              "source": null
-            },
-            {
-              "id": "monday",
-              "name": "Monday",
-              "source": null
-            },
-            {
-              "id": "holiday",
-              "name": "holiday",
-              "source": null
-            },
-            {
-              "id": "holiday",
-              "name": "holiday",
-              "source": null
-            }
-          ],
-          "merge_count": 5
-        }
-      }
-    },
-    {
-      "id": "2007",
-      "label": "2007",
-      "type": "DATE",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "2007",
-              "name": "2007",
-              "source": null
-            },
-            {
-              "id": "2007",
-              "name": "2007",
-              "source": null
-            },
-            {
-              "id": "2024",
-              "name": "2024",
-              "source": null
-            }
-          ],
-          "merge_count": 3
-        }
-      }
-    },
-    {
-      "id": "rosacea",
-      "label": "Rosacea",
-      "type": "CONCEPT",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "rosacea",
-              "name": "Rosacea",
-              "source": null
-            },
-            {
-              "id": "rosacea",
-              "name": "rosacea",
-              "source": null
-            },
-            {
-              "id": "rosacea_treatment_routine",
-              "name": "rosacea treatment routine",
-              "source": null
-            },
-            {
-              "id": "rosie",
-              "name": "Rosie",
-              "source": null
-            },
-            {
-              "id": "rosacea_therapy_skin_care_kit",
-              "name": "Rosacea Therapy Skin Care Kit",
-              "source": null
-            },
-            {
-              "id": "marnie",
-              "name": "Marnie",
-              "source": null
-            },
-            {
-              "id": "cavallo_point",
-              "name": "Cavallo Point",
-              "source": null
-            },
-            {
-              "id": "castor_seed_oil",
-              "name": "castor seed oil",
-              "source": null
-            }
-          ],
-          "merge_count": 8
-        }
-      }
-    },
-    {
-      "id": "dr._bailey",
-      "label": "Dr. Bailey",
-      "type": "PERSON",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "dr._bailey",
-              "name": "Dr. Bailey",
-              "source": null
-            },
-            {
-              "id": "dr._bailey",
-              "name": "Dr. Bailey",
-              "source": null
-            },
-            {
-              "id": "dr._bailey's_skincare",
-              "name": "Dr. Bailey's Skincare",
-              "source": null
-            },
-            {
-              "id": "dr._bailey's_skincare",
-              "name": "Dr. Bailey's Skincare",
-              "source": null
-            }
-          ],
-          "merge_count": 4
-        }
-      }
-    },
-    {
-      "id": "green_tea_antioxidant_skin_therapy",
-      "label": "Green Tea Antioxidant Skin Therapy",
-      "type": "PRODUCT",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "green_tea_antioxidant_skin_therapy",
-              "name": "Green Tea Antioxidant Skin Therapy",
-              "source": null
-            },
-            {
-              "id": "green_tea_antioxidant_skin_therapy",
-              "name": "Green Tea Antioxidant Skin Therapy",
-              "source": null
-            }
-          ],
-          "merge_count": 2
-        }
-      }
-    },
-    {
-      "id": "vol._892",
-      "label": "Vol. 892",
-      "type": "EVENT",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "vol._892",
-              "name": "Vol. 892",
-              "source": null
-            },
-            {
-              "id": "vol._891",
-              "name": "Vol. 891",
-              "source": null
-            }
-          ],
-          "merge_count": 2
-        }
-      }
-    },
-    {
-      "id": "laneige",
-      "label": "Laneige",
-      "type": "ORG",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "laneige",
-              "name": "Laneige",
-              "source": null
-            },
-            {
-              "id": "lanikai",
-              "name": "Lanikai",
-              "source": null
-            }
-          ],
-          "merge_count": 2
-        }
-      }
-    },
-    {
-      "id": "sausalito",
-      "label": "Sausalito",
-      "type": "GPE",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "sausalito",
-              "name": "Sausalito",
-              "source": null
-            },
-            {
-              "id": "sea_buckthorn",
-              "name": "sea buckthorn",
-              "source": null
-            }
-          ],
-          "merge_count": 2
-        }
-      }
-    },
-    {
-      "id": "ulta",
-      "label": "Ulta",
-      "type": "ORG",
-      "attributes": {
-        "confidence": 1.0,
-        "provenance": {
-          "merged_from": [
-            {
-              "id": "ulta",
-              "name": "Ulta",
-              "source": null
-            },
-            {
-              "id": "clotrimazole",
-              "name": "clotrimazole",
-              "source": null
-            }
-          ],
-          "merge_count": 2
-        }
-      }
-    },
-    {
-      "id": "december_15,_2025",
-      "label": "December 15, 2025",
-      "type": "DATE",
-      "attributes": {
-        "confidence": 1.0
-      }
-    },
-    {
-      "id": "jo_malone",
-      "label": "Jo Malone",
-      "type": "ORG",
-      "attributes": {
-        "confidence": 1
-      }
-    },
-    {
-      "id": "trader_joe",
-      "label": "Trader Joe",
-      "type": "ORG",
-      "attributes": {
-        "confidence": 1
-      }
-    },
-    {
-      "id": "hawaii",
-      "label": "hawaii",
-      "type": "GPE",
-      "attributes": {
-        "confidence": 1.0
-      }
-    },
-    {
-      "id": "benzoyl_peroxide_cream",
-      "label": "Benzoyl Peroxide Cream",
-      "type": "PRODUCT",
-      "attributes": {
-        "confidence": 1
-      }
-    },
-    {
-      "id": "facial_dandruff",
-      "label": "Facial dandruff",
-      "type": "CONCEPT",
-      "attributes": {
-        "confidence": 1
-      }
-    },
-    {
-      "id": "calming_zinc_soap",
-      "label": "Calming Zinc Soap",
-      "type": "PRODUCT",
-      "attributes": {
-        "confidence": 1
-      }
-    },
-    {
-      "id": "hydrate",
-      "label": "Hydrate",
-      "type": "CONCEPT",
-      "attributes": {
-        "confidence": 1.0
-      }
-    },
-    {
-      "id": "daily_moisturizing_face_cream",
-      "label": "Daily Moisturizing Face Cream",
-      "type": "PRODUCT",
-      "attributes": {
-        "confidence": 1.0
-      }
-    },
-    {
-      "id": "omega_enriched_face_booster_oil",
-      "label": "Omega Enriched Face Booster Oil",
-      "type": "PRODUCT",
-      "attributes": {
-        "confidence": 1.0
-      }
-    }
-  ],
-  "edges": [
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Monday Poll",
-      "type": "hosts",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Monday Poll",
-      "target": "December 15, 2025",
-      "type": "occurs on",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Makeup and Beauty Blog Monday Poll, Vol. 893",
-      "type": "publishes",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Monday",
-      "type": "has",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "2007",
-      "type": "has",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Monday Poll",
-      "type": "hosts",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Makeup and Beauty Blog Monday Poll",
-      "type": "posts",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Vol. 892",
-      "type": "posts",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "2007",
-      "type": "has been active since",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "MBB",
-      "target": "Makeup and Beauty Blog",
-      "type": "related_to",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Makeup and Beauty Blog",
-      "type": "related_to",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Monday Poll",
-      "type": "hosts",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Vol. 891",
-      "type": "posts",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Makeup and Beauty Blog",
-      "target": "Monday Poll",
-      "type": "posts",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Cavallo Point",
-      "target": "Sausalito",
-      "type": "located_in",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Dr. Bailey",
-      "target": "Green Tea Antioxidant Skin Therapy",
-      "type": "prescribes",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Green Tea Antioxidant Skin Therapy",
-      "target": "Rosacea Therapy Skin Care Kit",
-      "type": "part of",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Dr. Bailey",
-      "target": "Rosacea Therapy Skin Care Kit",
-      "type": "uses",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Rosacea Therapy Skin Care Kit",
-      "target": "rosacea treatment routine",
-      "type": "part of",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Dr. Bailey",
-      "target": "rosacea treatment routine",
-      "type": "uses",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Facial dandruff",
-      "target": "rosacea",
-      "type": "often occurs with",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Facial dandruff",
-      "target": "rosacea",
-      "type": "needs to be addressed",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Calming Zinc Soap",
-      "target": "Facial dandruff",
-      "type": "is often sufficient to control",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Calming Zinc Soap",
-      "target": "rosacea",
-      "type": "is often sufficient to control",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Green Tea Antioxidant Skin Therapy",
-      "target": "Facial dandruff",
-      "type": "is often sufficient to control",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Green Tea Antioxidant Skin Therapy",
-      "target": "rosacea",
-      "type": "is often sufficient to control",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Dr. Bailey's Skincare",
-      "target": "Calming Zinc Soap",
-      "type": "produces",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Dr. Bailey's Skincare",
-      "target": "Green Tea Antioxidant Skin Therapy",
-      "type": "produces",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Dr. Bailey",
-      "target": "Calming Zinc Soap",
-      "type": "prescribes",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Dr. Bailey",
-      "target": "Green Tea Antioxidant Skin Therapy",
-      "type": "prescribes",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Hydrate",
-      "target": "Daily Moisturizing Face Cream",
-      "type": "is_achieved_by",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Daily Moisturizing Face Cream",
-      "target": "Omega Enriched Face Booster Oil",
-      "type": "can_be_combined_with",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Omega Enriched Face Booster Oil",
-      "target": "castor seed oil",
-      "type": "contains",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Omega Enriched Face Booster Oil",
-      "target": "sea buckthorn",
-      "type": "contains",
-      "attributes": {
-        "confidence": 0.9
-      }
-    },
-    {
-      "source": "Daily Moisturizing Face Cream",
-      "target": "Omega Enriched Face Booster Oil",
-      "type": "can_be_replaced_with",
-      "attributes": {
-        "confidence": 0.9
-      }
-    }
-  ],
-  "metadata": {
-    "num_entities": 20,
-    "num_relationships": 35,
-    "temporal_enabled": false,
-    "timestamp": "2025-12-24T12:46:41.535755",
-    "entity_resolution_applied": true
-  }
-}

cookbook/use_cases/advanced_rag/ground_truth.json

@@ -1 +0,0 @@
-{"entities": [{"id": "python_org", "name": "Python Software Foundation", "type": "Organization"}, {"id": "guido_van_rossum", "name": "Guido van Rossum", "type": "Person"}], "relationships": [{"source": "guido_van_rossum", "target": "python_org", "type": "FOUNDED"}]}

cookbook/use_cases/advanced_rag/skincare_base.json

@@ -1,38 +0,0 @@
-{
-  "entities": [
-    {
-      "id": "hyaluronic_acid",
-      "name": "Hyaluronic Acid",
-      "type": "Ingredient",
-      "properties": {
-        "role": "Humectant"
-      }
-    },
-    {
-      "id": "retinol",
-      "name": "Retinol",
-      "type": "Ingredient",
-      "properties": {
-        "role": "Anti-aging actives"
-      }
-    },
-    {
-      "id": "niacinamide",
-      "name": "Niacinamide",
-      "type": "Ingredient",
-      "properties": {
-        "role": "Barrier repair"
-      }
-    }
-  ],
-  "relationships": [
-    {
-      "source": "hyaluronic_acid",
-      "target": "niacinamide",
-      "type": "COMPLEMENTS",
-      "properties": {
-        "benefit": "Hydration + Barrier"
-      }
-    }
-  ]
-}

cookbook/use_cases/biomedical/01_Drug_Discovery_Pipeline.ipynb

@@ -1,693 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/biomedical/01_Drug_Discovery_Pipeline.ipynb)\n",
-    "\n",
-    "# Drug Discovery Pipeline - Vector Similarity Search\n",
-    "\n",
-    "## Overview\n",
-    "\n",
-    "This notebook demonstrates a **complete drug discovery pipeline** using Semantica's modular architecture. We'll use individual modules directly to build a comprehensive system for drug-target interaction prediction using vector similarity search and knowledge graphs.\n",
-    "\n",
-    "### Key Features\n",
-    "\n",
-    "- **Modular Architecture**: Uses Semantica modules directly (`NERExtractor`, `GraphBuilder`, `EmbeddingGenerator`, `VectorStore`)\n",
-    "- **Multiple Data Sources**: Ingests from 15+ PubMed RSS feeds, preprint servers, and journal feeds\n",
-    "- **Vector Similarity Search**: Emphasizes embeddings and vector similarity for drug-target interaction prediction\n",
-    "- **Entity Extraction**: Extracts drug compounds, proteins, targets, enzymes, and receptors\n",
-    "- **Knowledge Graph**: Builds structured drug-target relationship graphs\n",
-    "- **GraphRAG**: Hybrid vector + graph retrieval for enhanced querying\n",
-    "\n",
-    "### What You'll Learn\n",
-    "\n",
-    "- How to use Semantica modules directly (avoiding the core orchestrator)\n",
-    "- How to ingest biomedical data from multiple sources\n",
-    "- How to extract entities using `NERExtractor`\n",
-    "- How to extract relationships using `RelationExtractor`\n",
-    "- How to generate embeddings with `EmbeddingGenerator`\n",
-    "- How to build knowledge graphs with `GraphBuilder`\n",
-    "- How to perform similarity search with `VectorStore`\n",
-    "- How to use GraphRAG with `AgentContext` for hybrid retrieval\n",
-    "\n",
-    "### Pipeline Flow\n",
-    "\n",
-    "```mermaid\n",
-    "graph LR\n",
-    "    A[Data Ingestion] --> B[Text Processing]\n",
-    "    B --> C[Entity Extraction]\n",
-    "    C --> D[Relationship Extraction]\n",
-    "    D --> E[Deduplication]\n",
-    "    E --> F[Embedding Generation]\n",
-    "    F --> G[Vector Store]\n",
-    "    G --> H[Knowledge Graph]\n",
-    "    H --> I[Similarity Search]\n",
-    "    H --> J[GraphRAG Queries]\n",
-    "    I --> K[Visualization]\n",
-    "    J --> K\n",
-    "```\n",
-    "\n",
-    "### Data Sources\n",
-    "\n",
-    "**PubMed RSS Feeds:**\n",
-    "- Drug Discovery, Drug Target Interaction, Pharmacokinetics, Pharmacodynamics\n",
-    "- Clinical Trials, Protein Targets, Drug Repurposing, Molecular Docking\n",
-    "- ADME, Drug Metabolism, Drug Safety, Precision Medicine\n",
-    "- Biomarkers, Drug Resistance, Combinatorial Therapy\n",
-    "\n",
-    "**Preprint Servers:**\n",
-    "- BioRxiv (Pharmacology & Toxicology, Drug Discovery)\n",
-    "- MedRxiv (Clinical Trials)\n",
-    "- ChemRxiv\n",
-    "\n",
-    "**Journal RSS Feeds:**\n",
-    "- Nature (Drug Discovery, Pharmacology)\n",
-    "- Science Translational Medicine\n",
-    "- Cell Chemical Biology\n",
-    "- Journal of Medicinal Chemistry\n",
-    "- Drug Discovery Today\n",
-    "- Trends in Pharmacological Sciences\n",
-    "\n",
-    "\n",
-    "---\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Installation\n",
-    "\n",
-    "Install Semantica and required dependencies:\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq sentence-transformers scikit-learn\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Configuration & Setup\n",
-    "\n",
-    "Set up environment variables and configuration constants.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "\n",
-    "os.environ[\"GROQ_API_KEY\"] = os.getenv(\"GROQ_API_KEY\", \"\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "EMBEDDING_DIMENSION = 384\n",
-    "EMBEDDING_MODEL = \"all-MiniLM-L6-v2\"\n",
-    "CHUNK_SIZE = 1000\n",
-    "CHUNK_OVERLAP = 200\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Ingesting Biomedical Data from Multiple Sources\n",
-    "\n",
-    "Ingest data from comprehensive biomedical sources including PubMed RSS feeds, preprint servers, and journal feeds.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.ingest import FeedIngestor, FileIngestor\n",
-    "import os\n",
-    "from contextlib import redirect_stderr\n",
-    "from io import StringIO\n",
-    "\n",
-    "os.makedirs(\"data\", exist_ok=True)\n",
-    "\n",
-    "feed_sources = [\n",
-    "    # Nature Feeds\n",
-    "    (\"Nature - Drug Discovery\", \"https://www.nature.com/subjects/drug-discovery.rss\"),\n",
-    "    (\"Nature - Pharmacology\", \"https://www.nature.com/subjects/pharmacology.rss\"),\n",
-    "    (\"Nature Reviews Drug Discovery\", \"https://www.nature.com/nrd.rss\"),\n",
-    "    \n",
-    "    # FDA & Government Sources\n",
-    "    (\"FDA MedWatch\", \"https://www.fda.gov/AboutFDA/ContactFDA/StayInformed/RSSFeeds/MedWatch/rss.xml\"),\n",
-    "    (\"NCI News\", \"https://www.cancer.gov/syndication/rss\"),\n",
-    "    \n",
-    "    # Drug Information & News\n",
-    "    (\"Drugs.com - MedNews\", \"https://www.drugs.com/rss/mednews.xml\"),\n",
-    "    (\"Drugs.com - FDA Alerts\", \"https://www.drugs.com/rss/fda-alerts.xml\"),\n",
-    "    (\"Drugs.com - Clinical Trials\", \"https://www.drugs.com/rss/clinical-trials.xml\"),\n",
-    "    \n",
-    "    # Medical News\n",
-    "    (\"Labroots Health & Medicine\", \"http://www.labroots.com/rss/trending/health-and-medicine\"),\n",
-    "    (\"Biology News Net\", \"https://www.biologynews.net/rss.php\"),\n",
-    "    \n",
-    "    # Open Access Journals\n",
-    "    (\"PLOS ONE - Medicine\", \"https://journals.plos.org/plosone/feed/atom\"),\n",
-    "    (\"PLOS Biology\", \"https://journals.plos.org/plosbiology/feed/atom\"),\n",
-    "    (\"PLOS Medicine\", \"https://journals.plos.org/plosmedicine/feed/atom\"),\n",
-    "    \n",
-    "    # Preprint Servers\n",
-    "    (\"arXiv - q-bio\", \"http://arxiv.org/rss/q-bio\"),\n",
-    "    (\"arXiv - q-bio.BM\", \"http://arxiv.org/rss/q-bio.BM\"),\n",
-    "]\n",
-    "\n",
-    "feed_ingestor = FeedIngestor()\n",
-    "all_documents = []\n",
-    "\n",
-    "print(f\"Ingesting from {len(feed_sources)} feed sources...\")\n",
-    "for i, (feed_name, feed_url) in enumerate(feed_sources, 1):\n",
-    "    try:\n",
-    "        with redirect_stderr(StringIO()):\n",
-    "            feed_data = feed_ingestor.ingest_feed(feed_url, validate=False)\n",
-    "        \n",
-    "        feed_count = 0\n",
-    "        for item in feed_data.items:\n",
-    "            if not item.content:\n",
-    "                item.content = item.description or item.title or \"\"\n",
-    "            if item.content:\n",
-    "                if not hasattr(item, 'metadata'):\n",
-    "                    item.metadata = {}\n",
-    "                item.metadata['source'] = feed_name\n",
-    "                all_documents.append(item)\n",
-    "                feed_count += 1\n",
-    "        \n",
-    "        if feed_count > 0:\n",
-    "            print(f\"  [{i}/{len(feed_sources)}] {feed_name}: {feed_count} documents\")\n",
-    "    except Exception:\n",
-    "        continue\n",
-    "\n",
-    "if not all_documents:\n",
-    "    sample_drug_data = \"\"\"\n",
-    "    Aspirin (acetylsalicylic acid) is a medication used to reduce pain, fever, or inflammation. \n",
-    "    It targets cyclooxygenase enzymes COX-1 and COX-2. Aspirin is commonly used for cardiovascular protection.\n",
-    "    Ibuprofen is a nonsteroidal anti-inflammatory drug (NSAID) that targets COX-1 and COX-2 enzymes.\n",
-    "    Metformin is an antidiabetic medication that targets AMP-activated protein kinase (AMPK).\n",
-    "    Insulin targets the insulin receptor (INSR) to regulate glucose metabolism.\n",
-    "    Warfarin is an anticoagulant that targets vitamin K epoxide reductase complex subunit 1 (VKORC1).\n",
-    "    Atorvastatin is a statin medication that targets HMG-CoA reductase.\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    with open(\"data/sample_drugs.txt\", \"w\") as f:\n",
-    "        f.write(sample_drug_data)\n",
-    "    \n",
-    "    file_ingestor = FileIngestor()\n",
-    "    all_documents = file_ingestor.ingest(\"data/sample_drugs.txt\")\n",
-    "\n",
-    "documents = all_documents\n",
-    "print(f\"Ingested {len(documents)} documents\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Normalizing and Chunking Documents\n",
-    "\n",
-    "Clean and normalize text, then split into chunks using entity-aware chunking to preserve drug/protein entity boundaries.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.normalize import TextNormalizer\n",
-    "from semantica.split import TextSplitter\n",
-    "\n",
-    "normalizer = TextNormalizer()\n",
-    "splitter = TextSplitter(\n",
-    "    method=\"entity_aware\",\n",
-    "    ner_method=\"spacy\",\n",
-    "    chunk_size=CHUNK_SIZE,\n",
-    "    chunk_overlap=CHUNK_OVERLAP\n",
-    ")\n",
-    "\n",
-    "print(f\"Normalizing {len(documents)} documents...\")\n",
-    "normalized_documents = []\n",
-    "for i, doc in enumerate(documents, 1):\n",
-    "    normalized_text = normalizer.normalize(\n",
-    "        doc.content if hasattr(doc, 'content') else str(doc),\n",
-    "        clean_html=True,\n",
-    "        normalize_entities=True,\n",
-    "        remove_extra_whitespace=True,\n",
-    "        lowercase=False\n",
-    "    )\n",
-    "    normalized_documents.append(normalized_text)\n",
-    "    if i % 50 == 0 or i == len(documents):\n",
-    "        print(f\"  Normalized {i}/{len(documents)} documents...\")\n",
-    "\n",
-    "print(f\"Chunking {len(normalized_documents)} documents...\")\n",
-    "chunked_documents = []\n",
-    "for i, doc_text in enumerate(normalized_documents, 1):\n",
-    "    try:\n",
-    "        with redirect_stderr(StringIO()):\n",
-    "            chunks = splitter.split(doc_text)\n",
-    "        chunked_documents.extend(chunks)\n",
-    "    except Exception:\n",
-    "        simple_splitter = TextSplitter(method=\"recursive\", chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP)\n",
-    "        chunks = simple_splitter.split(doc_text)\n",
-    "        chunked_documents.extend(chunks)\n",
-    "    if i % 50 == 0 or i == len(normalized_documents):\n",
-    "        print(f\"  Chunked {i}/{len(normalized_documents)} documents ({len(chunked_documents)} chunks so far)\")\n",
-    "\n",
-    "print(f\"Created {len(chunked_documents)} chunks from {len(normalized_documents)} documents\")\n",
-    "\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.semantic_extract import NERExtractor\n",
-    "\n",
-    "# Using spaCy ML method (similar to NER cell)\n",
-    "entity_extractor = NERExtractor(method=\"ml\", model=\"en_core_web_sm\")\n",
-    "\n",
-    "all_entities = []\n",
-    "print(f\"Extracting entities from {len(chunked_documents)} chunks...\")\n",
-    "\n",
-    "for i, chunk in enumerate(chunked_documents, 1):\n",
-    "    chunk_text = chunk.text if hasattr(chunk, 'text') else str(chunk)\n",
-    "    try:\n",
-    "        entities = entity_extractor.extract_entities(chunk_text)\n",
-    "        all_entities.extend(entities)\n",
-    "    except Exception:\n",
-    "        continue\n",
-    "    \n",
-    "    if i % 20 == 0 or i == len(chunked_documents):\n",
-    "        remaining = len(chunked_documents) - i\n",
-    "        print(f\"  Processed {i}/{len(chunked_documents)} chunks ({len(all_entities)} entities found, {remaining} remaining)\")\n",
-    "\n",
-    "# Filter entities - spaCy returns standard types (PERSON, ORG, PRODUCT, etc.)\n",
-    "# Map to biomedical categories based on context\n",
-    "drugs = [e for e in all_entities if e.label == \"PRODUCT\" or (e.label == \"ORG\" and any(kw in e.text.lower() for kw in [\"drug\", \"pharma\", \"medication\"]))]\n",
-    "proteins = [e for e in all_entities if e.label == \"ORG\" or (e.label == \"PRODUCT\" and any(kw in e.text.lower() for kw in [\"protein\", \"enzyme\", \"receptor\", \"kinase\", \"target\"]))]\n",
-    "\n",
-    "print(f\"Extracted {len(drugs)} drugs and {len(proteins)} proteins\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Extracting Drug-Target Relationships\n",
-    "\n",
-    "Extract relationships between drugs and proteins to understand drug-target interactions.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.semantic_extract import RelationExtractor\n",
-    "\n",
-    "# Using spaCy dependency parsing (similar to NER cell)\n",
-    "relation_extractor = RelationExtractor(method=\"dependency\", model=\"en_core_web_sm\")\n",
-    "\n",
-    "all_relationships = []\n",
-    "print(f\"Extracting relationships from {len(chunked_documents)} chunks...\")\n",
-    "\n",
-    "for i, chunk in enumerate(chunked_documents, 1):\n",
-    "    chunk_text = chunk.text if hasattr(chunk, 'text') else str(chunk)\n",
-    "    try:\n",
-    "        relationships = relation_extractor.extract_relations(\n",
-    "            chunk_text,\n",
-    "            entities=all_entities,\n",
-    "            relation_types=[\"targets\", \"inhibits\", \"activates\", \"binds_to\", \"interacts_with\"]\n",
-    "        )\n",
-    "        all_relationships.extend(relationships)\n",
-    "    except Exception:\n",
-    "        continue\n",
-    "    \n",
-    "    if i % 20 == 0 or i == len(chunked_documents):\n",
-    "        print(f\"  Processed {i}/{len(chunked_documents)} chunks ({len(all_relationships)} relationships found)\")\n",
-    "\n",
-    "print(f\"Extracted {len(all_relationships)} relationships\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Resolving Duplicate Entities\n",
-    "\n",
-    "Detect and merge duplicate entities to ensure data quality and consistency.\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Conflict Detection and Resolution\n",
-    "\n",
-    "Detect and resolve conflicts in drug-target relationships from multiple research sources.\n",
-    "\n",
-    "- **Detection Method**: Relationship conflict detection identifies discrepancies in drug-target interactions across sources\n",
-    "- **Resolution Strategy**: Credibility-weighted resolution prioritizes higher-credibility sources (e.g., Nature journals over arXiv preprints)\n",
-    "- **Use Case**: Handles conflicting information when multiple sources report different drug-target relationships\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.conflicts import ConflictDetector, ConflictResolver\n",
-    "\n",
-    "detector = ConflictDetector()\n",
-    "resolver = ConflictResolver(default_strategy=\"credibility_weighted\")\n",
-    "\n",
-    "# Convert to dict format for conflict detection\n",
-    "entities = [\n",
-    "    {\n",
-    "        \"id\": ent.text if hasattr(ent, 'text') else str(ent),\n",
-    "        \"name\": ent.text if hasattr(ent, 'text') else str(ent),\n",
-    "        \"type\": ent.label if hasattr(ent, 'label') else \"ENTITY\",\n",
-    "        \"confidence\": getattr(ent, 'confidence', 1.0),\n",
-    "        \"source\": ent.metadata.get(\"source\", \"unknown\") if hasattr(ent, 'metadata') and ent.metadata else \"unknown\"\n",
-    "    }\n",
-    "    for ent in all_entities if hasattr(ent, 'text') or hasattr(ent, 'label')\n",
-    "]\n",
-    "\n",
-    "relationships = [\n",
-    "    {\n",
-    "        \"id\": f\"{rel.subject.text}_{rel.object.text}_{rel.predicate}\",\n",
-    "        \"source_id\": rel.subject.text,\n",
-    "        \"target_id\": rel.object.text,\n",
-    "        \"type\": rel.predicate,\n",
-    "        \"confidence\": getattr(rel, 'confidence', 1.0),\n",
-    "        \"source\": rel.metadata.get(\"source\", \"unknown\") if hasattr(rel, 'metadata') and rel.metadata else \"unknown\"\n",
-    "    }\n",
-    "    for rel in all_relationships if hasattr(rel, 'subject')\n",
-    "]\n",
-    "\n",
-    "# Detect and resolve conflicts\n",
-    "print(f\"Detecting conflicts in {len(entities)} entities, {len(relationships)} relationships...\")\n",
-    "entity_conflicts = detector.detect_conflicts(entities)\n",
-    "relationship_conflicts = detector.detect_relationship_conflicts(relationships)\n",
-    "print(f\"Detected {len(entity_conflicts)} entity conflicts, {len(relationship_conflicts)} relationship conflicts\")\n",
-    "\n",
-    "# Resolve conflicts\n",
-    "if entity_conflicts:\n",
-    "    resolver.resolve_conflicts(entity_conflicts, strategy=\"credibility_weighted\")\n",
-    "    print(f\"Resolved {len(entity_conflicts)} entity conflicts\")\n",
-    "\n",
-    "if relationship_conflicts:\n",
-    "    resolver.resolve_conflicts(relationship_conflicts, strategy=\"credibility_weighted\")\n",
-    "    print(f\"Resolved {len(relationship_conflicts)} relationship conflicts\")\n",
-    "\n",
-    "# GraphBuilder will use resolve_conflicts=True to apply resolutions automatically\n",
-    "print(\"Conflicts resolved. GraphBuilder will use cleaned data.\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Generating Vector Embeddings\n",
-    "\n",
-    "Generate embeddings for drugs and proteins to enable similarity search.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.embeddings import EmbeddingGenerator\n",
-    "from semantica.vector_store import VectorStore\n",
-    "\n",
-    "embedding_gen = EmbeddingGenerator(\n",
-    "    provider=\"sentence_transformers\",\n",
-    "    model=EMBEDDING_MODEL\n",
-    ")\n",
-    "\n",
-    "vector_store = VectorStore(backend=\"faiss\", dimension=EMBEDDING_DIMENSION)\n",
-    "\n",
-    "print(f\"Generating embeddings for {len(drugs)} drugs and {len(proteins)} proteins...\")\n",
-    "drug_texts = [d.text for d in drugs]\n",
-    "drug_embeddings = embedding_gen.generate_embeddings(drug_texts)\n",
-    "\n",
-    "protein_texts = [p.text for p in proteins]\n",
-    "protein_embeddings = embedding_gen.generate_embeddings(protein_texts)\n",
-    "\n",
-    "print(f\"Generated {len(drug_embeddings)} drug embeddings and {len(protein_embeddings)} protein embeddings\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Populating Vector Database\n",
-    "\n",
-    "Store drug and protein embeddings in the vector database with metadata for efficient similarity search.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(f\"Storing {len(drug_embeddings)} drug vectors and {len(protein_embeddings)} protein vectors...\")\n",
-    "drug_ids = vector_store.store_vectors(\n",
-    "    vectors=drug_embeddings,\n",
-    "    metadata=[{\"type\": \"drug\", \"name\": d.text, \"label\": d.label} for d in drugs]\n",
-    ")\n",
-    "\n",
-    "protein_ids = vector_store.store_vectors(\n",
-    "    vectors=protein_embeddings,\n",
-    "    metadata=[{\"type\": \"protein\", \"name\": p.text, \"label\": p.label} for p in proteins]\n",
-    ")\n",
-    "\n",
-    "print(f\"Stored {len(drug_ids)} drug vectors and {len(protein_ids)} protein vectors\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Building Drug-Target Knowledge Graph\n",
-    "\n",
-    "Construct a knowledge graph from extracted entities and relationships to enable graph-based reasoning.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.kg import GraphBuilder\n",
-    "\n",
-    "graph_builder = GraphBuilder()\n",
-    "\n",
-    "print(f\"Building graph from {len(all_entities)} entities, {len(all_relationships)} relationships...\")\n",
-    "kg = graph_builder.build({\n",
-    "    \"entities\": all_entities,\n",
-    "    \"relationships\": all_relationships\n",
-    "})\n",
-    "\n",
-    "entities_count = len(kg.get('entities', []))\n",
-    "relationships_count = len(kg.get('relationships', []))\n",
-    "print(f\"Graph: {entities_count} entities, {relationships_count} relationships\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Finding Similar Drugs via Vector Search\n",
-    "\n",
-    "Use vector similarity search to find drugs similar to a query drug based on their embeddings.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "query_drug = \"Aspirin\"\n",
-    "query_embedding = embedding_gen.generate_embeddings([query_drug])[0]\n",
-    "similar_drugs = vector_store.search_vectors(query_embedding, k=5)\n",
-    "\n",
-    "print(f\"Drugs similar to '{query_drug}':\")\n",
-    "for i, result in enumerate(similar_drugs, 1):\n",
-    "    metadata = result.get('metadata', {})\n",
-    "    name = metadata.get('name', 'Unknown') if metadata else 'Unknown'\n",
-    "    score = result.get('score', 0.0)\n",
-    "    print(f\"{i}. {name} (similarity: {score:.3f})\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## GraphRAG: Hybrid Vector + Graph Retrieval\n",
-    "\n",
-    "Use GraphRAG to combine vector similarity search with knowledge graph traversal for enhanced retrieval and reasoning.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.context import AgentContext, ContextRetriever\n",
-    "\n",
-    "# Option 1: Use AgentContext (high-level, recommended)\n",
-    "context = AgentContext(\n",
-    "    vector_store=vector_store, \n",
-    "    knowledge_graph=kg,\n",
-    "    hybrid_alpha=0.6,\n",
-    "    max_expansion_hops=2\n",
-    ")\n",
-    "\n",
-    "# Option 2: Use ContextRetriever directly (more control)\n",
-    "retriever = ContextRetriever(\n",
-    "    vector_store=vector_store,\n",
-    "    knowledge_graph=kg,\n",
-    "    hybrid_alpha=0.6,\n",
-    "    max_expansion_hops=2\n",
-    ")\n",
-    "\n",
-    "# GraphRAG query using AgentContext\n",
-    "query = \"What drugs target COX enzymes?\"\n",
-    "results = context.retrieve(\n",
-    "    query,\n",
-    "    max_results=10,\n",
-    "    use_graph=True,\n",
-    "    expand_graph=True,\n",
-    "    include_entities=True,\n",
-    "    include_relationships=True\n",
-    ")\n",
-    "\n",
-    "\n",
-    "print(f\"Query: '{query}'\")\n",
-    "print(f\"Retrieved {len(results)} results:\\n\")\n",
-    "for i, result in enumerate(results[:5], 1):\n",
-    "    print(f\"{i}. Score: {result.get('score', 0):.3f}\")\n",
-    "    if result.get('content'):\n",
-    "        print(f\"   {result['content'][:250]}\")\n",
-    "    if result.get('related_entities'):\n",
-    "        entities = result['related_entities']\n",
-    "        names = [e.get('name', e.get('id', '')) for e in entities[:3]]\n",
-    "        print(f\"   Entities: {', '.join(names)}\" + (f\" (+{len(entities)-3})\" if len(entities) > 3 else \"\"))\n",
-    "    if result.get('related_relationships'):\n",
-    "        print(f\"   Relationships: {len(result['related_relationships'])}\")\n",
-    "    print()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Visualizing the Knowledge Graph\n",
-    "\n",
-    "Generate an interactive visualization of the drug-target knowledge graph.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.visualization import KGVisualizer\n",
-    "\n",
-    "# Display interactive Plotly graph directly in notebook\n",
-    "visualizer = KGVisualizer(layout=\"force\", node_size=20)\n",
-    "fig = visualizer.visualize_network(kg, output=\"interactive\")\n",
-    "\n",
-    "# Display the figure (Plotly will show it automatically in notebook)\n",
-    "fig.show() if fig else None"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Exporting Results\n",
-    "\n",
-    "Export the knowledge graph to various formats for further analysis or integration with other tools.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.export import GraphExporter\n",
-    "\n",
-    "exporter = GraphExporter()\n",
-    "exporter.export(kg, output_path=\"drug_target_kg.json\", format=\"json\")\n",
-    "exporter.export(kg, output_path=\"drug_target_kg.graphml\", format=\"graphml\")\n",
-    "\n",
-    "print(\"Exported knowledge graph to JSON and GraphML formats\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

cookbook/use_cases/biomedical/02_Genomic_Variant_Analysis.ipynb

@@ -1,719 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/biomedical/02_Genomic_Variant_Analysis.ipynb)\n",
-    "\n",
-    "# Genomic Variant Analysis - Graph Analytics & Pathway Analysis\n",
-    "\n",
-    "## Overview\n",
-    "\n",
-    "This notebook demonstrates **genomic variant analysis** using Semantica's modular architecture with focus on **graph analytics**, **pathway analysis**, and **temporal knowledge graphs**. The pipeline analyzes genomic data to extract variant entities, build temporal genomic knowledge graphs, and analyze disease associations through reasoning.\n",
-    "\n",
-    "### Key Features\n",
-    "\n",
-    "- **Graph Analytics Focus**: Emphasizes graph reasoning, centrality measures, and pathway analysis\n",
-    "- **Temporal Analysis**: Builds temporal genomic knowledge graphs to track variant evolution\n",
-    "- **Disease Association**: Analyzes relationships between variants, genes, and diseases\n",
-    "- **Pathway Analysis**: Uses graph traversal to identify biological pathways\n",
-    "- **Impact Prediction**: Predicts variant impact using graph-based reasoning\n",
-    "\n",
-    "### What You'll Learn\n",
-    "\n",
-    "- How to use Semantica modules directly for genomic analysis\n",
-    "- How to ingest genomic data from multiple sources\n",
-    "- How to extract variant, gene, and disease entities\n",
-    "- How to build temporal knowledge graphs\n",
-    "- How to perform graph analytics (centrality, communities)\n",
-    "- How to use temporal queries for variant evolution\n",
-    "- How to analyze pathways using reasoning\n",
-    "- How to visualize and export genomic knowledge graphs\n",
-    "\n",
-    "### Pipeline Flow\n",
-    "\n",
-    "```mermaid\n",
-    "graph LR\n",
-    "    A[Data Ingestion] --> B[Text Processing]\n",
-    "    B --> C[Entity Extraction]\n",
-    "    C --> D[Relationship Extraction]\n",
-    "    D --> E[Deduplication]\n",
-    "    E --> F[Temporal KG]\n",
-    "    F --> G[Graph Analytics]\n",
-    "    F --> H[Temporal Queries]\n",
-    "    G --> I[Pathway Analysis]\n",
-    "    H --> I\n",
-    "    I --> J[Disease Associations]\n",
-    "    J --> K[Visualization]\n",
-    "```\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Installation\n",
-    "\n",
-    "Install Semantica and required dependencies:\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%pip install -qU semantica networkx matplotlib plotly pandas groq sentence-transformers\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Configuration & Setup\n",
-    "\n",
-    "Set up environment variables and configuration constants.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "\n",
-    "os.environ[\"GROQ_API_KEY\"] = os.getenv(\"GROQ_API_KEY\", \"\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "CHUNK_SIZE = 1000\n",
-    "CHUNK_OVERLAP = 200\n",
-    "TEMPORAL_GRANULARITY = \"day\"\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Ingesting Genomic Data from Multiple Sources\n",
-    "\n",
-    "Ingest data from comprehensive genomic sources including PubMed RSS feeds, preprint servers, and journal feeds.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.ingest import FeedIngestor, FileIngestor\n",
-    "import os\n",
-    "from contextlib import redirect_stderr\n",
-    "from io import StringIO\n",
-    "\n",
-    "os.makedirs(\"data\", exist_ok=True)\n",
-    "\n",
-    "feed_sources = [\n",
-    "    # PubMed RSS Feeds (simplified, working format)\n",
-    "    (\"PubMed - Genetics\", \"https://pubmed.ncbi.nlm.nih.gov/rss/search/1?term=genetics&limit=10\"),\n",
-    "    (\"PubMed - Genomics\", \"https://pubmed.ncbi.nlm.nih.gov/rss/search/1?term=genomics&limit=10\"),\n",
-    "    (\"PubMed - Variant Analysis\", \"https://pubmed.ncbi.nlm.nih.gov/rss/search/1?term=variant+analysis&limit=10\"),\n",
-    "    (\"PubMed - GWAS\", \"https://pubmed.ncbi.nlm.nih.gov/rss/search/1?term=GWAS&limit=10\"),\n",
-    "    (\"PubMed - Genomic Medicine\", \"https://pubmed.ncbi.nlm.nih.gov/rss/search/1?term=genomic+medicine&limit=10\"),\n",
-    "    (\"PubMed - Precision Medicine\", \"https://pubmed.ncbi.nlm.nih.gov/rss/search/1?term=precision+medicine&limit=10\"),\n",
-    "    (\"PubMed - Pharmacogenomics\", \"https://pubmed.ncbi.nlm.nih.gov/rss/search/1?term=pharmacogenomics&limit=10\"),\n",
-    "    \n",
-    "    # Nature Feeds (working format)\n",
-    "    (\"Nature Genetics\", \"https://www.nature.com/subjects/genetics.rss\"),\n",
-    "    (\"Nature - Genomics\", \"https://www.nature.com/subjects/genomics.rss\"),\n",
-    "    \n",
-    "    # PLOS Journals (working Atom feeds)\n",
-    "    (\"PLOS Genetics\", \"https://journals.plos.org/plosgenetics/feed/atom\"),\n",
-    "    (\"PLOS ONE - Genetics\", \"https://journals.plos.org/plosone/feed/atom\"),\n",
-    "    \n",
-    "    # Other working feeds\n",
-    "    (\"Genome Research\", \"https://genome.cshlp.org/rss/current.xml\"),\n",
-    "]\n",
-    "\n",
-    "feed_ingestor = FeedIngestor()\n",
-    "all_documents = []\n",
-    "\n",
-    "print(f\"Ingesting from {len(feed_sources)} feed sources...\")\n",
-    "for i, (feed_name, feed_url) in enumerate(feed_sources, 1):\n",
-    "    try:\n",
-    "        with redirect_stderr(StringIO()):\n",
-    "            feed_data = feed_ingestor.ingest_feed(feed_url, validate=False)\n",
-    "        \n",
-    "        feed_count = 0\n",
-    "        for item in feed_data.items:\n",
-    "            if not item.content:\n",
-    "                item.content = item.description or item.title or \"\"\n",
-    "            if item.content:\n",
-    "                if not hasattr(item, 'metadata'):\n",
-    "                    item.metadata = {}\n",
-    "                item.metadata['source'] = feed_name\n",
-    "                all_documents.append(item)\n",
-    "                feed_count += 1\n",
-    "        \n",
-    "        if feed_count > 0:\n",
-    "            print(f\"  [{i}/{len(feed_sources)}] {feed_name}: {feed_count} documents\")\n",
-    "    except Exception as e:\n",
-    "        print(f\"  [{i}/{len(feed_sources)}] {feed_name}: Failed\")\n",
-    "        continue\n",
-    "\n",
-    "# Always include fallback variant data for demonstration\n",
-    "variant_data = \"\"\"\n",
-    "Variant rs699 is located in the AGT gene and associated with hypertension.\n",
-    "Variant rs7412 in APOE gene is linked to Alzheimer's disease risk.\n",
-    "BRCA1 variant c.5266dupC increases breast cancer susceptibility.\n",
-    "CFTR variant F508del causes cystic fibrosis.\n",
-    "Variant rs1800566 in NAT2 gene affects drug metabolism.\n",
-    "Variant rs1042713 in ADRB2 gene is associated with asthma response.\n",
-    "TP53 variant R273H is linked to multiple cancer types.\n",
-    "Variant rs1799853 in CYP2C9 gene affects warfarin metabolism.\n",
-    "Variant rs1057910 in CYP2C9 affects phenytoin metabolism.\n",
-    "Variant rs9923231 in VKORC1 gene influences warfarin dosing.\n",
-    "\"\"\"\n",
-    "\n",
-    "with open(\"data/variants.txt\", \"w\") as f:\n",
-    "    f.write(variant_data)\n",
-    "\n",
-    "file_ingestor = FileIngestor()\n",
-    "fallback_docs = file_ingestor.ingest(\"data/variants.txt\")\n",
-    "all_documents.extend(fallback_docs)\n",
-    "\n",
-    "documents = all_documents\n",
-    "print(f\"\\nTotal ingested: {len(documents)} documents\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Normalizing and Chunking Genomic Documents\n",
-    "\n",
-    "Clean and normalize text, then split into chunks using entity-aware chunking to preserve variant/gene entity boundaries.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.normalize import TextNormalizer\n",
-    "from semantica.split import TextSplitter\n",
-    "\n",
-    "normalizer = TextNormalizer()\n",
-    "splitter = TextSplitter(\n",
-    "    method=\"entity_aware\",\n",
-    "    ner_method=\"spacy\",\n",
-    "    chunk_size=CHUNK_SIZE,\n",
-    "    chunk_overlap=CHUNK_OVERLAP\n",
-    ")\n",
-    "\n",
-    "print(f\"Normalizing {len(documents)} documents...\")\n",
-    "normalized_documents = []\n",
-    "for i, doc in enumerate(documents, 1):\n",
-    "    normalized_text = normalizer.normalize(\n",
-    "        doc.content if hasattr(doc, 'content') else str(doc),\n",
-    "        clean_html=True,\n",
-    "        normalize_entities=True,\n",
-    "        remove_extra_whitespace=True,\n",
-    "        lowercase=False\n",
-    "    )\n",
-    "    normalized_documents.append(normalized_text)\n",
-    "    if i % 50 == 0 or i == len(documents):\n",
-    "        print(f\"  Normalized {i}/{len(documents)} documents...\")\n",
-    "\n",
-    "print(f\"Chunking {len(normalized_documents)} documents...\")\n",
-    "chunked_documents = []\n",
-    "for i, doc_text in enumerate(normalized_documents, 1):\n",
-    "    try:\n",
-    "        with redirect_stderr(StringIO()):\n",
-    "            chunks = splitter.split(doc_text)\n",
-    "        chunked_documents.extend(chunks)\n",
-    "    except Exception:\n",
-    "        simple_splitter = TextSplitter(method=\"recursive\", chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP)\n",
-    "        chunks = simple_splitter.split(doc_text)\n",
-    "        chunked_documents.extend(chunks)\n",
-    "    if i % 50 == 0 or i == len(normalized_documents):\n",
-    "        print(f\"  Chunked {i}/{len(normalized_documents)} documents ({len(chunked_documents)} chunks so far)\")\n",
-    "\n",
-    "print(f\"Created {len(chunked_documents)} chunks from {len(normalized_documents)} documents\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.semantic_extract import NERExtractor\n",
-    "\n",
-    "# Using spaCy ML method (similar to Drug Discovery Pipeline)\n",
-    "entity_extractor = NERExtractor(method=\"ml\", model=\"en_core_web_sm\")\n",
-    "\n",
-    "all_entities = []\n",
-    "print(f\"Extracting entities from {len(chunked_documents)} chunks...\")\n",
-    "\n",
-    "for i, chunk in enumerate(chunked_documents, 1):\n",
-    "    chunk_text = chunk.text if hasattr(chunk, 'text') else str(chunk)\n",
-    "    try:\n",
-    "        entities = entity_extractor.extract_entities(chunk_text)\n",
-    "        all_entities.extend(entities)\n",
-    "    except Exception:\n",
-    "        continue\n",
-    "    \n",
-    "    if i % 20 == 0 or i == len(chunked_documents):\n",
-    "        remaining = len(chunked_documents) - i\n",
-    "        print(f\"  Processed {i}/{len(chunked_documents)} chunks ({len(all_entities)} entities found, {remaining} remaining)\")\n",
-    "\n",
-    "# Filter entities - spaCy returns standard types, map to genomic categories\n",
-    "# Look for variant patterns (rs numbers, c. notation, etc.)\n",
-    "variants = [\n",
-    "    e for e in all_entities \n",
-    "    if (e.text.startswith(\"rs\") or \n",
-    "        \"c.\" in e.text.lower() or \n",
-    "        \"variant\" in e.text.lower() or\n",
-    "        e.label == \"PRODUCT\" and any(kw in e.text.lower() for kw in [\"rs\", \"variant\", \"mutation\"]))\n",
-    "]\n",
-    "\n",
-    "# Look for gene patterns (gene names, protein names)\n",
-    "genes = [\n",
-    "    e for e in all_entities \n",
-    "    if (e.label == \"ORG\" or \n",
-    "        e.label == \"PRODUCT\" or\n",
-    "        any(kw in e.text.lower() for kw in [\"gene\", \"protein\", \"enzyme\", \"receptor\", \"kinase\"]))\n",
-    "]\n",
-    "\n",
-    "# Look for disease patterns\n",
-    "diseases = [\n",
-    "    e for e in all_entities \n",
-    "    if (e.label == \"ORG\" or\n",
-    "        any(kw in e.text.lower() for kw in [\"disease\", \"syndrome\", \"disorder\", \"cancer\", \"hypertension\", \"alzheimer\"]))\n",
-    "]\n",
-    "\n",
-    "print(f\"Extracted {len(variants)} variants, {len(genes)} genes, {len(diseases)} diseases\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Extracting Genomic Relationships\n",
-    "\n",
-    "Extract relationships between variants, genes, and diseases to understand genomic associations.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.semantic_extract import RelationExtractor\n",
-    "\n",
-    "# Using spaCy dependency parsing (similar to Drug Discovery Pipeline)\n",
-    "relation_extractor = RelationExtractor(method=\"dependency\", model=\"en_core_web_sm\")\n",
-    "\n",
-    "all_relationships = []\n",
-    "print(f\"Extracting relationships from {len(chunked_documents)} chunks...\")\n",
-    "\n",
-    "for i, chunk in enumerate(chunked_documents, 1):\n",
-    "    chunk_text = chunk.text if hasattr(chunk, 'text') else str(chunk)\n",
-    "    try:\n",
-    "        relationships = relation_extractor.extract_relations(\n",
-    "            chunk_text,\n",
-    "            entities=all_entities,\n",
-    "            relation_types=[\"associated_with\", \"located_in\", \"causes\", \"increases_risk\", \"affects\", \"linked_to\"]\n",
-    "        )\n",
-    "        all_relationships.extend(relationships)\n",
-    "    except Exception:\n",
-    "        continue\n",
-    "    \n",
-    "    if i % 20 == 0 or i == len(chunked_documents):\n",
-    "        print(f\"  Processed {i}/{len(chunked_documents)} chunks ({len(all_relationships)} relationships found)\")\n",
-    "\n",
-    "print(f\"Extracted {len(all_relationships)} relationships\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Building Temporal Genomic Knowledge Graph\n",
-    "\n",
-    "Construct a temporal knowledge graph from extracted entities and relationships to enable time-aware analysis and variant evolution tracking.\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Conflict Detection and Resolution\n",
-    "\n",
-    "Detect and resolve conflicts in genomic variant data from multiple research sources.\n",
-    "\n",
-    "- **Detection Method**: Entity and relationship conflict detection identifies discrepancies in variant-gene-disease associations across sources\n",
-    "- **Resolution Strategy**: Credibility-weighted resolution prioritizes higher-credibility sources (e.g., Nature Genetics over preprints)\n",
-    "- **Use Case**: Handles conflicting information when multiple sources report different variant associations, disease risks, or gene locations\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.conflicts import ConflictDetector, ConflictResolver\n",
-    "\n",
-    "# Initialize with best strategies for genomic analysis\n",
-    "detector = ConflictDetector()\n",
-    "resolver = ConflictResolver(default_strategy=\"credibility_weighted\")\n",
-    "\n",
-    "# Convert entities to format expected by detector\n",
-    "entities = [\n",
-    "    {\n",
-    "        \"id\": ent.text if hasattr(ent, 'text') else str(ent),\n",
-    "        \"name\": ent.text if hasattr(ent, 'text') else str(ent),\n",
-    "        \"type\": ent.label if hasattr(ent, 'label') else \"ENTITY\",\n",
-    "        \"confidence\": getattr(ent, 'confidence', 1.0),\n",
-    "        \"source\": ent.metadata.get(\"source\", \"unknown\") if hasattr(ent, 'metadata') and ent.metadata else \"unknown\"\n",
-    "    }\n",
-    "    for ent in all_entities if hasattr(ent, 'text') or hasattr(ent, 'label')\n",
-    "]\n",
-    "\n",
-    "# Convert relationships to format expected by detector\n",
-    "relationships = [\n",
-    "    {\n",
-    "        \"id\": f\"{rel.subject.text}_{rel.object.text}_{rel.predicate}\" if hasattr(rel, 'subject') else f\"{i}\",\n",
-    "        \"source_id\": rel.subject.text if hasattr(rel, 'subject') else str(rel.get(\"source\", \"\")),\n",
-    "        \"target_id\": rel.object.text if hasattr(rel, 'object') else str(rel.get(\"target\", \"\")),\n",
-    "        \"type\": rel.predicate if hasattr(rel, 'predicate') else rel.get(\"type\", \"related_to\"),\n",
-    "        \"confidence\": getattr(rel, 'confidence', 1.0),\n",
-    "        \"properties\": rel.metadata if hasattr(rel, 'metadata') else {},\n",
-    "        \"source\": rel.metadata.get(\"source\", \"unknown\") if hasattr(rel, 'metadata') and rel.metadata else \"unknown\"\n",
-    "    }\n",
-    "    for i, rel in enumerate(all_relationships) if hasattr(rel, 'subject') or isinstance(rel, dict)\n",
-    "]\n",
-    "\n",
-    "# Detect both entity and relationship conflicts\n",
-    "print(f\"Detecting conflicts in {len(entities)} entities, {len(relationships)} relationships...\")\n",
-    "\n",
-    "# Detect entity conflicts\n",
-    "entity_conflicts = detector.detect_conflicts(entities)\n",
-    "print(f\"Detected {len(entity_conflicts)} entity conflicts\")\n",
-    "\n",
-    "# Detect relationship conflicts\n",
-    "relationship_conflicts = detector.detect_relationship_conflicts(relationships)\n",
-    "print(f\"Detected {len(relationship_conflicts)} relationship conflicts\")\n",
-    "\n",
-    "# Resolve entity conflicts\n",
-    "if entity_conflicts:\n",
-    "    resolver.resolve_conflicts(entity_conflicts, strategy=\"credibility_weighted\")\n",
-    "    print(f\"Resolved {len(entity_conflicts)} entity conflicts\")\n",
-    "\n",
-    "# Resolve relationship conflicts\n",
-    "if relationship_conflicts:\n",
-    "    resolver.resolve_conflicts(relationship_conflicts, strategy=\"credibility_weighted\")\n",
-    "    print(f\"Resolved {len(relationship_conflicts)} relationship conflicts\")\n",
-    "\n",
-    "# GraphBuilder will use resolve_conflicts=True to apply resolutions automatically\n",
-    "if entity_conflicts or relationship_conflicts:\n",
-    "    print(\"Conflicts resolved. GraphBuilder will use cleaned data.\")\n",
-    "else:\n",
-    "    print(\"No conflicts detected. Data is clean.\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.kg import GraphBuilder\n",
-    "\n",
-    "# Conflicts already detected and resolved in previous cell\n",
-    "# Enable temporal features for genomic variant tracking\n",
-    "graph_builder = GraphBuilder(\n",
-    "    resolve_conflicts=False,  # Conflicts already handled\n",
-    "    enable_temporal=True,\n",
-    "    temporal_granularity=TEMPORAL_GRANULARITY\n",
-    ")\n",
-    "\n",
-    "print(f\"Building temporal knowledge graph from {len(all_entities)} entities, {len(all_relationships)} relationships...\")\n",
-    "kg = graph_builder.build({\n",
-    "    \"entities\": all_entities,\n",
-    "    \"relationships\": all_relationships\n",
-    "})\n",
-    "\n",
-    "entities_count = len(kg.get('entities', []))\n",
-    "relationships_count = len(kg.get('relationships', []))\n",
-    "print(f\"Graph: {entities_count} entities, {relationships_count} relationships\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Analyzing Graph Structure\n",
-    "\n",
-    "Perform comprehensive graph analytics including centrality measures, community detection, and connectivity analysis.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.kg import GraphAnalyzer, CentralityCalculator, CommunityDetector\n",
-    "\n",
-    "graph_analyzer = GraphAnalyzer()\n",
-    "centrality_calc = CentralityCalculator()\n",
-    "community_detector = CommunityDetector()\n",
-    "\n",
-    "analysis = graph_analyzer.analyze_graph(kg)\n",
-    "\n",
-    "degree_centrality = centrality_calc.calculate_degree_centrality(kg)\n",
-    "betweenness_centrality = centrality_calc.calculate_betweenness_centrality(kg)\n",
-    "closeness_centrality = centrality_calc.calculate_closeness_centrality(kg)\n",
-    "\n",
-    "communities = community_detector.detect_communities(kg, method=\"louvain\")\n",
-    "connectivity = graph_analyzer.analyze_connectivity(kg)\n",
-    "\n",
-    "print(f\"Graph analytics:\")\n",
-    "print(f\"  - Communities: {len(communities)}\")\n",
-    "print(f\"  - Connected components: {len(connectivity.get('components', []))}\")\n",
-    "print(f\"  - Graph density: {analysis.get('density', 0):.3f}\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Temporal Graph Queries\n",
-    "\n",
-    "Query the temporal knowledge graph at specific time points, analyze temporal evolution, and detect temporal patterns.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.kg import TemporalGraphQuery\n",
-    "\n",
-    "temporal_query = TemporalGraphQuery(\n",
-    "    enable_temporal_reasoning=True,\n",
-    "    temporal_granularity=TEMPORAL_GRANULARITY\n",
-    ")\n",
-    "\n",
-    "# Query variants at specific time point\n",
-    "query_results = temporal_query.query_at_time(\n",
-    "    kg,\n",
-    "    query=\"Variant\",\n",
-    "    at_time=\"2024-01-01\"\n",
-    ")\n",
-    "\n",
-    "# Analyze graph evolution\n",
-    "evolution = temporal_query.analyze_evolution(kg)\n",
-    "\n",
-    "# Detect temporal patterns\n",
-    "pattern_results = temporal_query.query_temporal_pattern(\n",
-    "    kg,\n",
-    "    pattern=\"sequence\"\n",
-    ")\n",
-    "\n",
-    "print(f\"Temporal query: {query_results.get('num_relationships', 0)} relationships valid at query time\")\n",
-    "print(f\"Evolution analysis: {evolution.get('num_relationships', 0)} relationships tracked\")\n",
-    "print(f\"Temporal patterns detected: {pattern_results.get('num_patterns', 0)}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Pathway Analysis & Reasoning\n",
-    "\n",
-    "Use graph reasoning to find pathways between variants and diseases, and infer biological pathways through logical reasoning.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.reasoning import Reasoner\n",
-    "from semantica.kg import GraphAnalyzer\n",
-    "\n",
-    "reasoner = Reasoner()\n",
-    "graph_analyzer = GraphAnalyzer()\n",
-    "\n",
-    "# Find entities by type\n",
-    "variants = [e for e in kg.get('entities', []) if e.get('type') == 'Variant']\n",
-    "diseases = [e for e in kg.get('entities', []) if e.get('type') == 'Disease']\n",
-    "\n",
-    "print(f\"Found {len(variants)} variants and {len(diseases)} diseases\")\n",
-    "\n",
-    "# Find pathways\n",
-    "pathways = []\n",
-    "for variant in variants[:5]:\n",
-    "    variant_id = variant.get('id') or variant.get('name')\n",
-    "    for disease in diseases[:3]:\n",
-    "        disease_id = disease.get('id') or disease.get('name')\n",
-    "        path = graph_analyzer.connectivity_analyzer.calculate_shortest_paths(\n",
-    "            kg, source=variant_id, target=disease_id\n",
-    "        )\n",
-    "        if path.get('exists'):\n",
-    "            pathways.append({\n",
-    "                'variant': variant_id,\n",
-    "                'disease': disease_id,\n",
-    "                'distance': path.get('distance', -1)\n",
-    "            })\n",
-    "\n",
-    "# Add rule and infer facts\n",
-    "reasoner.add_rule(\"IF Variant associated_with Gene AND Gene causes Disease THEN Variant increases_risk Disease\")\n",
-    "inferred_facts = reasoner.infer_facts(kg)\n",
-    "\n",
-    "print(f\"Pathway analysis: {len(pathways)} variant-disease pathways found\")\n",
-    "print(f\"Inferred facts: {len(inferred_facts)}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Analyzing Disease Associations\n",
-    "\n",
-    "Use graph traversal to find variant-disease associations and calculate association scores.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.kg import GraphAnalyzer\n",
-    "\n",
-    "graph_analyzer = GraphAnalyzer()\n",
-    "\n",
-    "# Find entities by type\n",
-    "variants = [e for e in kg.get('entities', []) if e.get('type') == 'Variant']\n",
-    "diseases = [e for e in kg.get('entities', []) if e.get('type') == 'Disease']\n",
-    "\n",
-    "# Find disease associations\n",
-    "disease_associations = []\n",
-    "for variant in variants[:10]:\n",
-    "    variant_id = variant.get('name') or variant.get('id')\n",
-    "    if not variant_id:\n",
-    "        continue\n",
-    "    for disease in diseases[:5]:\n",
-    "        disease_id = disease.get('name') or disease.get('id')\n",
-    "        if not disease_id:\n",
-    "            continue\n",
-    "        path = graph_analyzer.connectivity_analyzer.calculate_shortest_paths(\n",
-    "            kg, source=variant_id, target=disease_id\n",
-    "        )\n",
-    "        if path.get('exists') and path.get('distance', -1) <= 2:\n",
-    "            disease_associations.append({\n",
-    "                'variant': variant_id,\n",
-    "                'disease': disease_id,\n",
-    "                'path_length': path.get('distance', -1),\n",
-    "                'confidence': variant.get('confidence', 1.0)\n",
-    "            })\n",
-    "\n",
-    "disease_associations.sort(key=lambda x: x['confidence'], reverse=True)\n",
-    "\n",
-    "print(f\"Top disease associations:\")\n",
-    "for i, assoc in enumerate(disease_associations[:5], 1):\n",
-    "    print(f\"{i}. {assoc['variant']} -> {assoc['disease']} (path length: {assoc['path_length']}, confidence: {assoc['confidence']:.3f})\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Visualizing the Temporal Knowledge Graph\n",
-    "\n",
-    "Generate an interactive visualization of the temporal genomic knowledge graph.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.visualization import TemporalVisualizer\n",
-    "\n",
-    "# Visualize temporal dashboard\n",
-    "temporal_viz = TemporalVisualizer()\n",
-    "fig = temporal_viz.visualize_temporal_dashboard(\n",
-    "    kg,\n",
-    "    output=\"interactive\"\n",
-    ")\n",
-    "\n",
-    "# Display the figure\n",
-    "fig.show() if fig else None\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Exporting Results\n",
-    "\n",
-    "Export the temporal knowledge graph to various formats for further analysis or integration with other tools.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from semantica.export import GraphExporter\n",
-    "\n",
-    "exporter = GraphExporter()\n",
-    "exporter.export(kg, output_path=\"genomic_variant_kg.json\", format=\"json\")\n",
-    "exporter.export(kg, output_path=\"genomic_variant_kg.graphml\", format=\"graphml\")\n",
-    "\n",
-    "print(\"Exported knowledge graph to JSON and GraphML formats\")\n"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

cookbook/use_cases/biomedical/data/chunks/chunk_0.txt

@@ -1 +0,0 @@
-Apoptotic signatures allow early and rapid screening of drug-induced liver injury to accelerate drug discovery

cookbook/use_cases/biomedical/data/chunks/chunk_1.txt

@@ -1 +0,0 @@
-SynergyGraph: predicting cell line specific drug combination synergy scores using knowledge graph representation and hypergraph modeling

cookbook/use_cases/biomedical/data/chunks/chunk_10.txt

@@ -1 +0,0 @@
-Unraveling the mechanism of curcumin in coronary slow flow phenomenon through network pharmacology and molecular docking

cookbook/use_cases/biomedical/data/chunks/chunk_11.txt

@@ -1 +0,0 @@
-Penicillium chrysogenum originated chloro-diydropyridyl-oxopropanimidic acid derivative as a potent EPSP synthase-targeted bioherbicide against invasive weed species

cookbook/use_cases/biomedical/data/chunks/chunk_12.txt

@@ -1 +0,0 @@
-Enhancing the anti-cancer potential of resveratrol through cocrystal technology in colorectal cancerous rats

cookbook/use_cases/biomedical/data/chunks/chunk_13.txt

@@ -1 +0,0 @@
-Synthesis, spectral, thermal, and biological characterization of Se(IV) nanocomplexes derived from vitamin E and amino acid mixed ligands as a metal-drug model

cookbook/use_cases/biomedical/data/chunks/chunk_14.txt

@@ -1 +0,0 @@
-BT-11 targets the LANCL2 pathway to attenuate cognitive deficits and hippocampal pathology in Alzheimer’s transgenic rats

cookbook/use_cases/biomedical/data/chunks/chunk_15.txt

@@ -1 +0,0 @@
-Saffron as a natural modulator of reverse cholesterol transport genes in atherosclerotic rabbits, with molecular docking insights

cookbook/use_cases/biomedical/data/chunks/chunk_16.txt

@@ -1 +0,0 @@
-Simultaneous targeting of KRAS and CDK4 synergistically induces durable growth arrest in pancreatic cancer cells

cookbook/use_cases/biomedical/data/chunks/chunk_17.txt

@@ -1 +0,0 @@
-Antiproliferative activity of Moringa oleifera (L.) Lam. and Moringa stenopetala (Bak.) Cufod. leaves extract against selected cancer cells in primary cell culture

cookbook/use_cases/biomedical/data/chunks/chunk_18.txt

@@ -1 +0,0 @@
-Applications of Sombor topological indices and entropy measures for QSPR modeling of anticancer drugs: a Python-based methodology

cookbook/use_cases/biomedical/data/chunks/chunk_19.txt

@@ -1 +0,0 @@
-In-silico studies, synthesis, and pharmacological screening of novel multitarget diphenylpyrazole scaffold as EGFR/BRAF and cyclooxygenase-2 inhibitors