diff --git a/docs/reference/change_management.md b/docs/reference/change_management.md
index 7f74124f..0981f4ba 100644
--- a/docs/reference/change_management.md
+++ b/docs/reference/change_management.md
@@ -1,21 +1,38 @@
---
title: "Change Management Module"
-description: "Enterprise-grade version control, SHA-256 checksums, and audit trails for knowledge graphs."
+description: "Version control, SHA-256 checksums, diff analysis, rollback, and audit trails for knowledge graphs and ontologies."
icon: "clock-rotate-left"
---
-> Version control and audit trails for knowledge graphs with data integrity verification.
+> Enterprise-grade version control and audit trails for knowledge graphs with data integrity verification.
---
## Overview
-The **Change Management Module** provides version control for knowledge graphs — SHA-256 checksums, full snapshot history, diff analysis, and rollback protection.
+The **Change Management Module** provides versioning for knowledge graphs — SHA-256 checksums, full snapshot history, diff analysis, rollback protection, and compliance-grade audit trails (HIPAA, SOX, FDA 21 CFR Part 11).
+
+
+
+ Snapshot, diff, rollback, and audit trail for knowledge graphs.
+
+
+ Version control for OWL ontologies with diff and migration support.
+
+
+ Pluggable storage — in-memory for tests, SQLite for production.
+
+
+ SHA-256 checksums to detect unauthorized modifications.
+
+
---
## TemporalVersionManager
+Version control for knowledge graphs:
+
```python
from semantica.change_management import TemporalVersionManager
@@ -29,7 +46,8 @@ snapshot_id = manager.create_snapshot(
message="Initial knowledge graph"
)
-print(f"Snapshot {snapshot_id} — checksum: {manager.get_checksum(snapshot_id)}")
+print(f"Snapshot {snapshot_id}")
+print(f"Checksum: {manager.get_checksum(snapshot_id)}")
```
---
@@ -42,10 +60,10 @@ versions = manager.list_versions()
for v in versions:
print(f"{v.version} — {v.author} — {v.created_at} — {v.checksum[:8]}...")
-# Get a specific version
+# Retrieve a specific version
kg_v1 = manager.get_version("v1.0")
-# Rollback to a previous version
+# Rollback
manager.rollback(target_version="v1.0", allow_data_loss=False)
```
@@ -56,8 +74,8 @@ manager.rollback(target_version="v1.0", allow_data_loss=False)
```python
diff = manager.diff("v1.0", "v2.0")
-print(f"Added nodes: {len(diff.added_nodes)}")
-print(f"Removed nodes: {len(diff.removed_nodes)}")
+print(f"Added nodes: {len(diff.added_nodes)}")
+print(f"Removed nodes: {len(diff.removed_nodes)}")
print(f"Modified edges: {len(diff.modified_edges)}")
for change in diff.changes:
@@ -66,17 +84,70 @@ for change in diff.changes:
---
-## Integrity Verification
+## OntologyVersionManager
+
+Version control for OWL ontologies:
```python
-# Verify current graph against stored checksum
-is_valid = manager.verify_integrity(kg, version="v2.0")
+from semantica.change_management import OntologyVersionManager, OntologyVersion
-if not is_valid:
- print("Warning: Graph has been modified since v2.0 was created")
+manager = OntologyVersionManager()
+
+# Save a version
+version: OntologyVersion = manager.save_version(
+ ontology=ontology,
+ version="1.2.0",
+ author="ontology-team",
+ message="Added FHIR alignment mappings"
+)
+
+# Diff two ontology versions
+diff = manager.diff("1.1.0", "1.2.0")
+for change in diff.changes:
+ print(f"[{change.type}] {change.class_name}: {change.description}")
```
-SHA-256 checksums are computed over the serialized graph to detect unauthorized modifications.
+---
+
+## VersionStorage
+
+Pluggable storage backends:
+
+```python
+from semantica.change_management import (
+ VersionStorage,
+ InMemoryVersionStorage,
+ SQLiteVersionStorage,
+)
+
+# In-memory (tests and development)
+storage = InMemoryVersionStorage()
+
+# SQLite (production — persistent across restarts)
+storage = SQLiteVersionStorage(db_path="versions.db")
+
+# Pass to a version manager
+manager = TemporalVersionManager(storage=storage)
+```
+
+---
+
+## Integrity Verification
+
+SHA-256 checksums detect any unauthorized modification to the graph:
+
+```python
+from semantica.change_management import compute_checksum, verify_checksum
+
+# Compute checksum for a graph
+checksum = compute_checksum(kg)
+
+# Verify graph against stored checksum
+is_valid = verify_checksum(kg, expected_checksum=checksum)
+
+if not is_valid:
+ print("Warning: Graph has been modified since the checksum was recorded")
+```
---
@@ -89,12 +160,31 @@ for entry in trail:
print(f"{entry.timestamp} — {entry.author}: {entry.action} — {entry.description}")
# Export audit trail
-manager.export_audit_trail("audit.csv", format="csv")
+manager.export_audit_trail("audit.csv", format="csv")
manager.export_audit_trail("audit.json", format="json")
```
---
+## ChangeLogEntry
+
+Every version snapshot includes a structured `ChangeLogEntry`:
+
+```python
+from semantica.change_management import ChangeLogEntry
+
+entry: ChangeLogEntry = manager.get_log_entry(snapshot_id)
+
+print(entry.version)
+print(entry.author)
+print(entry.message)
+print(entry.checksum)
+print(entry.created_at)
+print(entry.changes) # list of individual change records
+```
+
+---
+
## See Also
diff --git a/docs/reference/conflicts.md b/docs/reference/conflicts.md
index da38f6b6..5185ca57 100644
--- a/docs/reference/conflicts.md
+++ b/docs/reference/conflicts.md
@@ -1,34 +1,46 @@
---
title: "Conflicts Module"
-description: "Multi-source conflict detection and resolution — value, type, temporal, and logical conflicts."
+description: "Multi-source conflict detection and resolution — value, type, temporal, and logical conflicts with investigation guides."
icon: "triangle-exclamation"
---
-> Comprehensive conflict detection and resolution for data discrepancies across multiple sources.
+> Detect and resolve contradictions across multiple data sources before they silently corrupt your knowledge graph.
---
## Overview
-When multiple sources disagree on the same fact, the **Conflicts Module** detects and resolves the conflict rather than silently picking one value.
+When multiple sources disagree on the same fact, the **Conflicts Module** detects and resolves the conflict rather than silently picking one value. It supports five detection types, seven resolution strategies, and generates investigation guides for manual review.
-Detection types: **value**, **type**, **temporal**, and **logical** conflicts.
+
+
+ Value, type, temporal, logical, and relationship conflict detection.
+
+
+ 7 resolution strategies including voting, credibility-weighted, and temporal.
+
+
+ Pattern analysis, severity grouping, and trend identification.
+
+
+ Auto-generate step-by-step investigation checklists for human review.
+
+
---
## ConflictDetector
```python
-from semantica.conflicts import ConflictDetector
+from semantica.conflicts import ConflictDetector, ConflictType
detector = ConflictDetector()
conflicts = detector.detect_conflicts(kg)
for conflict in conflicts:
- print(f"Conflict on '{conflict.entity}' — {conflict.attribute}")
- print(f" Source A: {conflict.value_a} (from {conflict.source_a})")
- print(f" Source B: {conflict.value_b} (from {conflict.source_b})")
- print(f" Type: {conflict.conflict_type}")
+ print(f"[{conflict.conflict_type}] '{conflict.entity}' — {conflict.attribute}")
+ print(f" Sources: {conflict.sources}")
+ print(f" Severity: {conflict.severity:.2f}")
```
---
@@ -36,69 +48,147 @@ for conflict in conflicts:
## Detection Types
```python
+# Detect all types (default)
+conflicts = detector.detect_conflicts(kg)
+
# Detect specific types only
-conflicts = detector.detect_conflicts(
- kg,
- types=["value", "temporal"] # value | type | temporal | logical
-)
+conflicts = detector.detect_value_conflicts(entities, "name")
+conflicts = detector.detect_type_conflicts(entities)
+conflicts = detector.detect_relationship_conflicts(kg)
```
-**Value conflicts** — same entity, same attribute, different values across sources.
-**Type conflicts** — same entity classified as different types in different sources.
-**Temporal conflicts** — overlapping validity windows with contradictory facts.
-**Logical conflicts** — facts that violate ontology axioms or constraints.
+| Type | What It Detects |
+|------|-----------------|
+| `VALUE` | Same entity, same attribute, different values across sources |
+| `TYPE` | Same entity classified as different types in different sources |
+| `TEMPORAL` | Overlapping validity windows with contradictory facts |
+| `LOGICAL` | Facts that violate ontology axioms or SHACL constraints |
+| `RELATIONSHIP` | Inconsistent relationship properties across sources |
---
-## Conflict Resolution
+## ConflictResolver
```python
-from semantica.conflicts import ConflictResolver
+from semantica.conflicts import ConflictResolver, ResolutionStrategy
resolver = ConflictResolver()
-resolved_kg = resolver.resolve(
- kg,
- conflicts,
- strategy="most_recent" # see strategies below
-)
+results = resolver.resolve_conflicts(conflicts, strategy=ResolutionStrategy.VOTING)
+
+for result in results:
+ print(f"Resolved '{result.attribute}' → {result.resolved_value}")
+ print(f" Strategy used: {result.strategy}")
```
Resolution strategies:
-| Strategy | Description |
-|----------|-------------|
-| `most_recent` | Prefer the most recently updated fact |
-| `most_reliable` | Prefer the source with the highest reliability score |
-| `majority_vote` | Use the value agreed upon by most sources |
-| `highest_confidence` | Prefer the fact with the highest confidence score |
-| `flag_for_review` | Mark conflicting facts for manual resolution |
+| Strategy | Enum | Description |
+|----------|------|-------------|
+| `voting` | `ResolutionStrategy.VOTING` | Most common value wins (majority vote) |
+| `credibility_weighted` | `ResolutionStrategy.CREDIBILITY_WEIGHTED` | Weighted average by source credibility score |
+| `most_recent` | `ResolutionStrategy.MOST_RECENT` | Prefer the most recently updated fact |
+| `first_seen` | `ResolutionStrategy.FIRST_SEEN` | Prefer the first observed value |
+| `highest_confidence` | `ResolutionStrategy.HIGHEST_CONFIDENCE` | Prefer the fact with the highest confidence score |
+| `manual_review` | `ResolutionStrategy.MANUAL_REVIEW` | Flag for human review |
+| `expert_review` | `ResolutionStrategy.EXPERT_REVIEW` | Escalate to domain expert |
----
-
-## Source Reliability Scoring
+Use the convenience aliases for shorter code:
```python
-resolver = ConflictResolver(
- source_reliability={
- "pubmed": 0.95,
- "wikipedia": 0.80,
- "user_input": 0.60
- }
-)
-resolved_kg = resolver.resolve(kg, conflicts, strategy="most_reliable")
+from semantica.conflicts import voting, credibility_weighted, most_recent, highest_confidence
+
+results = resolver.resolve_conflicts(conflicts, strategy=voting)
```
---
-## Conflict Report
+## Source Credibility Scoring
```python
-report = detector.generate_report(conflicts)
-print(f"Total conflicts: {report.total}")
-print(f"By type: {report.by_type}")
-print(f"Most conflicted entities: {report.top_entities[:5]}")
+from semantica.conflicts import SourceTracker
-report.export("conflicts.json")
+tracker = SourceTracker()
+tracker.set_credibility("pubmed", 0.95)
+tracker.set_credibility("wikipedia", 0.80)
+tracker.set_credibility("user_input", 0.60)
+
+# Pass to resolver for credibility-weighted resolution
+resolver = ConflictResolver(source_tracker=tracker)
+results = resolver.resolve_conflicts(
+ conflicts, strategy=ResolutionStrategy.CREDIBILITY_WEIGHTED
+)
+```
+
+`SourceTracker` also tracks property-to-source mapping, entity source references, and builds traceability chains:
+
+```python
+from semantica.conflicts import SourceTracker, SourceReference, PropertySource
+
+tracker = SourceTracker()
+tracker.track_entity_source("apple_inc", "crunchbase")
+tracker.track_property_source("apple_inc", "revenue", "annual_report_2023")
+
+chain = tracker.get_traceability_chain("apple_inc")
+```
+
+---
+
+## ConflictAnalyzer
+
+Identify patterns across conflict sets:
+
+```python
+from semantica.conflicts import ConflictAnalyzer, ConflictPattern
+
+analyzer = ConflictAnalyzer()
+
+# Detect recurring patterns
+patterns = analyzer.identify_patterns(conflicts)
+for pattern in patterns:
+ print(f"Pattern: {pattern.type} — {pattern.frequency} occurrences")
+
+# Group by severity
+by_severity = analyzer.group_by_severity(conflicts)
+print(f"Critical: {len(by_severity['critical'])}")
+print(f"High: {len(by_severity['high'])}")
+
+# Trend analysis
+trends = analyzer.analyze_trends(conflicts, time_window="30d")
+```
+
+---
+
+## InvestigationGuideGenerator
+
+Auto-generate human-readable investigation guides for conflicts that can't be automatically resolved:
+
+```python
+from semantica.conflicts import InvestigationGuideGenerator, InvestigationGuide
+
+generator = InvestigationGuideGenerator()
+guide: InvestigationGuide = generator.generate(conflict)
+
+print(guide.title)
+print(guide.context)
+for step in guide.steps:
+ print(f" [{step.order}] {step.description}")
+ print(f" Check: {step.check}")
+```
+
+---
+
+## Convenience Functions
+
+```python
+from semantica.conflicts import (
+ detect_conflicts, resolve_conflicts, analyze_conflicts,
+ track_sources, generate_investigation_guide
+)
+
+conflicts = detect_conflicts(entities, method="value")
+resolved = resolve_conflicts(conflicts, strategy="voting")
+analysis = analyze_conflicts(conflicts, method="pattern")
+guide = generate_investigation_guide(conflicts[0])
```
---
@@ -110,7 +200,7 @@ report.export("conflicts.json")
Resolve duplicate entities before conflict detection.
- Logical conflicts use ontology axioms.
+ Logical conflicts use SHACL shapes and ontology axioms.
Track which source each conflicting fact came from.
diff --git a/docs/reference/deduplication.md b/docs/reference/deduplication.md
index 05923a41..57264462 100644
--- a/docs/reference/deduplication.md
+++ b/docs/reference/deduplication.md
@@ -1,93 +1,195 @@
---
title: "Deduplication Module"
-description: "Entity deduplication v1/v2, similarity scoring, and merging — up to 7x faster with v2 strategies."
+description: "Entity deduplication v1/v2 — similarity scoring, blocking, merging, and cluster-based batch processing."
icon: "copy"
---
-> Advanced entity deduplication and resolution for a clean, single-source-of-truth knowledge graph.
+> Identify and merge duplicate entities across sources for a clean, single-source-of-truth knowledge graph.
---
## Overview
-The **Deduplication Module** identifies and merges duplicate entities across sources using similarity scoring and blocking strategies.
+The **Deduplication Module** detects and merges duplicate entities using similarity scoring, blocking strategies, and configurable merge policies. **v2 strategies** (`blocking_v2`, `hybrid_v2`, `semantic_v2`) are up to **7x faster** than v1 and support fine-grained result control.
-**v2 strategies** (`blocking_v2`, `hybrid_v2`, `semantic_v2`) are up to **7x faster** than v1 and support fine-grained result control via `max_results`, `top_k_per_entity`, `min_similarity`, and `sort_by`.
-
----
-
-## EntityResolver
-
-```python
-from semantica.deduplication import EntityResolver
-
-resolver = EntityResolver()
-merged_entities = resolver.resolve(entities, strategy="semantic_v2")
-```
-
-Strategies:
-
-| Strategy | Method | Speed | Accuracy |
-|----------|--------|-------|----------|
-| `jaro_winkler` | String similarity (v1) | Fast | Medium |
-| `blocking_v2` | Blocking + Jaro-Winkler (v2) | Very fast | Medium |
-| `hybrid_v2` | Blocking + semantic + string (v2) | Fast | High |
-| `semantic_v2` | Embedding similarity (v2) | Medium | Highest |
+
+
+ Pairwise and batch duplicate detection with similarity scoring.
+
+
+ Merge duplicate groups using configurable strategies with provenance preservation.
+
+
+ Multi-factor similarity: Levenshtein, Jaro-Winkler, cosine, Jaccard, embedding.
+
+
+ Union-Find and hierarchical clustering for batch deduplication at scale.
+
+
---
## DuplicateDetector
-Fine-grained control over duplicate detection results.
+The primary class for finding duplicate entity pairs:
```python
from semantica.deduplication import DuplicateDetector
-detector = DuplicateDetector()
-duplicates = detector.find_duplicates(
- entities,
- strategy="semantic_v2",
- min_similarity=0.85, # minimum score to consider a duplicate
- top_k_per_entity=3, # max candidates per entity
- max_results=100, # total result cap
- sort_by="similarity" # "similarity" | "entity_id" | "cluster_size"
-)
+detector = DuplicateDetector(similarity_threshold=0.85)
+duplicates = detector.detect_duplicates(entities)
for dup in duplicates:
- print(f"{dup['entity_a']} ≈ {dup['entity_b']} ({dup['similarity']:.2f})")
+ print(f"{dup.entity_a} ≈ {dup.entity_b} ({dup.similarity:.2f})")
```
+Fine-grained control:
+
+```python
+duplicates = detector.detect_duplicates(
+ entities,
+ strategy="semantic_v2", # see strategies below
+ min_similarity=0.85, # minimum score to consider a match
+ top_k_per_entity=3, # max candidates per entity
+ max_results=100, # total result cap
+ sort_by="similarity", # "similarity" | "entity_id" | "cluster_size"
+)
+```
+
+Detection strategies:
+
+| Strategy | Algorithm | Speed | Accuracy |
+|----------|-----------|-------|----------|
+| `jaro_winkler` | String similarity (v1) | Fast | Medium |
+| `blocking_v2` | Blocking + Jaro-Winkler (v2) | Very fast | Medium |
+| `hybrid_v2` | Blocking + semantic + string (v2) | Fast | High |
+| `semantic_v2` | Embedding similarity (v2) | Medium | Highest |
+
- **v0.5.0 fix:** `ConflictDetector` no longer produces duplicate definition errors when the same entity appears in multiple sources with identical definitions.
+ **v0.5.0 fix:** `DuplicateDetector` no longer produces duplicate definition errors when the same entity appears in multiple sources with identical definitions.
---
-## Merging Entities
+## EntityMerger
+
+Merges detected duplicate groups into canonical entities:
```python
from semantica.deduplication import EntityMerger
merger = EntityMerger()
-merged_kg = merger.merge(
- kg,
- duplicates,
- strategy="union", # "union" | "intersection" | "most_recent" | "most_confident"
- preserve_provenance=True # keep source references after merge
+merged_entities = merger.merge_duplicates(
+ entities,
+ strategy="keep_most_complete", # see strategies below
+ preserve_provenance=True, # keep source references after merge
)
```
+Merge strategies:
+
+| Strategy | Behavior |
+|----------|----------|
+| `keep_first` | Keep the first entity in each duplicate group |
+| `keep_last` | Keep the most recently seen entity |
+| `keep_most_complete` | Keep the entity with the most non-null properties |
+| `union` | Merge all properties — non-conflicting fields combined |
+| `voting` | Most common property value wins |
+
+```python
+# Fine-grained merge with custom property rules
+from semantica.deduplication import PropertyMergeRule
+
+merger = EntityMerger(
+ property_rules={
+ "name": PropertyMergeRule.KEEP_FIRST,
+ "aliases": PropertyMergeRule.UNION,
+ "description": PropertyMergeRule.KEEP_LONGEST,
+ }
+)
+```
+
+---
+
+## SimilarityCalculator
+
+Compute multi-factor similarity between entity pairs:
+
+```python
+from semantica.deduplication import SimilarityCalculator
+
+calc = SimilarityCalculator()
+
+score = calc.calculate_similarity(entity_a, entity_b)
+# → SimilarityResult(score=0.91, components={...})
+
+print(score.score) # overall score 0.0–1.0
+print(score.components["label"]) # label similarity
+print(score.components["embedding"]) # semantic similarity
+print(score.components["property"]) # property overlap
+```
+
+Individual metrics:
+
+```python
+from semantica.deduplication import SimilarityCalculator
+
+calc = SimilarityCalculator()
+
+# String metrics
+lev = calc.levenshtein("Apple Inc.", "Apple Inc")
+jaro = calc.jaro_winkler("Steve Jobs", "Steven Jobs")
+cos = calc.cosine_similarity(embedding_a, embedding_b)
+jacc = calc.jaccard({"founded", "tech"}, {"founded", "technology"})
+```
+
+---
+
+## ClusterBuilder
+
+Build clusters from detected duplicate groups for large-scale batch processing:
+
+```python
+from semantica.deduplication import ClusterBuilder
+
+builder = ClusterBuilder(algorithm="union_find") # or "hierarchical"
+result = builder.build_clusters(entities, similarity_threshold=0.85)
+
+print(f"Clusters: {len(result.clusters)}")
+for cluster in result.clusters:
+ print(f" [{cluster.id}] {cluster.members} — quality: {cluster.cohesion:.2f}")
+```
+
+---
+
+## Convenience Functions
+
+```python
+from semantica.deduplication import detect_duplicates, merge_entities, calculate_similarity
+
+# Quick detection
+duplicates = detect_duplicates(entities, method="semantic_v2", similarity_threshold=0.85)
+
+# Quick merge
+merged = merge_entities(entities, duplicates, method="keep_most_complete")
+
+# Quick similarity
+score = calculate_similarity(entity_a, entity_b, method="hybrid_v2")
+```
+
---
## Blocking Strategies
-Blocking reduces the comparison search space for large entity sets.
+Blocking reduces the O(n²) pairwise comparison to a manageable subset:
```python
-resolver = EntityResolver(
+from semantica.deduplication import DuplicateDetector
+
+detector = DuplicateDetector(
blocking_strategy="token", # "token" | "phonetic" | "ngram"
blocking_threshold=0.6,
- comparison_strategy="semantic_v2"
+ similarity_threshold=0.85
)
```
@@ -96,11 +198,15 @@ resolver = EntityResolver(
## Custom Similarity Functions
```python
-def custom_similarity(entity_a, entity_b):
- # Domain-specific matching logic
+from semantica.deduplication import MethodRegistry, method_registry
+
+def domain_similarity(entity_a, entity_b):
+ # e.g., match drug names by active compound
return score # 0.0 to 1.0
-resolver = EntityResolver(similarity_fn=custom_similarity)
+method_registry.register("similarity", "drug_name", domain_similarity)
+
+detector = DuplicateDetector(similarity_method="drug_name")
```
---
diff --git a/docs/reference/embeddings.md b/docs/reference/embeddings.md
index b5098af5..9df0ba09 100644
--- a/docs/reference/embeddings.md
+++ b/docs/reference/embeddings.md
@@ -1,21 +1,44 @@
---
title: "Embeddings Module"
-description: "Text embedding generation with Sentence-Transformers, FastEmbed, OpenAI, and BGE model support."
+description: "Text and graph embedding generation — Sentence-Transformers, FastEmbed, OpenAI, BGE, LlamaStore, with pooling strategies and graph embedding managers."
icon: "vector-square"
---
-> Unified interface for generating vector representations of text.
+> Unified interface for generating vector representations of text, nodes, and graphs.
---
## Overview
-The **Embeddings Module** converts text into dense vectors for semantic search, entity resolution, and GraphRAG retrieval. It abstracts multiple providers behind a single API.
+The **Embeddings Module** converts text and graph structures into dense vectors for semantic search, entity resolution, and GraphRAG retrieval. It abstracts multiple providers behind a single API and supports five pooling strategies.
+
+
+
+ Main entry point — provider-agnostic text embedding generation.
+
+
+ Text-specific embedding with batching and caching.
+
+
+ Node and subgraph embedding for structural similarity.
+
+
+ Embedding lifecycle for vector store integration.
+
+
+ Pluggable backends: OpenAI, BGE, FastEmbed, LlamaStore.
+
+
+ Mean, Max, CLS, Attention, Hierarchical pooling.
+
+
---
## EmbeddingGenerator
+Main entry point — handles provider selection and batching:
+
```python
from semantica.embeddings import EmbeddingGenerator
@@ -28,11 +51,15 @@ generator = EmbeddingGenerator(model="BAAI/bge-large-en-v1.5")
embeddings = generator.generate(texts)
# OpenAI
+import os
generator = EmbeddingGenerator(
model="openai",
model_name="text-embedding-3-small",
api_key=os.getenv("OPENAI_API_KEY")
)
+
+# FastEmbed (fast CPU-optimized)
+generator = EmbeddingGenerator(model="fastembed")
```
---
@@ -47,53 +74,199 @@ generator = EmbeddingGenerator(
| `fastembed` | `BAAI/bge-small-en-v1.5` | 384 | Fast, CPU-optimized |
| `openai` | `text-embedding-3-small` | 1536 | OpenAI API |
| `openai` | `text-embedding-3-large` | 3072 | OpenAI API, highest quality |
+| `llama` | any Ollama model | varies | Local inference |
+
+---
+
+## TextEmbedder
+
+Specialized for text with automatic batching:
+
+```python
+from semantica.embeddings import TextEmbedder
+
+embedder = TextEmbedder(model="sentence-transformers", cache_dir=".emb_cache")
+
+# Single text
+embedding = embedder.embed("Hello world")
+
+# Batch
+embeddings = embedder.embed_batch(
+ ["Text 1", "Text 2", ..., "Text 10000"],
+ batch_size=128,
+ show_progress=True
+)
+```
+
+---
+
+## Provider Stores
+
+Each provider implements the `ProviderStore` interface and can be used independently:
+
+```python
+from semantica.embeddings import (
+ OpenAIStore, BGEStore, FastEmbedStore, LlamaStore,
+ ProviderStoreFactory
+)
+
+# OpenAI
+store = OpenAIStore(api_key=os.getenv("OPENAI_API_KEY"), model="text-embedding-3-small")
+embedding = store.embed("Hello world")
+
+# BGE (Sentence-Transformers wrapper)
+store = BGEStore(model="BAAI/bge-large-en-v1.5")
+embedding = store.embed("Hello world")
+
+# FastEmbed
+store = FastEmbedStore(model="BAAI/bge-small-en-v1.5")
+embedding = store.embed("Hello world")
+
+# LlamaStore (Ollama local)
+store = LlamaStore(model="llama3.2", base_url="http://localhost:11434")
+embedding = store.embed("Hello world")
+
+# Auto-select from config
+store = ProviderStoreFactory.create(provider="openai", model="text-embedding-3-small")
+```
+
+---
+
+## Pooling Strategies
+
+Control how token-level embeddings are aggregated into a single vector:
+
+```python
+from semantica.embeddings import (
+ MeanPooling, MaxPooling, CLSPooling,
+ AttentionPooling, HierarchicalPooling, PoolingStrategyFactory
+)
+
+# Mean pooling (default — best for most tasks)
+pooler = MeanPooling()
+pooled = pooler.pool(token_embeddings)
+
+# Max pooling (captures strongest features)
+pooler = MaxPooling()
+
+# CLS token pooling (first token — good for classification)
+pooler = CLSPooling()
+
+# Attention-weighted pooling
+pooler = AttentionPooling()
+
+# Hierarchical: chunk-level → global mean (best for long documents)
+pooler = HierarchicalPooling(chunk_size=512)
+
+# Create from config
+pooler = PoolingStrategyFactory.create(strategy="mean")
+```
+
+---
+
+## GraphEmbeddingManager
+
+Embed graph nodes and subgraphs for structural similarity and GraphRAG:
+
+```python
+from semantica.embeddings import GraphEmbeddingManager
+
+manager = GraphEmbeddingManager(
+ text_embedder=TextEmbedder(model="sentence-transformers"),
+ graph_store=graph_store
+)
+
+# Embed all nodes
+node_embeddings = manager.embed_nodes(kg)
+
+# Embed a specific subgraph (for GraphRAG context)
+subgraph_embedding = manager.embed_subgraph(
+ kg, center_node="Apple Inc.", hops=2
+)
+
+# Find similar nodes
+similar = manager.find_similar_nodes("apple_inc", top_k=5)
+```
+
+---
+
+## VectorEmbeddingManager
+
+Manages the full embedding lifecycle for vector store integration:
+
+```python
+from semantica.embeddings import VectorEmbeddingManager
+from semantica.vector_store import VectorStore
+
+vector_store = VectorStore(backend="faiss", dimension=768)
+
+manager = VectorEmbeddingManager(
+ embedder=TextEmbedder(model="sentence-transformers"),
+ vector_store=vector_store
+)
+
+# Embed and store documents
+ids = manager.embed_and_store(documents, metadata=metadata_list)
+
+# Search
+results = manager.search("machine learning algorithms", top_k=10)
+```
---
## Similarity Computation
```python
-# Cosine similarity between two embeddings
-similarity = generator.similarity(embeddings[0], embeddings[1])
-print(f"Similarity: {similarity:.3f}") # 0.0 to 1.0
+from semantica.embeddings import calculate_similarity
-# Find top-k most similar texts
-query_embedding = generator.generate(["machine learning"])[0]
-scores = generator.rank(query_embedding, embeddings)
+score = calculate_similarity(embedding_a, embedding_b, method="cosine")
+# → 0.0 to 1.0
+
+# Euclidean distance converted to similarity
+score = calculate_similarity(embedding_a, embedding_b, method="euclidean")
```
---
-## Batch Generation
+## Convenience Functions
```python
-# Efficient batched generation
-texts = ["Text 1", "Text 2", ..., "Text 10000"]
-embeddings = generator.generate_batch(texts, batch_size=128, show_progress=True)
-```
-
----
-
-## Caching
-
-```python
-generator = EmbeddingGenerator(
- model="sentence-transformers",
- cache_dir=".embeddings_cache", # persist embeddings to disk
- cache_ttl=3600 # cache TTL in seconds
+from semantica.embeddings import (
+ embed_text, generate_embeddings, calculate_similarity,
+ pool_embeddings, check_available_providers
)
-```
-The distance intelligence feature (v0.5.0) uses embedding cache optimization to avoid recomputing embeddings for large distance matrix calculations.
+# Single text
+emb = embed_text("Hello world", method="sentence_transformers")
+
+# Batch
+embs = generate_embeddings(texts, method="openai")
+
+# Check what's installed
+providers = check_available_providers()
+# → {"sentence_transformers": True, "fastembed": True, "openai": False}
+```
---
## GPU Acceleration
```python
-generator = EmbeddingGenerator(
+generator = EmbeddingGenerator(model="sentence-transformers", device="cuda")
+# device: "cpu" | "cuda" | "mps"
+```
+
+---
+
+## Caching
+
+Embedding cache reuse is used by Distance Intelligence (v0.5.0) to avoid recomputing embeddings for large distance matrix calculations:
+
+```python
+embedder = TextEmbedder(
model="sentence-transformers",
- device="cuda" # "cpu" | "cuda" | "mps"
+ cache_dir=".embeddings_cache",
+ cache_ttl=3600 # TTL in seconds
)
```
@@ -109,7 +282,7 @@ generator = EmbeddingGenerator(
Chunk text before embedding.
- Distance Intelligence uses embeddings for semantic neighborhoods.
+ Distance Intelligence uses graph embeddings.
Semantic deduplication uses embeddings for entity resolution.
diff --git a/docs/reference/export.md b/docs/reference/export.md
index c84c0b6c..90d0d56a 100644
--- a/docs/reference/export.md
+++ b/docs/reference/export.md
@@ -1,10 +1,37 @@
---
title: "Export Module"
-description: "Export knowledge graphs to RDF (Turtle, JSON-LD, N-Triples, XML), Parquet, ArangoDB AQL, OWL, and CSV."
+description: "Export knowledge graphs to RDF, Parquet, LPG, ArangoDB AQL, CSV, GraphML, OWL, JSON-LD, and vector formats."
icon: "file-export"
---
-> Export knowledge graphs and data to multiple formats with W3C-compliant serialization.
+> Export knowledge graphs and embeddings to 12+ formats with W3C-compliant serialization.
+
+---
+
+## Overview
+
+The **Export Module** serializes knowledge graphs to every downstream format — semantic web standards, analytics pipelines, graph databases, and vector stores.
+
+
+
+ Turtle, JSON-LD, N-Triples, RDF/XML with namespace management.
+
+
+ Columnar storage for Spark, BigQuery, Databricks, Snowflake.
+
+
+ Labeled Property Graph — Cypher CREATE statements for Neo4j and Memgraph.
+
+
+ ArangoDB AQL INSERT statements for multi-model graph databases.
+
+
+ GraphML, GEXF, DOT formats for visualization tools like Gephi.
+
+
+ OWL 2.0 ontology export in Turtle, XML, and JSON-LD.
+
+
---
@@ -20,18 +47,106 @@ rdf = exporter.export_to_rdf(graph, format="turtle")
exporter.export_to_file(graph, "output.ttl", format="turtle")
# JSON-LD
-rdf = exporter.export_to_rdf(graph, format="json-ld")
+exporter.export_to_file(graph, "output.jsonld", format="json-ld")
# N-Triples
-rdf = exporter.export_to_rdf(graph, format="nt")
+exporter.export_to_file(graph, "output.nt", format="nt")
# RDF/XML
-rdf = exporter.export_to_rdf(graph, format="xml")
+exporter.export_to_file(graph, "output.xml", format="xml")
+```
+
+Namespace management:
+
+```python
+from semantica.export import NamespaceManager, RDFSerializer
+
+ns_manager = NamespaceManager()
+ns_manager.register("ex", "http://example.org/")
+ns_manager.register("schema", "https://schema.org/")
+
+exporter = RDFExporter(namespace_manager=ns_manager)
```
---
-## OWL Exporter
+## ParquetExporter
+
+For Spark, BigQuery, Databricks, and Snowflake analytics pipelines:
+
+```python
+from semantica.export import ParquetExporter
+
+exporter = ParquetExporter(compression="snappy") # snappy | gzip | brotli | zstd | lz4
+
+# Export nodes and edges separately
+exporter.export_nodes(graph, "nodes.parquet")
+exporter.export_edges(graph, "edges.parquet")
+
+# Export full graph (partitioned)
+exporter.export(graph, output_dir="graph_parquet/", partition_by="node_type")
+```
+
+Parquet schema is explicitly typed with PyArrow for clean Spark/BigQuery ingestion.
+
+---
+
+## LPGExporter
+
+Labeled Property Graph — Cypher CREATE statements for Neo4j and Memgraph:
+
+```python
+from semantica.export import LPGExporter
+
+exporter = LPGExporter()
+
+# Generate Cypher
+cypher = exporter.to_cypher(graph)
+exporter.export(graph, "import.cypher", format="cypher")
+
+# Generate MERGE statements (idempotent)
+cypher_merge = exporter.to_cypher(graph, use_merge=True)
+```
+
+---
+
+## ArangoAQLExporter
+
+ArangoDB AQL INSERT statements for vertex and edge collections:
+
+```python
+from semantica.export import ArangoAQLExporter
+
+exporter = ArangoAQLExporter(
+ vertex_collection="entities",
+ edge_collection="relationships"
+)
+
+aql = exporter.export(graph) # returns AQL string
+exporter.export_to_file(graph, "import.aql")
+```
+
+---
+
+## GraphExporter
+
+GraphML, GEXF, and DOT formats for visualization tools:
+
+```python
+from semantica.export import GraphExporter
+
+exporter = GraphExporter()
+
+exporter.export(graph, "graph.graphml", format="graphml") # Gephi, yEd
+exporter.export(graph, "graph.gexf", format="gexf") # Gephi streaming
+exporter.export(graph, "graph.dot", format="dot") # Graphviz
+```
+
+---
+
+## OWLExporter
+
+OWL 2.0 ontology export:
```python
from semantica.export import OWLExporter
@@ -39,62 +154,105 @@ from semantica.export import OWLExporter
exporter = OWLExporter()
exporter.export(ontology, path="ontology.ttl", format="turtle")
exporter.export(ontology, path="ontology.owl", format="xml")
+exporter.export(ontology, path="ontology.json", format="json-ld")
```
---
-## Parquet Exporter
-
-For Spark, BigQuery, and Databricks pipelines.
-
-```python
-from semantica.export import ParquetExporter
-
-exporter = ParquetExporter()
-
-# Export nodes
-exporter.export_nodes(graph, "nodes.parquet")
-
-# Export edges
-exporter.export_edges(graph, "edges.parquet")
-
-# Export full graph (partitioned)
-exporter.export(graph, output_dir="graph_parquet/", partition_by="node_type")
-```
-
----
-
-## ArangoDB AQL Exporter
-
-```python
-from semantica.export import ArangoExporter
-
-exporter = ArangoExporter()
-aql = exporter.export(graph) # returns ready-to-run INSERT statements
-exporter.export_to_file(graph, "arango_import.aql")
-```
-
----
-
-## CSV Exporter
+## CSVExporter
```python
from semantica.export import CSVExporter
-exporter = CSVExporter()
+exporter = CSVExporter(delimiter=",")
exporter.export_nodes(graph, "nodes.csv")
exporter.export_edges(graph, "edges.csv")
```
---
-## GraphML Exporter
+## VectorExporter
+
+Export embedding vectors for external vector stores:
```python
-from semantica.export import GraphMLExporter
+from semantica.export import VectorExporter
-exporter = GraphMLExporter()
-exporter.export(graph, "graph.graphml")
+exporter = VectorExporter()
+exporter.export(embeddings, metadata, "vectors.json", format="json")
+exporter.export(embeddings, metadata, "vectors.npy", format="numpy")
+
+# FAISS-compatible binary format
+exporter.export(embeddings, metadata, "vectors.faiss", format="faiss")
+```
+
+---
+
+## ArrowExporter
+
+Apache Arrow IPC format for zero-copy inter-process transfer:
+
+```python
+from semantica.export import ArrowExporter
+
+exporter = ArrowExporter()
+exporter.export(graph, "graph.arrow")
+```
+
+Requires `pyarrow`. Falls back gracefully if not installed.
+
+---
+
+## DistanceExporter
+
+Export semantic distance matrices from Distance Intelligence:
+
+```python
+from semantica.export import DistanceExporter
+
+exporter = DistanceExporter()
+exporter.export_matrix(distance_matrix, node_labels, "distances.csv")
+exporter.export_ego(ego_neighborhood, center_node="Apple Inc.", path="ego.json")
+```
+
+---
+
+## ReportGenerator
+
+Generate human-readable reports from graph analytics:
+
+```python
+from semantica.export import ReportGenerator
+
+generator = ReportGenerator()
+
+# HTML report
+generator.generate(graph, analytics_result, "report.html", format="html")
+
+# Markdown
+generator.generate(graph, analytics_result, "report.md", format="markdown")
+
+# JSON
+generator.generate(graph, analytics_result, "report.json", format="json")
+```
+
+---
+
+## Convenience Functions
+
+```python
+from semantica.export import (
+ export_rdf, export_parquet, export_csv, export_lpg,
+ export_arango, export_graph, export_owl, export_vector,
+ export_arrow, generate_report
+)
+
+export_rdf(graph, "output.ttl", format="turtle")
+export_parquet(graph, "output/", compression="snappy")
+export_csv(graph, "nodes.csv", target="nodes")
+export_lpg(graph, "import.cypher", method="cypher")
+export_arango(graph, "import.aql")
+export_graph(graph, "graph.graphml", format="graphml")
```
---
@@ -102,13 +260,13 @@ exporter.export(graph, "graph.graphml")
## Selective Export
```python
-# Export subgraph by node IDs
+# Export a subgraph
subgraph = graph.subgraph(node_ids=["apple_inc", "steve_jobs"])
-exporter.export_to_file(subgraph, "subgraph.ttl", format="turtle")
+export_rdf(subgraph, "subgraph.ttl", format="turtle")
-# Export by node type
+# Export nodes by type
org_nodes = graph.filter(node_type="Organization")
-exporter.export_nodes(org_nodes, "organizations.parquet")
+export_parquet(org_nodes, "organizations.parquet")
```
---
diff --git a/docs/reference/normalize.md b/docs/reference/normalize.md
index 99b45a78..fb87abdf 100644
--- a/docs/reference/normalize.md
+++ b/docs/reference/normalize.md
@@ -1,6 +1,6 @@
---
title: "Normalize Module"
-description: "Text cleaning, entity standardization, date normalization, and encoding repair."
+description: "Text cleaning, entity canonicalization, date normalization, number/unit conversion, language detection, and encoding repair."
icon: "broom"
---
@@ -8,104 +8,385 @@ icon: "broom"
---
-## DataNormalizer
+## Overview
+
+The **Normalize Module** standardizes raw data before extraction and graph construction — fixing encodings, canonicalizing entity names, normalizing dates, and detecting languages. All normalizers expose both convenience functions and stateful class instances.
+
+
+
+ Unicode, whitespace, HTML stripping, smart-quote/dash replacement.
+
+
+ Alias resolution, disambiguation, name variant handling.
+
+
+ ISO 8601 output, timezone conversion, relative date parsing.
+
+
+ Currency, unit conversion, scientific notation.
+
+
+ Duplicate detection, schema validation, missing value handling.
+
+
+ 50+ language detection with confidence scoring.
+
+
+
+---
+
+## Convenience Functions
+
+The quickest way to normalize — dispatch via function with a `method` parameter:
```python
-from semantica.normalize import DataNormalizer
+from semantica.normalize import (
+ normalize_text, normalize_entity, normalize_date,
+ normalize_number, clean_data, detect_language, handle_encoding
+)
-normalizer = DataNormalizer()
-
-# Text cleaning
-clean = normalizer.normalize_text(" Hello, World!! \n\n")
+clean = normalize_text(" Hello, World!! \n\n")
# → "Hello, World!!"
-# Date standardization
-date = normalizer.normalize_date("Jan 1st, 2020")
-# → "2020-01-01"
-
-# Entity normalization
-entity = normalizer.normalize_entity("Apple Computer Inc.")
+entity = normalize_entity("Apple Computer Inc.", entity_type="Organization")
# → "Apple Inc."
-# Number normalization
-num = normalizer.normalize_number("$1,234.56")
+date = normalize_date("Jan 1st, 2020")
+# → "2020-01-01"
+
+num = normalize_number("$1,234.56")
# → 1234.56
+
+lang = detect_language("Bonjour le monde")
+# → {"language": "fr", "confidence": 0.98}
```
---
-## Text Normalization
+## TextNormalizer
```python
-text = normalizer.normalize_text(
+from semantica.normalize import TextNormalizer
+
+normalizer = TextNormalizer()
+
+normalized = normalizer.normalize_text(
raw_text,
- lowercase=False, # convert to lowercase
- remove_punctuation=False, # strip punctuation
+ lowercase=False,
+ remove_punctuation=False,
remove_extra_whitespace=True,
- fix_encoding=True, # repair cp1252/latin-1 mojibake
- strip_html=True, # remove HTML tags
- normalize_unicode=True # NFC normalization
+ strip_html=True, # remove HTML tags
+ normalize_unicode=True, # NFC normalization
)
```
+
+
+
+
+```python
+from semantica.normalize import UnicodeNormalizer
+
+normalizer = UnicodeNormalizer(form="NFC")
+text = normalizer.normalize("café") # NFC → canonical composition
+```
+
+
+
+
+
+```python
+from semantica.normalize import WhitespaceNormalizer
+
+normalizer = WhitespaceNormalizer()
+text = normalizer.normalize("Hello World\t\n") # → "Hello World"
+```
+
+
+
+
+
+```python
+from semantica.normalize import SpecialCharacterProcessor
+
+processor = SpecialCharacterProcessor()
+text = processor.process("‘Hello’") # '' → ''
+```
+
+
+
+
+
- **v0.5.0 fix:** The encoding repair now handles cp1252/latin-1 characters that previously caused crashes on Windows when processing documents with non-ASCII content.
+ **v0.5.0 fix:** Encoding repair now handles cp1252/latin-1 characters that previously caused crashes on Windows when processing documents with non-ASCII content.
---
-## Entity Normalization
+## EntityNormalizer
```python
-# Company name normalization
-companies = [
- "Apple Computer, Inc.",
- "Apple Inc",
- "APPLE INC.",
-]
+from semantica.normalize import EntityNormalizer
+
+normalizer = EntityNormalizer()
+
+# Normalize company names — handles suffixes, punctuation, case
+companies = ["Apple Computer, Inc.", "Apple Inc", "APPLE INC."]
normalized = [normalizer.normalize_entity(c) for c in companies]
# All → "Apple Inc."
-# Person name normalization
-name = normalizer.normalize_person_name("JOBS, STEVE")
+# Person names
+name = normalizer.normalize_entity("JOBS, STEVE", entity_type="Person")
# → "Steve Jobs"
```
----
+
-## Date & Time Normalization
+
```python
+from semantica.normalize import AliasResolver
+
+resolver = AliasResolver(aliases={
+ "ML": "Machine Learning",
+ "AI": "Artificial Intelligence",
+ "DL": "Deep Learning",
+})
+
+resolved = resolver.resolve("ML and DL are subsets of AI")
+# → "Machine Learning and Deep Learning are subsets of Artificial Intelligence"
+```
+
+
+
+
+
+```python
+from semantica.normalize import EntityDisambiguator
+
+disambiguator = EntityDisambiguator()
+result = disambiguator.disambiguate(
+ "Apple", context="Steve Jobs founded Apple in Cupertino"
+)
+# → {"entity": "Apple Inc.", "type": "Organization", "confidence": 0.96}
+```
+
+
+
+
+
+```python
+from semantica.normalize import NameVariantHandler
+
+handler = NameVariantHandler()
+canonical = handler.normalize("Dr. JOHN P. SMITH Jr.")
+# → "John P. Smith"
+```
+
+
+
+
+
+---
+
+## DateNormalizer
+
+```python
+from semantica.normalize import DateNormalizer
+
+normalizer = DateNormalizer()
+
dates = [
"January 1st, 2020",
"01/01/2020",
"2020-01-01T00:00:00Z",
- "yesterday", # relative dates supported
+ "yesterday", # relative dates supported
+ "3 weeks ago",
]
normalized = [normalizer.normalize_date(d) for d in dates]
-# All → "2020-01-01" (ISO 8601)
+# All → ISO 8601 strings
+
+# With timezone conversion to UTC
+normalizer_utc = DateNormalizer(target_timezone="UTC")
+utc_date = normalizer_utc.normalize_date("2024-01-01 09:00 EST")
+```
+
+
+
+
+
+```python
+from semantica.normalize import TimeZoneNormalizer
+
+tz_normalizer = TimeZoneNormalizer(target_tz="UTC")
+utc_dt = tz_normalizer.normalize("2024-01-01 09:00", source_tz="America/New_York")
+```
+
+
+
+
+
+```python
+from semantica.normalize import RelativeDateProcessor
+from datetime import datetime
+
+processor = RelativeDateProcessor(reference_date=datetime(2025, 1, 15))
+result = processor.process("3 days ago")
+# → datetime(2025, 1, 12)
+```
+
+
+
+
+
+```python
+from semantica.normalize import TemporalExpressionParser
+
+parser = TemporalExpressionParser()
+result = parser.parse("from January 2020 to March 2021")
+# → {"start": "2020-01-01", "end": "2021-03-31", "type": "range"}
+```
+
+
+
+
+
+---
+
+## NumberNormalizer
+
+```python
+from semantica.normalize import NumberNormalizer
+
+normalizer = NumberNormalizer()
+
+# Currency
+normalizer.normalize_number("$1,234.56") # → 1234.56
+normalizer.normalize_number("€42K") # → 42000.0
+normalizer.normalize_number("$1.2B") # → 1200000000.0
+
+# Scientific notation
+normalizer.normalize_number("3.14e-2") # → 0.0314
+
+# Percentages
+normalizer.normalize_number("42%") # → 0.42
+```
+
+
+
+
+
+```python
+from semantica.normalize import UnitConverter
+
+converter = UnitConverter()
+result = converter.convert(100, from_unit="km/h", to_unit="m/s")
+# → 27.78
+
+# All supported categories: length, weight, volume, temperature, speed, area
+categories = converter.list_categories()
+```
+
+
+
+
+
+```python
+from semantica.normalize import CurrencyNormalizer
+
+normalizer = CurrencyNormalizer()
+result = normalizer.normalize("$42.50")
+# → {"amount": 42.50, "currency": "USD", "raw": "$42.50"}
+```
+
+
+
+
+
+---
+
+## DataCleaner
+
+```python
+from semantica.normalize import DataCleaner, DataValidator, DuplicateDetector
+
+cleaner = DataCleaner()
+
+# Remove duplicates from a dataset
+deduped = cleaner.remove_duplicates(records, similarity_threshold=0.9)
+
+# Fill missing values
+filled = cleaner.fill_missing(records, strategy="mean") # or "median", "mode", "remove"
+
+# Validate schema
+validator = DataValidator()
+result = validator.validate(records, schema={"name": str, "age": int})
+print(result.valid_count, result.errors)
```
---
-## Quantity Normalization
+## LanguageDetector
```python
-# Currency
-normalizer.normalize_number("$1,234.56M") # → 1234560000.0
-normalizer.normalize_number("€42K") # → 42000.0
+from semantica.normalize import LanguageDetector
-# Units
-normalizer.normalize_unit("100 km/h") # → {"value": 100, "unit": "km/h", "si": 27.78}
+detector = LanguageDetector()
+
+# Single text
+lang = detector.detect("Bonjour le monde")
+# → {"language": "fr", "confidence": 0.98}
+
+# Top N languages
+langs = detector.detect_top_n("This might be mixed", n=3)
+# → [{"language": "en", "probability": 0.85}, ...]
+
+# Batch
+results = detector.detect_batch(["Hello", "Hola", "Bonjour"])
+```
+
+---
+
+## EncodingHandler
+
+```python
+from semantica.normalize import EncodingHandler
+
+handler = EncodingHandler()
+
+# Detect encoding
+encoding = handler.detect_encoding(raw_bytes)
+# → {"encoding": "windows-1252", "confidence": 0.73}
+
+# Convert to UTF-8
+utf8_text = handler.to_utf8(raw_bytes)
+
+# Remove BOM
+clean = handler.remove_bom(text_with_bom)
```
---
## Batch Processing
+All normalizers support batch input:
+
```python
+from semantica.normalize import normalize_text
+
texts = ["Text 1...", "Text 2...", "Text 3..."]
-normalized = normalizer.normalize_batch(texts, batch_size=100)
+normalized = [normalize_text(t) for t in texts]
+```
+
+For large datasets, use the pipeline:
+
+```python
+from semantica.pipeline import Pipeline
+from semantica.normalize import TextNormalizer
+
+pipeline = Pipeline()
+pipeline.add_step("normalize", TextNormalizer())
+result = pipeline.run(documents)
```
---
diff --git a/docs/reference/reasoning.md b/docs/reference/reasoning.md
index f4afd6a4..a0515173 100644
--- a/docs/reference/reasoning.md
+++ b/docs/reference/reasoning.md
@@ -1,169 +1,256 @@
---
title: "Reasoning Module"
-description: "Forward chaining, Rete, deductive, abductive, SPARQL, and Datalog reasoning with explainable inference paths."
+description: "Forward chaining, Rete, deductive, abductive, SPARQL, Datalog, and temporal reasoning with explainable inference paths."
icon: "microchip"
---
-> Logical inference engine supporting rule-based inference, SPARQL, Rete, and Datalog reasoning.
+> Logical inference engine supporting rule-based, SPARQL, Rete, Datalog, and temporal reasoning — all with explainable paths.
---
## Overview
-The **Reasoning Module** derives new knowledge from existing facts using logical rules. All engines produce **explainable inference paths** — not black-box conclusions.
+The **Reasoning Module** derives new knowledge from existing facts using logical rules. Every engine produces **explainable inference paths** — not black-box conclusions.
-
- Forward-chaining inference with variable substitution.
+
+ Main facade — forward chaining with IF/THEN rules and variable substitution.
-
+
+ High-performance pattern matching for large rule sets via the Rete algorithm.
+
+
Query expansion and property chain inference over RDF graphs.
-
- High-performance pattern matching for large rule sets.
-
-
+
Recursive Horn clause rules with bottom-up fixpoint semantics (v0.4.0).
+
+ All 13 Allen interval algebra relations for time-aware inference.
+
+
+ Structured explanation paths — how each conclusion was derived.
+
---
-## ReasoningEngine (Forward Chaining)
+## Reasoner (Main Facade)
+
+The unified entry point for rule-based forward-chaining inference:
```python
-from semantica.reasoning import ReasoningEngine
+from semantica.reasoning import Reasoner, Rule, Fact, RuleType
-engine = ReasoningEngine()
-engine.add_rule({
- "if": [
- {"subject": "?person", "predicate": "parent_of", "object": "?child"},
- {"subject": "?child", "predicate": "parent_of", "object": "?grandchild"}
+reasoner = Reasoner()
+
+# Add facts
+reasoner.add_fact(Fact(subject="John", predicate="is_a", obj="Manager"))
+reasoner.add_fact(Fact(subject="John", predicate="is_a", obj="Employee"))
+
+# Add rules
+reasoner.add_rule(Rule(
+ rule_type=RuleType.FORWARD_CHAIN,
+ conditions=[
+ {"subject": "?x", "predicate": "is_a", "object": "Manager"}
],
- "then": {"subject": "?person", "predicate": "grandparent_of", "object": "?grandchild"}
-})
+ conclusion={"subject": "?x", "predicate": "has_authority", "object": "true"}
+))
-inferences = engine.infer(kg)
-for inf in inferences:
- print(f"{inf['subject']} {inf['predicate']} {inf['object']}")
- print(f" Derived via: {inf['explanation']}")
+# Run inference
+result = reasoner.infer()
+for inference in result.derived_facts:
+ print(f"{inference.subject} {inference.predicate} {inference.obj}")
+ print(f" Derived via: {inference.explanation}")
```
---
-## ReteEngine (High Performance)
+## GraphReasoner
+
+Inference over the full knowledge graph structure:
```python
-from semantica.reasoning import ReteEngine
+from semantica.reasoning import GraphReasoner
+
+graph_reasoner = GraphReasoner(kg)
+
+# Infer transitive closure
+graph_reasoner.add_rule({
+ "if": [
+ {"subject": "?a", "predicate": "parent_of", "object": "?b"},
+ {"subject": "?b", "predicate": "parent_of", "object": "?c"}
+ ],
+ "then": {"subject": "?a", "predicate": "ancestor_of", "object": "?c"}
+})
+
+inferences = graph_reasoner.infer(kg)
+for inf in inferences:
+ print(f"{inf['subject']} {inf['predicate']} {inf['object']}")
+```
+
+---
+
+## ReteEngine
+
+High-performance pattern matching using the Rete algorithm — far faster than naive forward chaining for large rule sets because it caches partial matches:
+
+```python
+from semantica.reasoning import ReteEngine, ReteNode, AlphaNode, BetaNode
engine = ReteEngine()
engine.load_rules("rules/domain_rules.json")
results = engine.run(kg)
+
+# Inspect the network
+root: ReteNode = engine.get_root()
+alpha_nodes = engine.get_alpha_nodes() # single-condition filters
+beta_nodes = engine.get_beta_nodes() # join nodes
```
-The Rete algorithm efficiently evaluates large rule sets by caching partial matches — far faster than naive forward chaining for hundreds of rules.
+Rule format for Rete:
----
-
-## DeductiveEngine
-
-```python
-from semantica.reasoning import DeductiveEngine
-
-engine = DeductiveEngine()
-engine.add_axiom("Person", "is_a", "Agent")
-engine.add_axiom("Employee", "is_a", "Person")
-
-# Infer: Employee is_a Agent (transitivity)
-inferences = engine.close_under_transitivity(kg)
+```json
+{
+ "rules": [
+ {
+ "name": "manager_authority",
+ "conditions": [
+ { "subject": "?x", "predicate": "role", "object": "Manager" }
+ ],
+ "action": { "subject": "?x", "predicate": "has_authority", "object": "true" }
+ }
+ ]
+}
```
---
-## AbductiveEngine
-
-```python
-from semantica.reasoning import AbductiveEngine
-
-engine = AbductiveEngine()
-hypotheses = engine.explain(
- observation=("apple_inc", "high_revenue", True),
- knowledge_graph=kg
-)
-
-for h in hypotheses:
- print(f"Hypothesis: {h['explanation']} (probability: {h['probability']:.2f})")
-```
-
-Abductive reasoning infers the most likely explanation for an observation given the current knowledge graph.
-
----
-
-## DatalogEngine (v0.4.0)
-
-Pure-Python bottom-up semi-naive fixpoint evaluation for recursive Horn clause rules.
-
-```python
-from semantica.reasoning import DatalogEngine
-
-datalog = DatalogEngine()
-
-# Add facts
-datalog.add_fact("parent(alice, bob).")
-datalog.add_fact("parent(bob, charlie).")
-
-# Add recursive rule
-datalog.add_rule("ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")
-datalog.add_rule("ancestor(X, Y) :- parent(X, Y).")
-
-# Query
-results = datalog.query("ancestor(alice, ?)")
-# Returns: [("charlie",), ("bob",)]
-```
-
-
- Datalog termination is guaranteed — the engine detects fixpoint convergence and stops. No infinite loops.
-
-
----
-
## SPARQLReasoner
+Query-based inference over RDF graphs:
+
```python
-from semantica.reasoning import SPARQLReasoner
+from semantica.reasoning import SPARQLReasoner, SPARQLQueryResult
reasoner = SPARQLReasoner(graph=rdf_graph)
-query = """
-SELECT ?person ?company WHERE {
- ?person :founded ?company .
- ?company :located_in :SiliconValley .
-}
-"""
+result: SPARQLQueryResult = reasoner.query("""
+ PREFIX ex:
+ SELECT ?person ?company WHERE {
+ ?person ex:founded ?company .
+ ?company ex:located_in ex:SiliconValley .
+ }
+""")
-results = reasoner.query(query)
+for row in result.bindings:
+ print(row["person"], row["company"])
```
-Also supports property chain inference:
+Property chain inference:
```python
-reasoner.add_property_chain("knows", ["friend_of", "colleague_of"])
+# Infer: if A knows B and B is colleague_of C, then A knows C
+reasoner.add_property_chain("knows", ["knows", "colleague_of"])
inferences = reasoner.infer_property_chains()
```
---
+## DatalogReasoner (v0.4.0)
+
+Pure-Python bottom-up semi-naive fixpoint evaluation for recursive Horn clause rules. Termination is guaranteed:
+
+```python
+from semantica.reasoning import DatalogReasoner, DatalogFact, DatalogRule
+
+datalog = DatalogReasoner()
+
+# Add base facts
+datalog.add_fact(DatalogFact("parent", ("alice", "bob")))
+datalog.add_fact(DatalogFact("parent", ("bob", "charlie")))
+
+# Add recursive rules (Horn clauses)
+datalog.add_rule(DatalogRule("ancestor(?X, ?Y) :- parent(?X, ?Y)."))
+datalog.add_rule(DatalogRule("ancestor(?X, ?Z) :- parent(?X, ?Y), ancestor(?Y, ?Z)."))
+
+# Evaluate to fixpoint
+datalog.evaluate()
+
+# Query
+results = datalog.query("ancestor(alice, ?Z)")
+# → [{"Z": "bob"}, {"Z": "charlie"}]
+```
+
+
+ Datalog termination is guaranteed — the engine detects fixpoint convergence and stops automatically. No infinite loops.
+
+
+---
+
+## TemporalReasoningEngine
+
+Reason about time intervals using all 13 Allen interval algebra relations:
+
+```python
+from semantica.reasoning import TemporalReasoningEngine, TemporalInterval, IntervalRelation
+
+engine = TemporalReasoningEngine()
+
+# Define intervals
+ceo_tenure = TemporalInterval(start="1997-09-16", end="2011-08-24")
+board_member = TemporalInterval(start="2000-01-01", end="2012-06-01")
+
+# Check interval relations (all 13 Allen relations supported)
+relation = engine.get_relation(ceo_tenure, board_member)
+# → IntervalRelation.DURING (ceo_tenure is during board_member)
+
+# Named relations
+IntervalRelation.BEFORE # a ends before b starts
+IntervalRelation.MEETS # a ends exactly when b starts
+IntervalRelation.OVERLAPS # a starts before b, ends inside b
+IntervalRelation.DURING # a is fully inside b
+IntervalRelation.STARTS # a and b start together, a ends first
+IntervalRelation.FINISHES # a and b end together, a starts later
+IntervalRelation.EQUALS # identical intervals
+# + 6 inverse relations (AFTER, MET_BY, OVERLAPPED_BY, CONTAINS, STARTED_BY, FINISHED_BY)
+```
+
+---
+
+## ExplanationGenerator
+
+Generate structured explanations for inferences:
+
+```python
+from semantica.reasoning import ExplanationGenerator, Explanation, ReasoningPath
+
+generator = ExplanationGenerator(reasoner)
+
+explanation: Explanation = generator.explain(
+ conclusion={"subject": "John", "predicate": "has_authority", "object": "true"}
+)
+
+print(explanation.conclusion)
+print(explanation.confidence)
+
+for step in explanation.reasoning_path.steps:
+ print(f" Step {step.depth}: {step.fact} via rule '{step.rule_name}'")
+```
+
+---
+
## Built-In Rule Templates
```python
-from semantica.reasoning import ReasoningEngine, RuleTemplates
+from semantica.reasoning import Reasoner
-engine = ReasoningEngine()
+engine = Reasoner()
# Apply common logical patterns
-engine.apply_template(RuleTemplates.TRANSITIVITY, predicate="located_in")
-engine.apply_template(RuleTemplates.SYMMETRY, predicate="knows")
-engine.apply_template(RuleTemplates.INVERSE, predicate1="parent_of", predicate2="child_of")
+engine.apply_transitivity("located_in") # A→B, B→C ⟹ A→C
+engine.apply_symmetry("knows") # A knows B ⟹ B knows A
+engine.apply_inverse("parent_of", "child_of") # A parent_of B ⟹ B child_of A
```
---
@@ -175,12 +262,12 @@ engine.apply_template(RuleTemplates.INVERSE, predicate1="parent_of", predicate2=
The knowledge graph being reasoned over.
- Ontology rules that constrain reasoning.
+ Ontology axioms and SHACL constraints.
-
+
RDF backend for SPARQL reasoning.
- Uses reasoning for agent intelligence.
+ Reasoning integrated into agent intelligence.
diff --git a/docs/reference/visualization.md b/docs/reference/visualization.md
index aba9737e..ad059df7 100644
--- a/docs/reference/visualization.md
+++ b/docs/reference/visualization.md
@@ -140,7 +140,7 @@ start_explorer(graph=kg, port=8080)
Visualize embedding space.
-
+
Full Knowledge Explorer UI.