diff --git a/docs/CodeExamples.md b/docs/CodeExamples.md
deleted file mode 100644
index a7372c5b..00000000
--- a/docs/CodeExamples.md
+++ /dev/null
@@ -1,1062 +0,0 @@
----
-title: "Code Examples"
-description: "Comprehensive code examples for all Semantica modules, processors, and integrations."
-icon: "code"
----
-
-## Quick Start
-
-### 📦 Installation Options
-
-```bash
-# Complete installation with all format support
-pip install "semantica[all]"
-
-# Lightweight installation
-pip install semantica
-
-# Specific format support
-pip install "semantica[pdf,web,feeds,office]"
-
-# Graph store backends
-pip install "semantica[graph-neo4j]" # Neo4j support
-pip install "semantica[graph-falkordb]" # FalkorDB (Redis-based)
-pip install "semantica[graph-all]" # All graph backends
-
-# Development installation
-git clone https://github.com/semantica-agi/semantica.git
-cd semantica
-pip install -e ".[dev]"
-```
-
-### ⚡ 30-Second Demo: From Any Format to Knowledge
-
-```python
-from semantica.ingest import FileIngestor
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder
-from semantica.embeddings import TextEmbedder
-
-# Use individual modules with preferred providers
-ingestor = FileIngestor()
-parser = DocumentParser()
-ner = NERExtractor(method="llm", provider="openai")
-rel_extractor = RelationExtractor()
-builder = GraphBuilder()
-embedder = TextEmbedder(method="openai", model="text-embedding-3-large")
- vector_store="weaviate",
- graph_db="neo4j"
-)
-
-# Process ANY data format
-sources = [
- "financial_report.pdf",
- "https://example.com/news/rss",
- "research_papers/",
- "data.json",
- "https://example.com/article"
-]
-
-# One-line semantic transformation
-knowledge_base = core.build_knowledge_base(sources)
-
-print(f"Processed {len(knowledge_base.documents)} documents")
-print(f"Extracted {len(knowledge_base.entities)} entities")
-print(f"Generated {len(knowledge_base.triplets)} semantic triplets")
-print(f"Created {len(knowledge_base.embeddings)} vector embeddings")
-
-# Query the knowledge base
-results = knowledge_base.query("What are the key financial trends?")
-```
-
----
-
-## 🔧 Data Processing Modules
-
-### 📄 Document Processing Module
-
-Process complex document formats with semantic understanding:
-
-```python
-from semantica.processors import DocumentProcessor
-
-# Initialize document processor
-doc_processor = DocumentProcessor(
- extract_tables=True,
- extract_images=True,
- extract_metadata=True,
- preserve_structure=True
-)
-
-# Process various document types
-pdf_content = doc_processor.process_pdf("report.pdf")
-docx_content = doc_processor.process_docx("document.docx")
-pptx_content = doc_processor.process_pptx("presentation.pptx")
-excel_content = doc_processor.process_excel("data.xlsx")
-
-# Extract semantic information
-for content in [pdf_content, docx_content, pptx_content]:
- semantics = core.extract_semantics(content)
- triplets = core.generate_triplets(semantics)
- embeddings = core.create_embeddings(content.chunks)
-```
-
-### 🌐 Web & Feed Processing Module
-
-Real-time web content and feed processing:
-
-```python
-from semantica.processors import WebProcessor, FeedProcessor
-
-# Web content processor
-web_processor = WebProcessor(
- respect_robots=True,
- extract_metadata=True,
- follow_redirects=True,
- max_depth=3
-)
-
-# RSS/Atom feed processor
-feed_processor = FeedProcessor(
- update_interval="5m",
- deduplicate=True,
- extract_full_content=True
-)
-
-# Process web content
-webpage = web_processor.process_url("https://example.com/article")
-semantics = core.extract_semantics(webpage.content)
-
-# Monitor RSS feeds
-feeds = [
- "https://feeds.feedburner.com/TechCrunch",
- "https://rss.cnn.com/rss/edition.rss",
- "https://feeds.reuters.com/reuters/topNews"
-]
-
-for feed_url in feeds:
- feed_processor.subscribe(feed_url)
-
-# Process new feed items
-async for item in feed_processor.stream_items():
- semantics = core.extract_semantics(item.content)
- knowledge_graph.add_triplets(core.generate_triplets(semantics))
-```
-
-### 🦆 Docling Clear Code Example
-
-High-accuracy document parsing with structural understanding:
-
-```python
-from semantica.parse import DoclingParser
-
-# 1. Initialize DoclingParser
-# Docling provides superior table extraction and structure understanding
-# Requires: pip install docling
-parser = DoclingParser(
- enable_ocr=True, # Enable OCR for scanned documents
- export_format="markdown" # Options: "markdown", "html", "json"
-)
-
-# 2. Parse a complex document
-# Supports PDF, DOCX, PPTX, XLSX, HTML, and images
-result = parser.parse("complex_invoice.pdf")
-
-# 3. Access structured content
-print(f"Content (Markdown):\n{result['full_text']}")
-
-# 4. Extract and iterate over tables with high precision
-for i, table in enumerate(result['tables']):
- print(f"\nTable {i+1}:")
- print(f"Headers: {table.get('headers', [])}")
- print(f"Data rows: {len(table.get('rows', []))}")
-
-# 5. Get document metadata
-metadata = result['metadata']
-print(f"\nMetadata: {metadata.get('title')} ({result.get('total_pages')} pages)")
-```
-
-### 📊 Structured Data Processing Module
-
-Handle structured and semi-structured data formats:
-
-```python
-from semantica.processors import StructuredDataProcessor
-
-# Initialize structured data processor
-structured_processor = StructuredDataProcessor(
- infer_schema=True,
- extract_relationships=True,
- generate_ontology=True
-)
-
-# Process various structured formats
-json_data = structured_processor.process_json("data.json")
-csv_data = structured_processor.process_csv("dataset.csv")
-yaml_data = structured_processor.process_yaml("config.yaml")
-xml_data = structured_processor.process_xml("data.xml")
-
-# Extract semantic relationships
-for data in [json_data, csv_data, yaml_data, xml_data]:
- schema = structured_processor.generate_schema(data)
- triplets = structured_processor.extract_triplets(data, schema)
- ontology = structured_processor.create_ontology(schema)
-```
-
-### 📧 Email & Archive Processing Module
-
-Process email archives and compressed files:
-
-```python
-from semantica.processors import EmailProcessor, ArchiveProcessor
-
-# Email processing
-email_processor = EmailProcessor(
- extract_attachments=True,
- parse_headers=True,
- thread_detection=True
-)
-
-# Archive processing
-archive_processor = ArchiveProcessor(
- recursive=True,
- supported_formats=['zip', 'tar', 'rar', '7z'],
- max_depth=5
-)
-
-# Process email archives
-mbox_data = email_processor.process_mbox("emails.mbox")
-pst_data = email_processor.process_pst("outlook.pst")
-
-# Process compressed archives
-archive_contents = archive_processor.process_archive("documents.zip")
-
-# Extract semantic information from all contents
-for content in archive_contents:
- semantics = core.extract_semantics(content)
- triplets = core.generate_triplets(semantics)
-```
-
-### 🔬 Scientific & Academic Processing Module
-
-Specialized processing for academic and scientific content:
-
-```python
-from semantica.processors import AcademicProcessor
-
-# Academic content processor
-academic_processor = AcademicProcessor(
- extract_citations=True,
- parse_references=True,
- identify_sections=True,
- extract_figures=True
-)
-
-# Process academic formats
-latex_content = academic_processor.process_latex("paper.tex")
-bibtex_content = academic_processor.process_bibtex("references.bib")
-jats_content = academic_processor.process_jats("article.xml")
-
-# Extract academic semantic triplets
-for content in [latex_content, bibtex_content, jats_content]:
- academic_semantics = academic_processor.extract_academic_entities(content)
- citation_graph = academic_processor.build_citation_network(content)
- research_triplets = academic_processor.generate_research_triples(content)
-```
-
----
-
-## 🧩 Semantic Extraction & Transformation
-
-### 🎯 Automatic Triplet Generation
-
-Generate semantic triplets from any content automatically:
-
-```python
-from semantica.extraction import TripletExtractor
-
-# Initialize triplet extractor
-triplet_extractor = TripletExtractor(
- confidence_threshold=0.8,
- include_implicit_relations=True,
- temporal_modeling=True
-)
-
-# Extract triplets from any content
-text = "Apple Inc. was founded by Steve Jobs in 1976 in Cupertino, California."
-triplets = triplet_extractor.extract_triplets(text)
-
-print(triplets)
-# [
-# Triplet(subject="Apple Inc.", predicate="founded_by", object="Steve Jobs"),
-# Triplet(subject="Apple Inc.", predicate="founded_in", object="1976"),
-# Triplet(subject="Apple Inc.", predicate="located_in", object="Cupertino"),
-# Triplet(subject="Cupertino", predicate="located_in", object="California")
-# ]
-
-# Export to various formats
-turtle_format = triplet_extractor.serialize_triplets(triplets, format="turtle")
-ntriples_format = triplet_extractor.serialize_triplets(triplets, format="ntriples")
-jsonld_format = triplet_extractor.serialize_triplets(triplets, format="jsonld")
-```
-
-### 🧠 Ontology Generation Module
-
-Automatically generate ontologies from extracted semantic patterns:
-
-```python
-from semantica.ontology import OntologyGenerator
-
-# Initialize ontology generator
-ontology_gen = OntologyGenerator(
- base_ontologies=["schema.org", "foaf", "dublin_core"],
- generate_classes=True,
- generate_properties=True,
- infer_hierarchies=True
-)
-
-# Generate ontology from documents
-documents = ["doc1.pdf", "doc2.html", "doc3.json"]
-ontology = ontology_gen.generate_from_documents(documents)
-
-# Export ontology in various formats
-owl_ontology = ontology.to_owl()
-rdf_ontology = ontology.to_rdf()
-turtle_ontology = ontology.to_turtle()
-
-# Save to triplet store
-ontology.save_to_triplet_store("http://localhost:9999/blazegraph/sparql")
-```
-
-### 📊 Graph Store - Persistent Property Graph Storage
-
-Store and query knowledge graphs in Neo4j or FalkorDB:
-
-```python
-from semantica.graph_store import GraphStore
-
-# Option 1: Neo4j for enterprise deployments
-store = GraphStore(
- backend="neo4j",
- uri="bolt://localhost:7687",
- user="neo4j",
- password="password"
-)
-
-# Option 3: FalkorDB for ultra-fast LLM applications
-store = GraphStore(backend="falkordb", host="localhost", port=6379, graph_name="kg")
-
-store.connect()
-
-# Create nodes
-company = store.create_node(
- labels=["Company"],
- properties={"name": "Apple Inc.", "founded": 1976, "industry": "Technology"}
-)
-
-person = store.create_node(
- labels=["Person"],
- properties={"name": "Tim Cook", "title": "CEO"}
-)
-
-# Create relationship
-store.create_relationship(
- start_node_id=person["id"],
- end_node_id=company["id"],
- rel_type="CEO_OF",
- properties={"since": 2011}
-)
-
-# Query with Cypher
-results = store.execute_query("""
- MATCH (p:Person)-[:CEO_OF]->(c:Company)
- RETURN p.name as ceo, c.name as company
-""")
-
-# Graph analytics
-neighbors = store.get_neighbors(company["id"], depth=2)
-path = store.shortest_path(person["id"], company["id"])
-stats = store.get_stats()
-
-store.close()
-```
-
-### 📊 Semantic Vector Generation
-
-Create context-aware embeddings optimized for semantic search:
-
-```python
-from semantica.embeddings import SemanticEmbedder
-
-# Initialize semantic embedder
-embedder = SemanticEmbedder(
- model="text-embedding-3-large",
- dimension=1536,
- preserve_context=True,
- semantic_chunking=True
-)
-
-# Generate semantic embeddings
-documents = load_documents()
-semantic_chunks = embedder.semantic_chunk(documents)
-embeddings = embedder.generate_embeddings(semantic_chunks)
-
-# Store in vector database
-vector_store = core.get_vector_store("weaviate")
-vector_store.store_embeddings(semantic_chunks, embeddings)
-
-# Semantic search
-query = "artificial intelligence applications in healthcare"
-results = vector_store.semantic_search(query, top_k=10)
-```
-
----
-
-## 🔄 Real-Time Processing & Streaming
-
-### 📡 Live Feed Processing
-
-Monitor and process live data feeds with semantic understanding:
-
-```python
-from semantica.streaming import LiveFeedProcessor
-
-# Initialize live feed processor
-feed_processor = LiveFeedProcessor(
- processing_interval="30s",
- batch_size=100,
- enable_deduplication=True
-)
-
-# Subscribe to multiple feeds
-feeds = {
- "tech_news": "https://feeds.feedburner.com/TechCrunch",
- "finance": "https://feeds.reuters.com/reuters/businessNews",
- "science": "https://rss.cnn.com/rss/edition_technology.rss"
-}
-
-for name, url in feeds.items():
- feed_processor.subscribe(url, category=name)
-
-# Process items in real-time
-async for feed_item in feed_processor.stream():
- # Extract semantics from new content
- semantics = core.extract_semantics(feed_item.content)
-
- # Generate triplets
- triplets = core.generate_triplets(semantics)
-
- # Update knowledge graph
- knowledge_graph.add_triplets(triplets)
-
- # Create embeddings for search
- embeddings = core.create_embeddings([feed_item.content])
- vector_store.add_embeddings(embeddings)
-
- print(f"Processed: {feed_item.title} from {feed_item.source}")
-```
-
-### 🌊 Stream Processing Integration
-
-Integrate with popular streaming platforms:
-
-```python
-from semantica.streaming import StreamProcessor
-
-# Kafka integration
-kafka_processor = StreamProcessor(
- platform="kafka",
- bootstrap_servers=["localhost:9092"],
- topics=["documents", "web_content", "feeds"]
-)
-
-# RabbitMQ integration
-rabbitmq_processor = StreamProcessor(
- platform="rabbitmq",
- host="localhost",
- port=5672,
- queues=["semantic_processing"]
-)
-
-# Process streaming data
-async for message in kafka_processor.consume():
- content = message.value
-
- # Determine content type and process accordingly
- if message.headers.get("content_type") == "application/pdf":
- processed = doc_processor.process_pdf_bytes(content)
- elif message.headers.get("content_type") == "text/html":
- processed = web_processor.process_html(content)
- else:
- processed = content
-
- # Extract semantics and build knowledge
- semantics = core.extract_semantics(processed)
- triplets = core.generate_triplets(semantics)
- knowledge_graph.add_triplets(triplets)
-```
-
----
-
-## 🎯 Advanced Use Cases
-
-### 🔐 Multi-Format Cybersecurity Intelligence
-
-```python
-from semantica.domains.cyber import CyberIntelProcessor
-
-# Initialize cybersecurity processor
-cyber_processor = CyberIntelProcessor(
- threat_feeds=[
- "https://feeds.feedburner.com/CyberSecurityNewsDaily",
- "https://www.us-cert.gov/ncas/current-activity.xml"
- ],
- formats=["pdf", "html", "xml", "json"],
- extract_iocs=True,
- map_to_mitre=True
-)
-
-# Process various cybersecurity sources
-sources = [
- "threat_report.pdf",
- "https://security-blog.com/rss",
- "vulnerability_data.json",
- "incident_reports/"
-]
-
-cyber_knowledge = cyber_processor.build_threat_intelligence(sources)
-
-# Generate STIX bundles
-stix_bundle = cyber_knowledge.to_stix()
-print(f"Generated STIX bundle with {len(stix_bundle.objects)} objects")
-
-# Export to threat intelligence platforms
-cyber_knowledge.export_to_misp()
-cyber_knowledge.export_to_opencti()
-```
-
-### 🧬 Biomedical Literature Processing
-
-```python
-from semantica.domains.biomedical import BiomedicalProcessor
-
-# Initialize biomedical processor
-bio_processor = BiomedicalProcessor(
- pubmed_integration=True,
- extract_drug_interactions=True,
- map_to_mesh=True,
- clinical_trial_detection=True
-)
-
-# Process biomedical literature
-sources = [
- "research_papers/",
- "https://pubmed.ncbi.nlm.nih.gov/rss/",
- "clinical_reports.pdf",
- "drug_databases.json"
-]
-
-biomedical_knowledge = bio_processor.build_medical_knowledge_base(sources)
-
-# Generate medical ontology
-medical_ontology = biomedical_knowledge.generate_ontology()
-
-# Export to medical databases
-biomedical_knowledge.export_to_umls()
-biomedical_knowledge.export_to_bioportal()
-```
-
-### 📊 Financial Data Aggregation & Analysis
-
-```python
-from semantica.domains.finance import FinancialProcessor
-
-# Initialize financial processor
-finance_processor = FinancialProcessor(
- sec_filings=True,
- news_sentiment=True,
- market_data_integration=True,
- regulatory_compliance=True
-)
-
-# Process financial data sources
-sources = [
- "earnings_reports/",
- "https://feeds.finance.yahoo.com/rss/",
- "sec_filings.xml",
- "market_data.csv",
- "financial_news/"
-]
-
-financial_knowledge = finance_processor.build_financial_knowledge_graph(sources)
-
-# Generate financial semantic triplets
-triplets = financial_knowledge.extract_financial_triplets()
-
-# Export to financial analysis platforms
-financial_knowledge.export_to_bloomberg_api()
-financial_knowledge.export_to_refinitiv()
-```
-
----
-
-## 🏗️ Enterprise Architecture
-
-### 🚀 Scalable Deployment Options
-
-```python
-from semantica.deployment import ScaleManager
-
-# Kubernetes deployment configuration
-k8s_config = {
- "replicas": 5,
- "resources": {
- "cpu": "2000m",
- "memory": "8Gi",
- "gpu": "1"
- },
- "auto_scaling": {
- "min_replicas": 2,
- "max_replicas": 20,
- "cpu_threshold": 70
- }
-}
-
-# Deploy to Kubernetes
-scale_manager = ScaleManager()
-deployment = scale_manager.deploy_kubernetes(config=k8s_config)
-
-# Monitor performance
-metrics = deployment.get_metrics()
-print(f"Processing rate: {metrics.documents_per_second} docs/sec")
-print(f"Memory usage: {metrics.memory_usage_percent}%")
-```
-
-### 🔧 Custom Pipeline Configuration
-
-```python
-from semantica.pipeline import PipelineBuilder
-
-# Build custom processing pipeline
-pipeline = PipelineBuilder() \
- .add_input_sources(["pdf", "html", "rss", "json"]) \
- .add_preprocessing([
- "text_cleaning",
- "language_detection",
- "content_extraction"
- ]) \
- .add_semantic_processing([
- "entity_extraction",
- "relation_extraction",
- "triplet_generation",
- "ontology_mapping"
- ]) \
- .add_enrichment([
- "context_expansion",
- "cross_reference_resolution",
- "metadata_enhancement"
- ]) \
- .add_output_formats([
- "knowledge_graph",
- "vector_embeddings",
- "rdf_triplets",
- "json_ld"
- ]) \
- .build()
-
-# Process data through custom pipeline
-results = pipeline.process(input_sources)
-```
-
----
-
-## 📈 Performance & Monitoring
-
-### 📊 Real-Time Analytics Dashboard
-
-```python
-from semantica.monitoring import AnalyticsDashboard
-
-# Initialize analytics dashboard
-dashboard = AnalyticsDashboard(
- port=8080,
- enable_real_time=True,
- metrics=[
- "processing_rate",
- "extraction_accuracy",
- "memory_usage",
- "knowledge_graph_growth"
- ]
-)
-
-# Start monitoring
-dashboard.start()
-
-# Custom metrics
-dashboard.add_custom_metric("semantic_quality_score",
- lambda: core.get_semantic_quality_score())
-
-# Alert configuration
-dashboard.add_alert(
- condition="processing_rate < 100",
- action="scale_up_workers",
- notification="slack://alerts-channel"
-)
-```
-
-### 🔍 Quality Assurance & Validation
-
-```python
-from semantica.quality import QualityAssurance
-
-# Initialize quality assurance
-qa = QualityAssurance(
- validation_rules=[
- "entity_consistency",
- "triplet_validity",
- "schema_compliance",
- "ontology_alignment"
- ],
- confidence_thresholds={
- "entity_extraction": 0.8,
- "relation_extraction": 0.7,
- "triplet_generation": 0.9
- }
-)
-
-# Validate processing results
-validation_report = qa.validate(processing_results)
-print(f"Overall quality score: {validation_report.quality_score:.2%}")
-print(f"Issues found: {len(validation_report.issues)}")
-
-# Continuous quality monitoring
-qa.enable_continuous_monitoring()
-```
-
-
-## 🏢 Enterprise Knowledge Graph Features
-
-### 📋 Schema-First Knowledge Graph Construction
-
-Unlike other libraries that infer schemas, Semantica enforces predefined business schemas:
-
-```python
-from semantica.schema import SchemaManager, BusinessEntity
-from pydantic import BaseModel
-from typing import List, Optional
-
-# Define your business schema upfront
-class Employee(BusinessEntity):
- name: str
- employee_id: str
- department: str
- role: str
- manager: Optional[str] = None
- email: str
- hire_date: str
-
-class Department(BusinessEntity):
- name: str
- budget: float
- head: str
- location: str
-
-class Product(BusinessEntity):
- name: str
- sku: str
- department: str
- owner: str
- price: float
- launch_date: str
-
-# Initialize schema manager with your business entities
-schema_manager = SchemaManager()
-schema_manager.register_entities([Employee, Department, Product])
-
-# Process documents with schema enforcement
-core = Semantica(schema_manager=schema_manager)
-results = core.process_with_schema("hr_documents/", strict_mode=True)
-
-# Only entities matching your schema are extracted and validated
-print(f"Extracted {len(results.employees)} employees")
-print(f"Extracted {len(results.departments)} departments")
-print(f"Schema violations: {len(results.violations)}")
-```
-
-### 🌱 Seed-Based Knowledge Graph Initialization
-
-Start with known entities and enhance with automated extraction:
-
-```python
-from semantica.knowledge import SeedManager
-
-# Initialize with known business entities
-seed_manager = SeedManager()
-
-# Load seed data from various sources
-seed_manager.load_from_csv("employees.csv", entity_type="Employee")
-seed_manager.load_from_json("departments.json", entity_type="Department")
-seed_manager.load_from_database("products", connection_string="postgresql://...")
-
-# Seed the knowledge graph
-knowledge_graph = core.create_knowledge_graph(seed_data=seed_manager.get_seeds())
-
-# Process new documents - will match against seeded entities
-new_documents = ["meeting_notes.pdf", "project_reports/", "emails.mbox"]
-results = core.process_documents(new_documents,
- knowledge_graph=knowledge_graph,
- enable_entity_linking=True)
-
-# Results show both seeded and newly discovered entities
-print(f"Seeded entities: {len(knowledge_graph.seeded_entities)}")
-print(f"Newly discovered: {len(results.new_entities)}")
-print(f"Linked to existing: {len(results.linked_entities)}")
-```
-
-### 🔄 Intelligent Duplicate Detection & Merging
-
-Automatic deduplication with configurable business rules:
-
-```python
-from semantica.deduplication import EntityDeduplicator
-
-# Configure deduplication rules for each entity type
-dedup_config = {
- "Employee": {
- "match_fields": ["email", "employee_id"],
- "fuzzy_fields": ["name"],
- "similarity_threshold": 0.85,
- "merge_strategy": "most_recent"
- },
- "Product": {
- "match_fields": ["sku"],
- "fuzzy_fields": ["name"],
- "similarity_threshold": 0.90,
- "merge_strategy": "highest_confidence"
- },
- "Department": {
- "match_fields": ["name"],
- "similarity_threshold": 0.95,
- "merge_strategy": "manual_review"
- }
-}
-
-# Initialize deduplicator
-deduplicator = EntityDeduplicator(config=dedup_config)
-
-# Process documents with automatic deduplication
-results = core.process_documents(
- sources=["hr_data/", "finance_reports/", "project_docs/"],
- deduplicator=deduplicator,
- enable_auto_merge=True
-)
-
-# Review deduplication results
-print(f"Duplicates found: {len(results.duplicates)}")
-print(f"Auto-merged: {len(results.auto_merged)}")
-print(f"Requires manual review: {len(results.manual_review_needed)}")
-
-# Access detailed merge information
-for merge in results.auto_merged:
- print(f"Merged {merge.entity_type}: {merge.canonical_name}")
- print(f" Sources: {', '.join(merge.source_documents)}")
- print(f" Confidence: {merge.confidence:.2%}")
-```
-
-### ⚠️ Conflict Detection & Source Traceability
-
-Flag contradictions with complete source tracking:
-
-```python
-from semantica.conflicts import ConflictDetector
-
-# Configure conflict detection rules
-conflict_detector = ConflictDetector(
- track_provenance=True,
- conflict_fields={
- "Employee": ["salary", "department", "role", "manager"],
- "Product": ["price", "owner", "department"],
- "Department": ["budget", "head", "location"]
- },
- confidence_threshold=0.7
-)
-
-# Process with conflict detection enabled
-results = core.process_documents(
- sources=["q1_report.pdf", "hr_database.csv", "manager_updates.docx"],
- conflict_detector=conflict_detector
-)
-
-# Review detected conflicts
-for conflict in results.conflicts:
- print(f"\n🚨 CONFLICT DETECTED: {conflict.entity_name}")
- print(f"Field: {conflict.field}")
- print(f"Conflicting values:")
-
- for claim in conflict.claims:
- print(f" • '{claim.value}' from {claim.source_document}")
- print(f" Page: {claim.page_number}, Confidence: {claim.confidence:.2%}")
- print(f" Context: {claim.context}")
-
- print(f"Recommended action: {conflict.recommended_action}")
-
-# Export conflicts for manual resolution
-conflict_report = results.export_conflicts_report()
-conflict_report.save_to_excel("conflicts_review.xlsx")
-
-# Resolve conflicts programmatically or through UI
-resolution_rules = {
- "Employee.salary": "use_most_recent",
- "Product.price": "use_highest_confidence",
- "Department.budget": "require_manual_review"
-}
-
-resolved_conflicts = conflict_detector.resolve_conflicts(
- results.conflicts,
- rules=resolution_rules
-)
-```
-
-### 📊 Business Rules & Validation Engine
-
-Implement custom business logic and constraints:
-
-```python
-from semantica.validation import BusinessRuleEngine
-
-# Define business rules
-rules = BusinessRuleEngine()
-
-# Add validation rules
-rules.add_rule(
- name="employee_department_exists",
- condition="Employee.department must exist in Department entities",
- severity="error"
-)
-
-rules.add_rule(
- name="salary_range_check",
- condition="Employee.salary must be between $30,000 and $500,000",
- severity="warning"
-)
-
-rules.add_rule(
- name="product_owner_validation",
- condition="Product.owner must be an existing Employee",
- severity="error"
-)
-
-rules.add_rule(
- name="department_budget_consistency",
- condition="Department.budget should align with sum of employee salaries",
- severity="info"
-)
-
-# Process with business rule validation
-results = core.process_documents(
- sources=["company_data/"],
- validation_engine=rules,
- fail_on_errors=False
-)
-
-# Review validation results
-validation_report = results.validation_report
-
-print(f"Total violations: {len(validation_report.violations)}")
-print(f"Errors: {validation_report.errors}")
-print(f"Warnings: {validation_report.warnings}")
-print(f"Info: {validation_report.info}")
-
-# Get detailed violation information
-for violation in validation_report.violations:
- print(f"\n❌ {violation.rule_name}")
- print(f"Entity: {violation.entity_name} ({violation.entity_type})")
- print(f"Issue: {violation.description}")
- print(f"Source: {violation.source_document}")
- print(f"Suggested fix: {violation.suggested_resolution}")
-```
-
-### 🎯 Interactive Conflict Resolution Dashboard
-
-Built-in UI for reviewing and resolving conflicts:
-
-```python
-from semantica.ui import ConflictResolutionDashboard
-
-# Start interactive dashboard
-dashboard = ConflictResolutionDashboard(
- knowledge_graph=knowledge_graph,
- conflicts=results.conflicts,
- port=8080
-)
-
-# Dashboard features:
-# - Side-by-side source comparison
-# - Confidence score visualization
-# - One-click conflict resolution
-# - Bulk resolution with rules
-# - Export resolved data
-
-dashboard.start()
-print("Dashboard available at http://localhost:8080")
-
-# Programmatic resolution after dashboard review
-resolved_data = dashboard.get_resolved_conflicts()
-knowledge_graph.apply_resolutions(resolved_data)
-```
-
-## 🎯 Advanced Use Cases
-
-### 🔐 Multi-Format Cybersecurity Intelligence
-
----
-
-## 🤝 Community & Support
-
-### 🎓 Learning Resources
-
-- **📚 [Documentation](https://semantica.readthedocs.io/)** - Comprehensive guides and API reference
-- **🎯 [Tutorials](https://semantica.readthedocs.io/tutorials/)** - Step-by-step tutorials for common use cases
-- **💡 [Examples Repository](https://github.com/semantica/examples)** - Real-world implementation examples
-- **🎥 [Video Tutorials](https://youtube.com/semantica)** - Visual learning content
-- **📖 [Blog](https://blog.semantica.io/)** - Latest updates and best practices
-
-### 💬 Community Support
-
-- **💬 [Discord Community](https://discord.gg/sV34vps5hH)** - Real-time chat and support
-- **🐙 [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)** - Community Q&A
-- **📧 [Mailing List](https://groups.google.com/g/semantica)** - Announcements and updates
-- **🐦 [Twitter](https://twitter.com/semantica)** - Latest news and tips
-
-### 🏢 Enterprise Support
-
-- **🎯 Professional Services** - Custom implementation and consulting
-- **📞 24/7 Support** - Enterprise-grade support with SLA
-- **🏫 Training Programs** - On-site and remote training for teams
-- **🔒 Security Audits** - Comprehensive security assessments
-
----
-
-## 📄 License
-
-This project is licensed under the MIT License - see the [LICENSE](https://github.com/semantica-agi/semantica/blob/main/LICENSE) file for details.
-
----
-
-## 🙏 Acknowledgments
-
-- **🧠 Research Community** - Built upon cutting-edge research in NLP and semantic web
-- **🤝 Open Source Contributors** - Hundreds of contributors making Semantica better
-- **🏢 Enterprise Partners** - Real-world feedback and requirements shaping development
-- **🎓 Academic Institutions** - Research collaborations and validation
-
----
-
-
-
-**🚀 Ready to transform your data into intelligent knowledge?**
-
-[Get Started Now](https://semantica.readthedocs.io/quickstart/) • [View Examples](https://github.com/semantica/examples) • [Join Community](https://discord.gg/sV34vps5hH)
-
-
diff --git a/docs/DOCS_README.md b/docs/DOCS_README.md
deleted file mode 100644
index 9c3395b1..00000000
--- a/docs/DOCS_README.md
+++ /dev/null
@@ -1,114 +0,0 @@
-# Semantica Documentation
-
-This documentation is built with [MkDocs](https://www.mkdocs.org/) - a fast, simple static site generator for project documentation.
-
-## Features
-
-- **Great themes available** - Using Material theme with beautiful design
-- **Easy to customize** - Custom CSS and theme configuration
-- **Preview as you work** - Built-in dev server with auto-reload
-- **Host anywhere** - Static HTML that works on GitHub Pages, Netlify, etc.
-
-## Quick Start
-
-### 1. Install Dependencies
-
-```bash
-pip install -r requirements-docs.txt
-```
-
-### 2. Preview Locally
-
-```bash
-mkdocs serve
-```
-
-Then open `http://127.0.0.1:8000` in your browser.
-
-### 3. Build for Production
-
-```bash
-mkdocs build
-```
-
-This creates a `site/` directory with static HTML files ready to deploy.
-
-## Project Structure
-
-```
-semantica/
-├── mkdocs.yml # MkDocs configuration
-├── requirements-docs.txt # Python dependencies
-├── docs/ # Documentation source files
-│ ├── index.md # Homepage
-│ ├── *.md # Documentation pages
-│ ├── css/
-│ │ └── custom.css # Custom styling
-│ └── assets/
-│ └── img/
-│ └── Semantica Logo.png
-└── site/ # Generated site (created by mkdocs build)
-```
-
-## Configuration
-
-Main configuration is in `mkdocs.yml`:
-- Site metadata
-- Theme settings (Material theme)
-- Navigation structure
-- Markdown extensions
-- Plugins
-
-## Adding New Pages
-
-1. Create a new `.md` file in `docs/`
-2. Add it to `nav:` section in `mkdocs.yml`
-3. Run `mkdocs serve` to preview
-
-## Customization
-
-### Theme
-
-Edit `mkdocs.yml` under `theme:` section to customize:
-- Color scheme
-- Logo
-- Features enabled
-- Icons
-
-### Styling
-
-Edit `docs/css/custom.css` for custom styles.
-
-## Deployment
-
-### GitHub Pages
-
-```bash
-mkdocs gh-deploy
-```
-
-### Netlify/Vercel
-
-1. Build: `mkdocs build`
-2. Deploy the `site/` directory
-
-### Manual
-
-1. Run `mkdocs build`
-2. Upload `site/` folder contents to your web server
-
-## Development Workflow
-
-1. Edit markdown files in `docs/`
-2. Run `mkdocs serve` to preview
-3. Changes auto-reload in browser
-4. When ready, build with `mkdocs build`
-
-## Benefits
-
-- ✅ Python-based (fits with your Python project)
-- ✅ Beautiful Material theme
-- ✅ Fast and lightweight
-- ✅ Easy to customize
-- ✅ Great search functionality
-- ✅ Mobile responsive
diff --git a/docs/LIBS_README.md b/docs/LIBS_README.md
deleted file mode 100644
index 05c69584..00000000
--- a/docs/LIBS_README.md
+++ /dev/null
@@ -1,1454 +0,0 @@
-# Semantica - Semantic Layer & Knowledge Engineering Framework
-
-[](https://www.python.org/downloads/)
-[](https://github.com/semantica-agi/semantica/blob/main/LICENSE)
-[](https://github.com/semantica-agi/semantica)
-[](https://docs.semantica.dev)
-
-**Semantica** is a comprehensive Python framework for building semantic layers and performing knowledge engineering from unstructured data. It provides production-ready tools for transforming raw data into structured, queryable knowledge graphs with advanced semantic understanding.
-
-## 🚀 Key Features
-
-### Core Capabilities
-- **Universal Data Ingestion**: Process documents, web content, structured data, emails, and more
-- **Advanced Semantic Processing**: Extract entities, relationships, and events with high accuracy
-- **Knowledge Graph Construction**: Build and manage complex knowledge graphs
-- **Multi-Modal Support**: Handle text, images, audio, and video content
-- **Real-Time Processing**: Stream processing and real-time analytics
-- **Production Ready**: Enterprise-grade quality assurance and monitoring
-
-### Semantic Intelligence
-- **Named Entity Recognition**: Extract and classify entities from text
-- **Relationship Extraction**: Identify relationships between entities
-- **Event Detection**: Detect and analyze events in text
-- **Coreference Resolution**: Resolve pronoun and entity references
-- **Semantic Similarity**: Calculate semantic similarity between texts
-- **Ontology Generation**: Automatically generate ontologies from data
-
-### Knowledge Engineering
-- **Knowledge Graph Management**: Build, query, and analyze knowledge graphs
-- **Graph Analytics**: Centrality measures, community detection, connectivity analysis
-- **Entity Resolution**: Deduplicate and resolve entity conflicts
-- **Provenance Tracking**: Track data sources and processing history
-
-
-### Visualization & Analytics
-- **Interactive Visualizations**: Plotly-based interactive charts and graphs
-- **Knowledge Graph Networks**: Network visualizations with community and centrality coloring
-- **Ontology Hierarchies**: Class hierarchy trees and property graphs
-- **Embedding Projections**: 2D/3D projections with UMAP, t-SNE, and PCA
-- **Analytics Visualizations**: Centrality rankings, community structures, connectivity analysis
-- **Temporal Views**: Timeline and evolution visualizations
-
-## 📦 Installation
-
-### Basic Installation
-```bash
-pip install semantica
-```
-
-### With GPU Support
-```bash
-pip install semantica[gpu]
-```
-
-### With Cloud Support
-```bash
-pip install semantica[cloud]
-```
-
-### With Monitoring
-```bash
-pip install semantica[monitoring]
-```
-
-### With Visualization (Optional)
-```bash
-pip install semantica[viz]
-```
-
-Note: Visualization dependencies (plotly, matplotlib, seaborn) are included by default. The `viz` extra includes optional dependencies like `umap-learn` and `graphviz` for advanced features.
-
-### Development Installation
-```bash
-git clone https://github.com/semantica-agi/semantica.git
-cd semantica
-pip install -e ".[dev]"
-```
-
-## 🎯 Quick Start
-
-> **User-Friendly API**: Semantica supports lazy initialization. No need to call `initialize()` explicitly - the framework auto-initializes on first use. Access submodules via dot notation like `semantica.kg`, `semantica.embeddings`, etc.
-
-#### API Usage Patterns
-
-**Pattern 1: Using Individual Modules (Recommended)**
-```python
-from semantica.ingest import FileIngestor
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder
-from semantica.embeddings import TextEmbedder
-
-# Use individual modules for full control
-ingestor = FileIngestor()
-parser = DocumentParser()
-ner = NERExtractor()
-rel_extractor = RelationExtractor()
-builder = GraphBuilder(merge_entities=True)
-embedder = TextEmbedder()
-
-# Build knowledge base step by step
-docs = ingestor.ingest_file("doc1.pdf")
-parsed = parser.parse_document("doc1.pdf")
-text = parsed.get("full_text", "")
-entities = ner.extract_entities(text)
-relationships = rel_extractor.extract_relations(text, entities=entities)
-kg = builder.build_graph(entities=entities, relationships=relationships)
-embeddings = embedder.embed_batch([e.text for e in entities])
-```
-
-**Pattern 2: Using Semantica Class (Orchestration)**
-```python
-from semantica.core import Semantica
-
-# Use Semantica class for orchestration of complex workflows
-# For orchestration, use Semantica class
-from semantica.core import Semantica
-framework = Semantica()
-framework.initialize()
-framework.initialize()
-result = framework.build_knowledge_base(["doc1.pdf", "doc2.docx"], embeddings=True, graph=True)
-framework.shutdown()
-```
-
-!!! tip "Which Pattern to Use?"
- - **Use Individual Modules** (Pattern 1) for most use cases - gives you full control and transparency
- - **Use Semantica Class** (Pattern 2) for complex workflows that need lifecycle management and orchestration
-
-### 1. Basic Document Processing
-```python
-from semantica.ingest import FileIngestor
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder
-from semantica.embeddings import TextEmbedder
-
-# Use individual modules
-documents = ["document1.pdf", "document2.docx", "document3.txt"]
-ingestor = FileIngestor()
-parser = DocumentParser()
-ner = NERExtractor()
-rel_extractor = RelationExtractor()
-builder = GraphBuilder(merge_entities=True)
-embedder = TextEmbedder()
-
-# Process each document
-all_entities = []
-all_relationships = []
-for doc_path in documents:
- doc = ingestor.ingest_file(doc_path)
- parsed = parser.parse_document(doc_path)
- text = parsed.get("full_text", "")
- entities = ner.extract_entities(text)
- relationships = rel_extractor.extract_relations(text, entities=entities)
- all_entities.extend(entities)
- all_relationships.extend(relationships)
-
-# Build knowledge graph and generate embeddings
-kg = builder.build_graph(entities=all_entities, relationships=all_relationships)
-embeddings = embedder.embed_batch([e.text for e in all_entities])
-
-# Access results
-knowledge_graph = result["knowledge_graph"]
-embeddings = result["embeddings"]
-statistics = result["statistics"]
-
-print(f"Processed {statistics['sources_processed']} documents")
-print(f"Success rate: {statistics['success_rate']:.2%}")
-
-# Visualize the knowledge graph
-from semantica.visualization import KGVisualizer
-
-kg_viz = KGVisualizer(layout="force", color_scheme="vibrant")
-fig = kg_viz.visualize_network(knowledge_graph, output="interactive")
-fig.show() # Display interactive visualization
-
-# Or save to HTML file
-kg_viz.visualize_network(knowledge_graph, output="html", file_path="knowledge_graph.html")
-```
-
-### 2. Web Content Processing
-```python
-from semantica.core import Semantica
-from semantica.ingest import WebIngestor
-
-# Ingest web content
-web_ingestor = WebIngestor(
- config={
- "delay": 1.0, # Rate limiting delay
- "respect_robots": True,
- "timeout": 30
- }
-)
-
-# Ingest single URL
-url = "https://example.com/article"
-web_content = web_ingestor.ingest_url(url)
-
-# Or crawl sitemap
-sitemap_url = "https://example.com/sitemap.xml"
-pages = web_ingestor.crawl_sitemap(sitemap_url)
-
-# Build knowledge base from web content
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder
-
-parser = DocumentParser()
-ner = NERExtractor()
-rel_extractor = RelationExtractor()
-builder = GraphBuilder()
-
-all_entities = []
-all_relationships = []
-for web_content in pages:
- parsed = parser.parse_document(web_content.url)
- text = parsed.get("full_text", "")
- entities = ner.extract_entities(text)
- relationships = rel_extractor.extract_relations(text, entities=entities)
- all_entities.extend(entities)
- all_relationships.extend(relationships)
-
-kg = builder.build_graph(entities=all_entities, relationships=all_relationships)
-```
-
-### 3. Knowledge Graph Analytics
-```python
-from semantica.ingest import FileIngestor
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector
-
-# Build knowledge graph using individual modules
-sources = ["document1.pdf", "document2.pdf"]
-ingestor = FileIngestor()
-parser = DocumentParser()
-ner = NERExtractor()
-rel_extractor = RelationExtractor()
-builder = GraphBuilder(merge_entities=True, entity_resolution_strategy="fuzzy")
-
-# Process documents
-all_entities = []
-all_relationships = []
-for source in sources:
- doc = ingestor.ingest_file(source)
- parsed = parser.parse_document(source)
- text = parsed.get("full_text", "")
- entities = ner.extract_entities(text)
- relationships = rel_extractor.extract_relations(text, entities=entities)
- all_entities.extend(entities)
- all_relationships.extend(relationships)
-
-kg = builder.build_graph(entities=all_entities, relationships=all_relationships)
-
-# Build graph object from extracted entities and relationships
-graph_builder = GraphBuilder(
- merge_entities=True,
- entity_resolution_strategy="fuzzy",
- resolve_conflicts=True
-)
-
-# Prepare sources with entities and relationships
-graph_sources = []
-for source_result in kg_data.get("results", []):
- graph_sources.append({
- "entities": source_result.get("entities", []),
- "relationships": source_result.get("relationships", [])
- })
-
-graph = graph_builder.build(graph_sources)
-
-# Analyze graph properties
-analyzer = GraphAnalyzer()
-
-# Calculate centrality using GraphAnalyzer
-centrality = analyzer.calculate_centrality(graph, centrality_type="degree")
-
-# Or use CentralityCalculator directly
-centrality_calc = CentralityCalculator()
-centrality = centrality_calc.calculate_all_centrality(
- graph,
- centrality_types=["degree", "betweenness", "closeness"]
-)
-
-# Detect communities
-community_detector = CommunityDetector()
-communities = community_detector.detect_communities(graph, algorithm="louvain")
-
-# Analyze connectivity
-connectivity = analyzer.analyze_connectivity(graph)
-
-# Or use ConnectivityAnalyzer directly
-from semantica.kg import ConnectivityAnalyzer
-connectivity_analyzer = ConnectivityAnalyzer()
-connectivity = connectivity_analyzer.analyze_connectivity(graph)
-
-print(f"Found {len(communities)} communities")
-print(f"Graph connectivity: {connectivity['is_connected']}")
-```
-
-## 🏗️ Architecture
-
-### Core Modules
-- **Core**: Framework orchestration and configuration
-- **Ingest**: Data ingestion from various sources
-- **Parse**: Content parsing and extraction
-- **Normalize**: Data normalization and cleaning
-- **Semantic Extract**: Entity and relationship extraction
-- **Ontology**: Ontology management and generation
-- **Knowledge Graph**: Graph construction and management
-- **Embeddings**: Vector embedding generation
-- **Vector Store**: Vector storage and retrieval
-- **Pipeline**: Processing pipeline orchestration
-- **Streaming**: Real-time stream processing
-- **Security**: Access control and data protection
-- **Quality**: Quality assurance and validation
-- **Export**: Data export and reporting
-
-### Supported Data Sources
-- **Documents**: PDF, DOCX, HTML, TXT, XML, JSON, CSV
-- **Web Content**: Websites, RSS feeds, APIs
-- **Databases**: SQL, NoSQL, Graph databases
-- **Streams**: Kafka, Pulsar, RabbitMQ, Kinesis
-- **Cloud Storage**: S3, GCS, Azure Blob
-- **Repositories**: Git repositories, code analysis
-
-## 📚 Documentation
-
-### Comprehensive Guides
-- [Getting Started](https://docs.semantica.dev/getting-started)
-- [API Reference](https://docs.semantica.dev/api-reference)
-- [Cookbook Examples](https://docs.semantica.dev/cookbook)
-- [Configuration Guide](https://docs.semantica.dev/configuration)
-- [Deployment Guide](https://docs.semantica.dev/deployment)
-
-### Tutorials
-- [Document Processing Tutorial](https://docs.semantica.dev/tutorials/document-processing)
-- [Knowledge Graph Tutorial](https://docs.semantica.dev/tutorials/knowledge-graph)
-- [Web Scraping Tutorial](https://docs.semantica.dev/tutorials/web-scraping)
-- [Multi-Modal Processing Tutorial](https://docs.semantica.dev/tutorials/multi-modal)
-
-## 🎨 Detailed Code Examples
-
-### 1. Data Ingestion Examples
-
-#### File Ingestion
-
-**Option 1: Using module-level build function (Recommended)**
-```python
-from semantica.ingest import FileIngestor
-
-# Initialize file ingestor
-ingestor = FileIngestor()
-
-# Ingest directory recursively
-files = ingestor.ingest_directory(
- "documents/",
- recursive=True,
- file_types=[".pdf", ".docx", ".txt"]
-)
-
-for file_obj in files:
- print(f"File: {file_obj.path}")
- print(f"Type: {file_obj.file_type}")
- print(f"Size: {file_obj.size} bytes")
-```
-
-**Option 2: Using FileIngestor for single files**
-```python
-from semantica.ingest import FileIngestor
-from pathlib import Path
-
-# Initialize file ingestor
-file_ingestor = FileIngestor()
-
-# Ingest single file
-file_obj = file_ingestor.ingest_file("document.pdf")
-
-# Ingest entire directory
-files = file_ingestor.ingest_directory(
- "documents/",
- recursive=True,
- extensions=[".pdf", ".docx", ".txt"]
-)
-
-# Process file objects
-for file_obj in files:
- print(f"File: {file_obj.path}")
- print(f"Type: {file_obj.file_type}")
- print(f"Size: {file_obj.size} bytes")
-```
-
-#### Web Content Ingestion
-```python
-from semantica.ingest import WebIngestor, FeedIngestor
-
-# Web ingestion
-web_ingestor = WebIngestor(
- config={
- "delay": 1.0,
- "respect_robots": True,
- "user_agent": "MyBot/1.0"
- }
-)
-
-# Ingest single URL
-content = web_ingestor.ingest_url("https://example.com/article")
-print(f"Title: {content.title}")
-print(f"Text: {content.text[:200]}...")
-
-# Crawl sitemap
-pages = web_ingestor.crawl_sitemap("https://example.com/sitemap.xml")
-print(f"Found {len(pages)} pages")
-
-# RSS/Atom feed ingestion
-feed_ingestor = FeedIngestor()
-feed_data = feed_ingestor.ingest_feed("https://example.com/feed.xml")
-
-for item in feed_data.items:
- print(f"Title: {item.title}")
- print(f"Published: {item.published}")
-```
-
-#### Stream Ingestion
-```python
-from semantica.ingest import StreamIngestor, KafkaProcessor, RabbitMQProcessor
-
-# Initialize stream ingestor
-stream_ingestor = StreamIngestor()
-
-# Ingest from Kafka
-kafka_processor = stream_ingestor.ingest_kafka(
- topic="documents",
- bootstrap_servers=["localhost:9092"],
- consumer_config={"group_id": "semantica_processor"}
-)
-
-# Or ingest from RabbitMQ
-rabbitmq_processor = stream_ingestor.ingest_rabbitmq(
- queue="documents",
- connection_url="amqp://user:pass@localhost:5672/"
-)
-
-# Or create processors directly
-kafka_processor = KafkaProcessor(
- topic="documents",
- bootstrap_servers=["localhost:9092"],
- consumer_config={"group_id": "semantica_processor"}
-)
-
-# Process messages with callback
-def process_message(message):
- result = kafka_processor.process_message(message)
- print(f"Received: {result['content']}")
- # Process message content...
-
-# Set message handler
-kafka_processor.message_handler = process_message
-
-# Start streaming
-stream_ingestor.start_streaming([kafka_processor])
-
-# Or start individual processor
-kafka_processor.start_consuming()
-```
-
-#### Database Ingestion
-```python
-from semantica.ingest import DBIngestor
-
-# Initialize database ingestor
-db_ingestor = DBIngestor(
- config={
- "batch_size": 1000
- }
-)
-
-# Export from specific table
-connection_string = "postgresql://user:pass@localhost/db"
-table_data = db_ingestor.export_table(
- connection_string,
- "articles",
- limit=1000
-)
-
-# Or ingest entire database
-database_data = db_ingestor.ingest_database(
- connection_string,
- include_tables=["articles", "authors"],
- max_rows_per_table=10000
-)
-
-# Access table data
-for row in table_data.rows:
- print(f"ID: {row['id']}, Title: {row['title']}")
-
-# Or execute custom query
-results = db_ingestor.execute_query(
- connection_string,
- "SELECT * FROM articles WHERE published_at > :date",
- date="2023-01-01"
-)
-```
-
-### 2. Semantic Extraction Examples
-
-#### Entity Extraction
-
-**Option 1: Using module-level build function (Recommended)**
-```python
-from semantica.semantic_extract import NamedEntityRecognizer
-
-text = "Apple Inc. is a technology company founded by Steve Jobs in Cupertino, California."
-
-# Extract entities using NamedEntityRecognizer
-ner = NamedEntityRecognizer()
-entities = ner.extract_entities(text)
-
-for entity in entities:
- print(f"Entity: {entity.get('text')}")
- print(f"Type: {entity.get('type')}")
- print(f"Confidence: {entity.get('confidence')}")
- print()
-```
-
-**Option 2: Using NERExtractor for more control**
-```python
-from semantica.semantic_extract import NERExtractor, NamedEntityRecognizer
-
-# Simple NER extractor
-ner_extractor = NERExtractor(
- model="en_core_web_sm",
- min_confidence=0.5
-)
-
-text = "Apple Inc. is a technology company founded by Steve Jobs in Cupertino, California."
-
-# Extract entities
-entities = ner_extractor.extract_entities(text)
-
-for entity in entities:
- print(f"Entity: {entity.text}")
- print(f"Type: {entity.entity_type}")
- print(f"Confidence: {entity.confidence}")
- print(f"Position: {entity.start_char}-{entity.end_char}")
- print()
-
-# Advanced entity recognizer
-entity_recognizer = NamedEntityRecognizer(
- config={
- "ner": {"model": "en_core_web_lg"},
- "classifier": {"enable": True}
- }
-)
-
-# Extract and classify entities
-entities = entity_recognizer.extract_entities(text)
-classified = entity_recognizer.classify_entities(entities)
-
-# Group entities by type
-for entity_type, entity_list in classified.items():
- print(f"{entity_type}: {len(entity_list)} entities")
-```
-
-#### Relationship Extraction
-```python
-from semantica.semantic_extract import RelationExtractor, NERExtractor
-
-# Initialize extractors
-ner_extractor = NERExtractor()
-relation_extractor = RelationExtractor()
-
-text = "Tim Cook is the CEO of Apple Inc. Apple was founded by Steve Jobs."
-
-# Extract entities first
-entities = ner_extractor.extract_entities(text)
-
-# Extract relationships
-relations = relation_extractor.extract_relations(text, entities)
-
-for relation in relations:
- print(f"Subject: {relation.subject}")
- print(f"Predicate: {relation.predicate}")
- print(f"Object: {relation.object}")
- print(f"Confidence: {relation.confidence}")
- print()
-```
-
-#### Triplet Extraction
-```python
-from semantica.semantic_extract import TripletExtractor
-
-# Initialize triplet extractor
-triplet_extractor = TripletExtractor(
- config={
- "validator": {"strict": True},
- "serializer": {"format": "turtle"}
- }
-)
-
-text = "Barack Obama was the President of the United States from 2009 to 2017."
-
-# Extract RDF triples
-triplets = triplet_extractor.extract_triples(text)
-
-for triplet in triples:
- print(f"Subject: {triplet.subject}")
- print(f"Predicate: {triplet.predicate}")
- print(f"Object: {triplet.object}")
- print(f"Confidence: {triplet.confidence}")
- print()
-```
-
-#### Event Detection
-```python
-from semantica.semantic_extract import EventDetector
-
-# Initialize event detector
-event_detector = EventDetector(
- config={
- "classifier": {"enable": True},
- "temporal": {"enable": True}
- }
-)
-
-text = "The company announced the merger on January 15, 2023. The deal was finalized in March."
-
-# Detect events
-events = event_detector.detect_events(text)
-
-for event in events:
- print(f"Event: {event.text}")
- print(f"Type: {event.event_type}")
- print(f"Time: {event.time}")
- print(f"Participants: {event.participants}")
- print()
-```
-
-### 3. Embeddings Generation Examples
-
-#### Text Embeddings
-
-**Option 1: Using module-level build function (Recommended)**
-```python
-import numpy as np
-
-# Generate embeddings using EmbeddingGenerator
-from semantica.embeddings import EmbeddingGenerator
-
-texts = [
- "First document text.",
- "Second document text.",
- "Third document text."
-]
-
-generator = EmbeddingGenerator()
-embeddings = [generator.generate_embeddings(t, data_type="text") for t in texts]
-print(f"Generated {len(embeddings)} embeddings")
-```
-
-**Option 2: Using TextEmbedder for more control**
-```python
-import numpy as np
-from semantica.embeddings import TextEmbedder, EmbeddingGenerator
-
-# Simple text embedder
-text_embedder = TextEmbedder(
- model_name="all-MiniLM-L6-v2",
- device="cpu",
- normalize=True
-)
-
-# Embed single text
-text = "This is a sample text for embedding."
-embedding = text_embedder.embed_text(text)
-print(f"Embedding shape: {embedding.shape}")
-print(f"Embedding norm: {np.linalg.norm(embedding)}")
-
-# Embed batch of texts
-texts = [
- "First document text.",
- "Second document text.",
- "Third document text."
-]
-embeddings = text_embedder.embed_batch(texts)
-print(f"Batch embeddings shape: {embeddings.shape}")
-
-# Advanced embedding generator
-embedding_generator = EmbeddingGenerator(
- config={
- "text": {"model_name": "sentence-transformers/all-mpnet-base-v2"},
- "image": {"model_name": "clip-vit-base-patch32"},
- "audio": {"model_name": "wav2vec2-base"}
- }
-)
-
-# Generate embeddings for different data types
-text_embedding = embedding_generator.generate_embeddings(
- "Sample text",
- data_type="text"
-)
-
-image_embedding = embedding_generator.generate_embeddings(
- "image.jpg",
- data_type="image"
-)
-```
-
-#### Multi-Modal Embeddings
-```python
-from semantica.embeddings import MultimodalEmbedder
-
-# Initialize multimodal embedder
-multimodal_embedder = MultimodalEmbedder(
- config={
- "text_model": "sentence-transformers/all-mpnet-base-v2",
- "image_model": "openai/clip-vit-base-patch32"
- }
-)
-
-# Embed text and image together
-text = "A red apple on a white table"
-image_path = "apple.jpg"
-
-# Joint embedding
-joint_embedding = multimodal_embedder.embed_multimodal(
- text=text,
- image=image_path
-)
-
-# Calculate similarity
-similarity = multimodal_embedder.calculate_similarity(
- text=text,
- image=image_path
-)
-print(f"Text-Image similarity: {similarity}")
-```
-
-### 4. Knowledge Graph Building Examples
-
-#### Building Knowledge Graph
-
-**Option 1: Using GraphBuilder (Recommended)**
-```python
-from semantica.kg import GraphBuilder
-
-# Build knowledge graph from entity/relationship data
-builder = GraphBuilder(
- merge_entities=True,
- entity_resolution_strategy="fuzzy",
- resolve_conflicts=True,
- enable_temporal=True
-)
-
-# Prepare sources with entities and relationships
-sources = [{
- "entities": [...], # Your extracted entities
- "relationships": [...] # Your extracted relationships
-}]
-
-graph = builder.build(sources)
-print(f"Total entities: {len(graph.get('entities', []))}")
-print(f"Total relationships: {len(graph.get('relationships', []))}")
-```
-
-**Option 2: Using GraphBuilder with full control**
-```python
-from semantica.kg import GraphBuilder, EntityResolver
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-
-# Initialize components
-graph_builder = GraphBuilder(
- merge_entities=True,
- entity_resolution_strategy="fuzzy",
- resolve_conflicts=True,
- enable_temporal=True,
- temporal_granularity="day"
-)
-
-entity_resolver = EntityResolver(
- similarity_threshold=0.8,
- strategy="fuzzy"
-)
-
-# Extract entities and relationships from multiple sources
-ner_extractor = NERExtractor()
-relation_extractor = RelationExtractor()
-
-sources = []
-for doc in documents:
- entities = ner_extractor.extract_entities(doc["text"])
- relations = relation_extractor.extract_relations(doc["text"], entities)
- sources.append({
- "entities": entities,
- "relationships": relations,
- "metadata": {"source": doc["path"]}
- })
-
-# Build knowledge graph
-graph = graph_builder.build(
- sources,
- entity_resolver=entity_resolver
-)
-
-# Access graph data
-print(f"Total entities: {len(graph.entities)}")
-print(f"Total relationships: {len(graph.relationships)}")
-```
-
-#### Temporal Knowledge Graph
-```python
-from semantica.kg import GraphBuilder, TemporalGraphQuery
-
-# Build temporal knowledge graph
-temporal_graph_builder = GraphBuilder(
- enable_temporal=True,
- track_history=True,
- version_snapshots=True
-)
-
-# Build graph with temporal information
-graph = temporal_graph_builder.build(sources)
-
-# Query temporal information
-temporal_query = TemporalGraphQuery(graph)
-
-# Query graph at specific time
-snapshot = temporal_query.query_at_time(
- "2023-01-15",
- include_entities=True,
- include_relationships=True
-)
-
-# Detect temporal patterns
-from semantica.kg import TemporalPatternDetector
-pattern_detector = TemporalPatternDetector()
-patterns = pattern_detector.detect_patterns(graph)
-
-for pattern in patterns:
- print(f"Pattern: {pattern.pattern_type}")
- print(f"Entities: {pattern.entities}")
- print(f"Time span: {pattern.start_time} - {pattern.end_time}")
-```
-
-### 5. Pipeline Building Examples
-
-#### Custom Pipeline
-```python
-from semantica.pipeline import PipelineBuilder
-from semantica.pipeline import ExecutionEngine
-
-# Build custom pipeline
-pipeline_builder = PipelineBuilder()
-
-pipeline = (
- pipeline_builder
- .add_step("ingest", "ingest", config={"source": "documents/"})
- .add_step("parse", "parse", config={"formats": ["pdf", "docx"]}, dependencies=["ingest"])
- .add_step("normalize", "normalize", config={}, dependencies=["parse"])
- .add_step("extract", "extract", config={"entities": True, "relations": True}, dependencies=["normalize"])
- .add_step("embed", "embed", config={"model": "text-embedding-3-large"}, dependencies=["extract"])
- .add_step("build_kg", "build_kg", config={}, dependencies=["extract", "embed"])
- .set_parallelism(4)
- .build("document_processing_pipeline")
-)
-
-# Execute pipeline
-execution_engine = ExecutionEngine()
-result = execution_engine.execute_pipeline(pipeline, data="documents/")
-
-print(f"Pipeline executed: {result.success}")
-print(f"Execution time: {result.execution_time:.2f}s")
-print(f"Steps completed: {result.steps_completed if hasattr(result, 'steps_completed') else 'N/A'}")
-```
-
-#### Using Pipeline Templates
-```python
-from semantica.pipeline import PipelineTemplateManager, PipelineBuilder, ExecutionEngine
-
-# Initialize template manager
-template_manager = PipelineTemplateManager()
-
-# Get pre-built template
-pipeline_template = template_manager.get_template("document_processing")
-
-# Build pipeline from template
-pipeline_builder = PipelineBuilder()
-# Note: You would need to implement from_template method or manually build from template
-custom_pipeline = pipeline_builder.build("custom_document_pipeline")
-
-# Execute
-execution_engine = ExecutionEngine()
-result = execution_engine.execute_pipeline(custom_pipeline)
-```
-
-### 6. Quality Assurance Examples
-
-Note: The `semantica.kg_qa` module is temporarily unavailable and will be reintroduced in a future release.
-
-### 7. Export Examples
-
-#### Export Knowledge Graph
-```python
-from semantica.export import JSONExporter, RDFExporter, GraphExporter, CSVExporter
-
-# Export to JSON
-json_exporter = JSONExporter()
-json_exporter.export(graph, "knowledge_graph.json")
-
-# Or export knowledge graph specifically
-json_exporter.export_knowledge_graph(graph, "knowledge_graph.json")
-
-# Export entities and relationships separately
-json_exporter.export_entities(graph.entities, "entities.json")
-json_exporter.export_relationships(graph.relationships, "relationships.json")
-
-# Export to RDF
-rdf_exporter = RDFExporter()
-rdf_exporter.export(graph, "knowledge_graph.ttl", format="turtle")
-
-# Or export to RDF directly
-rdf_content = rdf_exporter.export_to_rdf(graph, format="turtle")
-
-# Export to graph formats (GraphML, GEXF, DOT)
-graph_exporter = GraphExporter(format="graphml")
-graph_exporter.export_knowledge_graph(graph, "knowledge_graph.graphml")
-
-# Export to CSV
-csv_exporter = CSVExporter()
-csv_exporter.export_entities(graph.entities, "entities.csv")
-csv_exporter.export_relationships(graph.relationships, "relationships.csv")
-
-# Or export entire knowledge graph to CSV
-csv_exporter.export_knowledge_graph(graph, "knowledge_graph.csv")
-```
-
-### 8. Complete End-to-End Example
-
-```python
-from semantica.ingest import FileIngestor
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.embeddings import EmbeddingGenerator
-from semantica.kg import GraphBuilder
-from semantica.export import JSONExporter
-
-# No explicit initialization needed - framework auto-initializes on first use
-
-# Step 1: Ingest documents
-file_ingestor = FileIngestor()
-files = file_ingestor.ingest_directory("documents/", recursive=True)
-
-# Step 2: Extract entities and relationships
-ner_extractor = NERExtractor(model="en_core_web_lg")
-relation_extractor = RelationExtractor()
-
-all_entities = []
-all_relationships = []
-
-for file_obj in files:
- # Parse file (assuming parsed text available)
- text = file_obj.content.decode("utf-8") if file_obj.content else ""
-
- # Extract entities
- entities = ner_extractor.extract_entities(text)
- all_entities.extend(entities)
-
- # Extract relationships
- relations = relation_extractor.extract_relations(text, entities)
- all_relationships.extend(relations)
-
-# Step 3: Generate embeddings
-embedding_generator = EmbeddingGenerator()
-embeddings = embedding_generator.generate_embeddings(
- [e.text for e in all_entities],
- data_type="text"
-)
-
-# Step 4: Build knowledge graph
-graph_builder = GraphBuilder(
- merge_entities=True,
- resolve_conflicts=True
-)
-
-graph = graph_builder.build({
- "entities": all_entities,
- "relationships": all_relationships
-})
-
-# Step 5: (Optional) Quality assessment is temporarily unavailable
-# The `semantica.kg_qa` module will be reintroduced in a future release.
-
-# Step 6: Export results
-json_exporter = JSONExporter()
-json_exporter.export(graph, "final_knowledge_graph.json")
-
-print("Processing complete!")
-print(f"Total entities: {len(graph.entities)}")
-print(f"Total relationships: {len(graph.relationships)}")
-```
-
-### 9. Visualization Examples
-
-The Semantica visualization module provides comprehensive visualization capabilities for all knowledge artifacts. All visualizers support both interactive (Plotly) and static export formats (HTML, PNG, SVG, PDF).
-
-#### Knowledge Graph Visualization
-```python
-from semantica.visualization import KGVisualizer
-
-# Initialize KG visualizer
-kg_viz = KGVisualizer(layout="force", color_scheme="vibrant")
-
-# Visualize network graph
-graph = {
- "entities": all_entities,
- "relationships": all_relationships
-}
-
-# Interactive network visualization
-fig = kg_viz.visualize_network(graph, output="interactive")
-fig.show()
-
-# Save to HTML
-kg_viz.visualize_network(graph, output="html", file_path="kg_network.html")
-
-# Visualize with community coloring
-from semantica.kg import CommunityDetector
-community_detector = CommunityDetector()
-communities = community_detector.detect_communities(graph, algorithm="louvain")
-kg_viz.visualize_communities(graph, communities, output="html", file_path="kg_communities.html")
-
-# Visualize with centrality
-from semantica.kg import CentralityCalculator
-centrality_calc = CentralityCalculator()
-centrality = centrality_calc.calculate_all_centrality(graph, centrality_types=["degree"])
-kg_viz.visualize_centrality(graph, centrality, centrality_type="degree",
- output="html", file_path="kg_centrality.html")
-
-# Entity type distribution
-kg_viz.visualize_entity_types(graph, output="html", file_path="entity_types.html")
-
-# Relationship matrix
-kg_viz.visualize_relationship_matrix(graph, output="html", file_path="relationship_matrix.html")
-```
-
-#### Ontology Visualization
-```python
-from semantica.visualization import OntologyVisualizer
-from semantica.ontology import OntologyGenerator
-
-# Initialize ontology visualizer
-onto_viz = OntologyVisualizer(color_scheme="default")
-
-# Option 1: Visualize from ontology generator result
-ontology_generator = OntologyGenerator()
-semantic_model = ontology_generator.generate_ontology(data)
-
-# Visualize semantic model (handles both ontology and semantic network)
-onto_viz.visualize_semantic_model(semantic_model, output="html", file_path="semantic_model.html")
-
-# Option 2: Visualize class hierarchy directly
-ontology = {
- "classes": classes,
- "properties": properties
-}
-
-# Hierarchy tree visualization
-onto_viz.visualize_hierarchy(ontology, output="html", file_path="ontology_hierarchy.html")
-
-# Option 3: Visualize from semantic network (auto-extracts classes)
-from semantica.semantic_extract import SemanticNetworkExtractor
-extractor = SemanticNetworkExtractor()
-semantic_network = extractor.extract_network(text)
-
-# Can visualize directly - will extract classes automatically
-onto_viz.visualize_hierarchy({"semantic_network": semantic_network},
- output="html", file_path="ontology_from_network.html")
-
-# Property graph visualization
-onto_viz.visualize_properties(ontology, output="html", file_path="ontology_properties.html")
-
-# Ontology structure network
-onto_viz.visualize_structure(ontology, output="html", file_path="ontology_structure.html")
-
-# Class-property matrix
-onto_viz.visualize_class_property_matrix(ontology, output="html", file_path="class_property_matrix.html")
-
-# Ontology metrics dashboard
-onto_viz.visualize_metrics(ontology, output="html", file_path="ontology_metrics.html")
-```
-
-#### Embedding Visualization
-```python
-from semantica.visualization import EmbeddingVisualizer
-import numpy as np
-
-# Initialize embedding visualizer
-emb_viz = EmbeddingVisualizer(point_size=8)
-
-# 2D projection using UMAP
-embeddings = np.array([...]) # Your embeddings array
-fig = emb_viz.visualize_2d_projection(
- embeddings,
- labels=["Entity 1", "Entity 2", ...],
- method="umap",
- output="interactive"
-)
-fig.show()
-
-# 3D projection
-emb_viz.visualize_3d_projection(embeddings, method="pca",
- output="html", file_path="embeddings_3d.html")
-
-# Similarity heatmap
-emb_viz.visualize_similarity_heatmap(embeddings,
- output="html", file_path="similarity_heatmap.html")
-
-# Clustering visualization
-from sklearn.cluster import KMeans
-kmeans = KMeans(n_clusters=5)
-cluster_labels = kmeans.fit_predict(embeddings)
-emb_viz.visualize_clustering(embeddings, cluster_labels, method="umap",
- output="html", file_path="embedding_clusters.html")
-
-# Multi-modal comparison
-text_embeddings = np.array([...])
-image_embeddings = np.array([...])
-emb_viz.visualize_multimodal_comparison(
- text_embeddings=text_embeddings,
- image_embeddings=image_embeddings,
- output="html",
- file_path="multimodal_comparison.html"
-)
-
-# Quality metrics
-# emb_viz.visualize_quality_metrics(embeddings, output="html", file_path="embedding_quality.html")
-```
-
-#### Semantic Network Visualization
-```python
-from semantica.visualization import SemanticNetworkVisualizer
-
-# Initialize semantic network visualizer
-sem_net_viz = SemanticNetworkVisualizer()
-
-# Option 1: Visualize SemanticNetwork dataclass object
-from semantica.semantic_extract import SemanticNetworkExtractor
-extractor = SemanticNetworkExtractor()
-semantic_network = extractor.extract_network(text)
-
-# Network graph
-sem_net_viz.visualize_network(semantic_network, output="html", file_path="semantic_network.html")
-
-# Option 2: Visualize from dictionary format
-semantic_network_dict = {
- "nodes": [{"id": "n1", "label": "Node 1", "type": "Entity"}],
- "edges": [{"source": "n1", "target": "n2", "label": "relatedTo"}]
-}
-sem_net_viz.visualize_network(semantic_network_dict, output="html", file_path="semantic_network.html")
-
-# Option 3: Visualize from semantic model (ontology generator result)
-from semantica.ontology import OntologyGenerator
-generator = OntologyGenerator()
-semantic_model = generator.generate_ontology(data)
-sem_net_viz.visualize_network(semantic_model.semantic_network, output="html", file_path="semantic_model_network.html")
-
-# Node type distribution
-sem_net_viz.visualize_node_types(semantic_network, output="html", file_path="node_types.html")
-
-# Edge type distribution
-sem_net_viz.visualize_edge_types(semantic_network, output="html", file_path="edge_types.html")
-```
-
-
-
-#### Graph Analytics Visualization
-```python
-from semantica.visualization import AnalyticsVisualizer
-
-# Initialize analytics visualizer
-analytics_viz = AnalyticsVisualizer()
-
-# Centrality rankings
-from semantica.kg import CentralityCalculator
-centrality_calc = CentralityCalculator()
-centrality = centrality_calc.calculate_all_centrality(graph, centrality_types=["degree", "betweenness"])
-
-analytics_viz.visualize_centrality_rankings(centrality, centrality_type="degree", top_n=20,
- output="html", file_path="centrality_rankings.html")
-
-# Community structure
-from semantica.kg import CommunityDetector
-community_detector = CommunityDetector()
-communities = community_detector.detect_communities(graph)
-analytics_viz.visualize_community_structure(graph, communities,
- output="html", file_path="communities.html")
-
-# Connectivity analysis
-from semantica.kg import ConnectivityAnalyzer
-connectivity_analyzer = ConnectivityAnalyzer()
-connectivity = connectivity_analyzer.analyze_connectivity(graph)
-analytics_viz.visualize_connectivity(connectivity, output="html", file_path="connectivity.html")
-
-# Degree distribution
-analytics_viz.visualize_degree_distribution(graph, output="html", file_path="degree_distribution.html")
-
-# Metrics dashboard
-from semantica.kg import GraphAnalyzer
-analyzer = GraphAnalyzer()
-metrics = analyzer.compute_metrics(graph)
-analytics_viz.visualize_metrics_dashboard(metrics, output="html", file_path="metrics_dashboard.html")
-
-# Centrality comparison
-degree_centrality = centrality_calc.calculate_degree_centrality(graph)
-betweenness_centrality = centrality_calc.calculate_betweenness_centrality(graph)
-centrality_results = {
- "degree": degree_centrality,
- "betweenness": betweenness_centrality
-}
-analytics_viz.visualize_centrality_comparison(centrality_results, top_n=10,
- output="html", file_path="centrality_comparison.html")
-```
-
-#### Temporal Graph Visualization
-```python
-from semantica.visualization import TemporalVisualizer
-
-# Initialize temporal visualizer
-temporal_viz = TemporalVisualizer()
-
-# Timeline visualization
-temporal_data = {
- "events": [
- {"timestamp": "2023-01-15", "type": "entity_added", "entity": "Entity1"},
- {"timestamp": "2023-02-20", "type": "relationship_added", "entity": "Entity2"},
- ]
-}
-temporal_viz.visualize_timeline(temporal_data, output="html", file_path="timeline.html")
-
-# Temporal patterns
-from semantica.kg import TemporalPatternDetector
-pattern_detector = TemporalPatternDetector()
-patterns = pattern_detector.detect_patterns(temporal_graph)
-temporal_viz.visualize_temporal_patterns(patterns, output="html", file_path="temporal_patterns.html")
-
-# Snapshot comparison
-from semantica.kg import TemporalVersionManager
-version_manager = TemporalVersionManager()
-snapshots = {
- "2023-01-01": version_manager.get_snapshot("2023-01-01"),
- "2023-06-01": version_manager.get_snapshot("2023-06-01"),
- "2023-12-01": version_manager.get_snapshot("2023-12-01")
-}
-temporal_viz.visualize_snapshot_comparison(snapshots, output="html", file_path="snapshot_comparison.html")
-
-# Version history
-version_history = [
- {"version": "v1.0", "date": "2023-01-01", "changes": "Initial version"},
- {"version": "v1.1", "date": "2023-06-01", "changes": "Added new classes"},
- {"version": "v2.0", "date": "2023-12-01", "changes": "Major refactoring"}
-]
-temporal_viz.visualize_version_history(version_history, output="html", file_path="version_history.html")
-
-# Metrics evolution
-metrics_history = {
- "num_entities": [100, 150, 200, 250],
- "num_relationships": [200, 300, 400, 500],
- "density": [0.1, 0.12, 0.15, 0.18]
-}
-timestamps = ["2023-01-01", "2023-06-01", "2023-09-01", "2023-12-01"]
-temporal_viz.visualize_metrics_evolution(metrics_history, timestamps,
- output="html", file_path="metrics_evolution.html")
-```
-
-#### Quick Visualization Example
-
-```python
-from semantica.ingest import FileIngestor
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder
-from semantica.embeddings import TextEmbedder
-from semantica.visualization import KGVisualizer, EmbeddingVisualizer
-import numpy as np
-
-# Build knowledge graph using individual modules
-ingestor = FileIngestor()
-parser = DocumentParser()
-ner = NERExtractor()
-rel_extractor = RelationExtractor()
-builder = GraphBuilder()
-embedder = TextEmbedder()
-
-doc = ingestor.ingest_file("document.pdf")
-parsed = parser.parse_document("document.pdf")
-text = parsed.get("full_text", "")
-entities = ner.extract_entities(text)
-relationships = rel_extractor.extract_relations(text, entities=entities)
-kg = builder.build_graph(entities=entities, relationships=relationships)
-embeddings = embedder.embed_batch([e.text for e in entities])
-
-# Visualize knowledge graph
-kg_viz = KGVisualizer(layout="force", color_scheme="vibrant")
-kg_viz.visualize_network(
- result["knowledge_graph"],
- output="html",
- file_path="kg_visualization.html"
-)
-
-# Visualize embeddings
-if "embeddings" in result:
- emb_viz = EmbeddingVisualizer()
- embeddings_array = np.array([e["embedding"] for e in result["embeddings"]])
- emb_viz.visualize_2d_projection(
- embeddings_array,
- method="umap",
- output="html",
- file_path="embeddings_2d.html"
- )
-```
-
-## 🔧 Configuration
-
-### Basic Configuration
-```python
-from semantica.semantic_extract import NERExtractor
-from semantica.kg import GraphBuilder
-
-# Configure modules individually
-ner = NERExtractor(
- method="llm",
- provider="openai",
- model="gpt-4",
- confidence_threshold=0.7
-)
-
-builder = GraphBuilder(
- merge_entities=True,
- merge_threshold=0.9
-)
-```
-
-### Advanced Configuration
-```python
-from semantica.core import Config, ConfigManager
-from semantica.semantic_extract import NERExtractor
-from semantica.kg import GraphBuilder
-
-# Load configuration from file
-config_manager = ConfigManager()
-config = config_manager.load_from_file("config.yaml")
-
-# Use configuration with modules
-ner = NERExtractor(
- method="llm",
- provider=config.get("llm_provider.name"),
- model=config.get("llm_provider.model"),
- api_key=config.get("llm_provider.api_key")
- },
- "embedding_model": {
- "name": "sentence-transformers",
- "model": "all-MiniLM-L6-v2"
- },
- "vector_store": {
- "backend": "faiss",
- "index_type": "IVF"
- },
- "graph_db": {
- "backend": "neo4j",
- "uri": "bolt://localhost:7687",
- "username": "neo4j",
- "password": "password"
- }
-})
-
-# Use advanced configuration with Semantica
-semantica = Semantica(config=config)
-result = semantica.build_knowledge_base(["document.pdf"])
-```
-
-## 🚀 Performance
-
-### Benchmarks
-- **Processing Speed**: Optimized for high-throughput document processing
-- **Memory Usage**: Optimized for large-scale processing
-- **Accuracy**: High accuracy entity extraction
-- **Scalability**: Horizontal scaling support
-- **Latency**: Fast query response times
-
-### Optimization
-- **Parallel Processing**: Multi-threaded and multi-process support
-- **Caching**: Intelligent caching for improved performance
-- **Streaming**: Real-time processing capabilities
-- **GPU Support**: CUDA acceleration for deep learning models
-- **Cloud Integration**: Native cloud deployment support
-
-## 🔒 Security
-
-### Security Features
-- **Access Control**: Role-based access control (RBAC)
-- **Data Encryption**: End-to-end encryption support
-- **PII Protection**: Automatic PII detection and redaction
-- **Audit Logging**: Comprehensive audit trail
-- **Compliance**: GDPR, HIPAA, SOC2 compliance support
-
-### Privacy Protection
-- **Data Masking**: Automatic sensitive data masking
-- **Anonymization**: Data anonymization capabilities
-- **Secure Storage**: Encrypted data storage
-- **Access Logging**: Detailed access logging and monitoring
-
-## 🤝 Contributing
-
-We welcome contributions! Please see our [Contributing Guide](contributing.md) for details.
-
-### Development Setup
-```bash
-git clone https://github.com/semantica-agi/semantica.git
-cd semantica
-pip install -e ".[dev]"
-pre-commit install
-```
-
-### Running Tests
-```bash
-pytest tests/
-pytest tests/ -m "not slow"
-pytest tests/ -m "integration"
-```
-
-## 📄 License
-
-This project is licensed under the MIT License - see the [LICENSE](https://github.com/semantica-agi/semantica/blob/main/LICENSE) file for details.
-
-## 🙏 Acknowledgments
-
-- Built with ❤️ by the Semantica team
-- Powered by state-of-the-art NLP and ML libraries
-- Inspired by the open-source community
-- Special thanks to all contributors and users
-
-## 📞 Support
-
-- **Documentation**: [https://docs.semantica.dev](https://docs.semantica.dev)
-- **Issues**: [GitHub Issues](https://github.com/semantica-agi/semantica/issues)
-- **Discussions**: [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
-- **Email**: support@semantica.dev
-
-## 🌟 Star History
-
-[](https://star-history.com/#semantica-agi/semantica&Date)
-
----
-
-**Semantica** - Transform your data into intelligent knowledge. 🚀
diff --git a/docs/MIGRATION_V2.md b/docs/MIGRATION_V2.md
deleted file mode 100644
index 413806c2..00000000
--- a/docs/MIGRATION_V2.md
+++ /dev/null
@@ -1,158 +0,0 @@
----
-title: "Deduplication V2 Migration"
-description: "Migration guide for the Deduplication V2 engine with blocking_v2 and semantic_v2 strategies."
-icon: "arrows-rotate"
----
-
-## Semantica Deduplication V2: Migration & Performance Guide
-
-Welcome to the Deduplication V2 engine!! This release specifically targets severe CI delays and production bottlenecks caused by massive knowledge graph deduplication workloads. By introducing smarter candidate generation, fast-fail prefilters, and semantic triplet canonicalization, we have reduced worst-case execution times by up to **80%**.
-
-**Note:** This upgrade is **100% backward compatible.** All existing scripts, tests, and API signatures will continue to work exactly as they did before.
-
-
-
-To utilize this new addition, you must explicitly **opt-in** using the new configuration keys detailed below.
-
----
-
-### 1. Candidate Generation V2 (Beating the $O(N^2)$ Pair Explosion)
-
-**The Problem:** The legacy engine relied on a naive first-character blocking strategy. If your dataset contained 5,000 companies starting with letter "A", the engine generated nearly 12.5 million candidate pairs.
-
-**The V2 Solution:** Multi-key token blocking, prefix matching, and deterministic candidate budgeting.
-
-
-
-**How to Opt-In**
-
-Pass the keys into the `similarity`configuration dictionary when initializing the `DuplicateDetector`:
-
-
-
-```python
-from semantica.deduplication import DuplicateDetector
-
-detector = DuplicateDetector(
- similarity_threshold=0.8,
- similarity = {
- # Switches from legacy to v2
- "candidate_strategy": "blocking_v2",
-
- # Highly recommended: Limits the max number of comparisons
- # per entity to prevent adversarial latency spikes.
- "max_candidates_per_entity": 50,
-
- # Optional: Generates blocks using Soundex algorithm to catch
- # phonetic misspellings (e.g, "Jon" vs "John")
- "enable_phonetic_blocking": True
-}
-)
-```
-
-
-
-### 2. Two-Stage scoring (The Fast Prefilter)
-
-**The Problem**: Calculating multi-factor semantic scores (Levenshtein, Jaro-Winkler, property intersections, and Embeddings) is computationally expensive. Running these
-
-calculations on two entities that share absolutely zero words or have vastly different string lengths is a waste of resources.
-
-**The V2 Solution:** A lightning-fast prefilter gate that instantly drops obvious non-matches before they ever reach the heavy semantic scorers.
-
-
-
-**How to Opt-In**
-
-Enable the prefilter and define your rejection thresholds:
-
-
-
-```python
-from semantica.deduplication import DuplicateDetector
-
-detector = DuplicateDetector(
- similarity_threshold=0.8,
- similarity={
- "candidate_strategy": "blocking_v2",
-
- # Enable prefilter
- "prefilter_enabled": True,
-
- "prefilter_thresholds": {
- # Rejects pairs if shortest string is less than 40% the length
- # of the longest
- "min_length_ratio": 0.4,
-
- # Instantly rejects pairs if they don't share at least one
- # valid word token
- "required_shared_token": True
-},
- # Optional Explainability: Injects a 'score_breakdown' dict into
- # the candidate metadata so you can see exactly how the string,
- # property, and relationships scores contributed.
-
- "score_breakdown_enabled": True
-}
-)
-```
-
-
-
-### 3. Semantic Relationship & Triplet Deduplication
-
-**The problem:** The legacy relationship deduplication relied on exact `(Subject, Predicate, Object)` string matches. It couldn't recognize that `(Person, "works_for", Company)` is semantically identical to `(Person, "employed_by", Company)` .
-
-**The V2 Solution:** A new `semantic_v2` mode that introduces predicate synonym mapping, literal normalization (cleaning up rogue spaces/casing), and a highly optimized $O(1)$ canonical hash path for fast matching.
-
-
-
-**How to Opt-In**
-
-When calling relationship-specific dedup methods, pass the new configuration keys:
-
-```python
-from semantica.deduplication import DuplicateDetector
-from semantica.deduplication.methods import dedup_triplets
-
-
-# Approach A: Using the Detector explicitly
-detector = DuplicateDetector()
-duplicates = detector.detect_relationship_duplicates(
- relationship_list,
- relationship_dedup_mode="semantic_v2",
-
- # Cleans up messy object strings
- # (e.g., " Apple Inc. " -> "apple inc.")
- literal_normalization_enabled=True,
-
- # Maps various synonyms to a single canonical predicate
- # before hashing
- predicate_synonym_map={
- "works_for": "employed_by",
- "is_employee_of": "employed_by",
- "has_employer": "employed_by"
-}
-)
-
-
-# Approach B: Using the new simplified wrapper in methods.py
-duplicates = dedup_triplets(
- relationships_list,
- mode="semantic_v2",
- literal_normalization_enabled=True,
- predicate_synonym_map={"works_for": "employed_by"}
-)
-```
-
-
-
-###### Note on Merge Strategies
-
-When using `semantic_v2` for relationships, the `MergeStrategyManager` will now automatically respect your canonicalized keys. If two entities share a relationship that differs only by a mapped synonym, the engine will correctly identify them as the same relationship and prevent duplicate graph edges during the merge phase.
-
-
-
-### Need Help?
-
-If you experience any unexpected behavior when switching from `legacy` to `blocking_v2` or `semantic_v2`, please check the explainability metadata (by setting `"score_breakdown_enabled": True`) to audit the exact scoring process, or open an issue on GitHub.
\ No newline at end of file
diff --git a/docs/architecture.md b/docs/architecture.md
index 9cb9a2ff..48c5c711 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -193,7 +193,7 @@ Centralized `ConfigManager` with environment variable overrides. No magic defaul
Full module documentation with code examples.
-
+
Internals, algorithms, and advanced extension patterns.
diff --git a/docs/arrow_exporter.md b/docs/arrow_exporter.md
deleted file mode 100644
index fee500dd..00000000
--- a/docs/arrow_exporter.md
+++ /dev/null
@@ -1,273 +0,0 @@
----
-title: "Apache Arrow Exporter"
-description: "High-performance columnar export for knowledge graphs, entities, and relationships using Apache Arrow IPC format."
-icon: "file-arrow-down"
----
-
-## Overview
-
-The Apache Arrow exporter provides high-performance columnar data export for Semantica's knowledge graphs, entities, and relationships. It uses explicit schemas (no inference) and writes Arrow IPC files (.arrow) that are compatible with Pandas and DuckDB.
-
-## Features
-
-- **Explicit Schemas**: Pre-defined schemas for entities and relationships (no inference)
-- **Columnar Format**: Efficient storage and fast analytics
-- **Metadata Support**: Converts metadata dictionaries to Arrow struct fields
-- **Field Normalization**: Handles various entity and relationship field name variations
-- **Progress Tracking**: Integrated progress monitoring
-- **Error Handling**: Structured error handling with detailed logging
-- **Pandas/DuckDB Compatible**: Direct conversion to DataFrames and SQL queries
-
-## Installation
-
-The Arrow exporter requires PyArrow:
-
-```bash
-pip install pyarrow
-```
-
-## Usage
-
-### Basic Usage
-
-```python
-from semantica.export import ArrowExporter
-
-# Initialize exporter
-exporter = ArrowExporter()
-
-# Export entities
-entities = [
- {"id": "e1", "text": "Alice", "type": "Person", "confidence": 0.95},
- {"id": "e2", "text": "Acme Corp", "type": "Organization", "confidence": 0.88}
-]
-exporter.export_entities(entities, "entities.arrow")
-
-# Export relationships
-relationships = [
- {"id": "r1", "source_id": "e1", "target_id": "e2", "type": "WORKS_FOR"}
-]
-exporter.export_relationships(relationships, "relationships.arrow")
-
-# Export knowledge graph
-knowledge_graph = {
- "entities": entities,
- "relationships": relationships
-}
-exporter.export_knowledge_graph(knowledge_graph, "kg_base")
-# Creates: kg_base_entities.arrow, kg_base_relationships.arrow
-```
-
-### Using Convenience Function
-
-```python
-from semantica.export import export_arrow
-
-# Simple export
-export_arrow(entities, "entities.arrow")
-
-# Export multiple types
-data = {
- "entities": entities,
- "relationships": relationships
-}
-export_arrow(data, "output_base")
-```
-
-### With Compression
-
-```python
-# Use LZ4 compression
-exporter = ArrowExporter(compression="lz4")
-exporter.export_entities(entities, "entities_compressed.arrow")
-```
-
-## Schemas
-
-### Entity Schema
-
-```python
-ENTITY_SCHEMA = pa.schema([
- pa.field("id", pa.string(), nullable=False),
- pa.field("text", pa.string(), nullable=True),
- pa.field("type", pa.string(), nullable=True),
- pa.field("confidence", pa.float64(), nullable=True),
- pa.field("start", pa.int64(), nullable=True),
- pa.field("end", pa.int64(), nullable=True),
- pa.field("metadata", pa.struct([
- pa.field("keys", pa.list_(pa.string())),
- pa.field("values", pa.list_(pa.string()))
- ]), nullable=True),
-])
-```
-
-### Relationship Schema
-
-```python
-RELATIONSHIP_SCHEMA = pa.schema([
- pa.field("id", pa.string(), nullable=False),
- pa.field("source_id", pa.string(), nullable=False),
- pa.field("target_id", pa.string(), nullable=False),
- pa.field("type", pa.string(), nullable=True),
- pa.field("confidence", pa.float64(), nullable=True),
- pa.field("metadata", pa.struct([
- pa.field("keys", pa.list_(pa.string())),
- pa.field("values", pa.list_(pa.string()))
- ]), nullable=True),
-])
-```
-
-## Field Normalization
-
-The exporter automatically normalizes field names:
-
-**Entities:**
-- `text`, `label`, `name` → `text`
-- `type`, `entity_type` → `type`
-- `id`, `entity_id` → `id`
-- `start`, `start_offset` → `start`
-- `end`, `end_offset` → `end`
-
-**Relationships:**
-- `source`, `source_id` → `source_id`
-- `target`, `target_id` → `target_id`
-- `type`, `relationship_type` → `type`
-
-## Reading Arrow Files
-
-### With PyArrow
-
-```python
-import pyarrow as pa
-import pyarrow.ipc as ipc
-
-with pa.OSFile("entities.arrow", 'rb') as source:
- with ipc.open_file(source) as reader:
- table = reader.read_all()
- print(table.schema)
- print(table.to_pandas())
-```
-
-### With Pandas
-
-```python
-import pandas as pd
-import pyarrow.ipc as ipc
-
-with ipc.open_file("entities.arrow") as reader:
- df = reader.read_all().to_pandas()
- print(df)
-```
-
-### With DuckDB
-
-```python
-import duckdb
-
-# Query Arrow file directly
-result = duckdb.query("SELECT * FROM 'entities.arrow' WHERE type = 'Person'")
-print(result.df())
-```
-
-## Methods
-
-### `export(data, file_path, schema=None, **options)`
-
-Generic export method that handles both single and multiple files.
-
-**Parameters:**
-- `data`: List of dicts or dict with list values
-- `file_path`: Output file path (base path for dict exports)
-- `schema`: Optional Arrow schema (auto-detected if not provided)
-- `**options`: Additional options
-
-### `export_entities(entities, file_path, **options)`
-
-Export entities to Arrow IPC file with normalization.
-
-**Parameters:**
-- `entities`: List of entity dictionaries
-- `file_path`: Output Arrow file path
-- `**options`: Additional options
-
-### `export_relationships(relationships, file_path, **options)`
-
-Export relationships to Arrow IPC file with normalization.
-
-**Parameters:**
-- `relationships`: List of relationship dictionaries
-- `file_path`: Output Arrow file path
-- `**options`: Additional options
-
-### `export_knowledge_graph(knowledge_graph, base_path, **options)`
-
-Export knowledge graph to multiple Arrow files.
-
-**Parameters:**
-- `knowledge_graph`: Knowledge graph dictionary with 'entities' and 'relationships'
-- `base_path`: Base path for output files (without extension)
-- `**options`: Additional options
-
-## Examples
-
-See `examples/arrow_export_example.py` for comprehensive usage examples.
-
-## Testing
-
-Run the test suite:
-
-```bash
-# All Arrow exporter tests
-pytest tests/test_arrow_exporter.py -v
-
-# Integration tests
-pytest tests/test_export_module.py::TestExportModule::test_arrow_exporter -v
-```
-
-## Performance Benefits
-
-- **Columnar Storage**: Faster analytics on specific columns
-- **Compression**: Smaller file sizes (especially with LZ4/ZSTD)
-- **Zero-Copy**: Memory-efficient data transfer
-- **Cross-Language**: Works with Python, R, Julia, JavaScript, and more
-- **SQL Queries**: Direct querying with DuckDB without loading into memory
-
-## Comparison with Other Formats
-
-| Feature | Arrow | CSV | JSON |
-|---------|-------|-----|------|
-| Type Safety | ✓ | ✗ | ✗ |
-| Compression | ✓ | ✗ | ✗ |
-| Schema Validation | ✓ | ✗ | ✗ |
-| Pandas Compatible | ✓ | ✓ | ✓ |
-| DuckDB Native | ✓ | ✓ | ✗ |
-| Binary Format | ✓ | ✗ | ✗ |
-| Human Readable | ✗ | ✓ | ✓ |
-
-## Architecture
-
-The Arrow exporter follows Semantica's export architecture:
-
-1. **Normalization**: Field names are normalized to consistent format
-2. **Schema Application**: Explicit schemas ensure type safety
-3. **Metadata Conversion**: Dicts converted to Arrow struct fields
-4. **Progress Tracking**: Integrated with Semantica's progress tracker
-5. **Error Handling**: Structured exceptions with detailed messages
-
-## Contributing
-
-When contributing to the Arrow exporter:
-
-1. Maintain explicit schemas (no inference)
-2. Follow existing code style and patterns
-3. Add comprehensive tests for new features
-4. Update this documentation
-5. Ensure Pandas/DuckDB compatibility
-
-## License
-
-MIT License - See LICENSE file for details.
-
-## Author
-
-Semantica Contributors
diff --git a/docs/css/custom.css b/docs/css/custom.css
deleted file mode 100644
index 9b8b59ae..00000000
--- a/docs/css/custom.css
+++ /dev/null
@@ -1,315 +0,0 @@
-/* Semantica Documentation - Monochrome Pro Theme */
-
-/* Smooth scrolling */
-html {
- scroll-behavior: smooth;
-}
-
-/*
- ==========================================================================
- Color Variables - Monochrome Pro
- Primary: #212121 (Grey 900)
- Accent: #2962FF (Electric Blue)
- ==========================================================================
-*/
-:root {
- /* Light Mode */
- --md-default-bg-color: #FFFFFF;
- --md-default-fg-color: #212121;
- --md-default-fg-color--light: #616161;
- --md-default-fg-color--lighter: #9E9E9E;
- --md-default-fg-color--lightest: #E0E0E0;
-
- --md-primary-fg-color: #212121;
- --md-primary-fg-color--light: #484848;
- --md-primary-fg-color--dark: #000000;
-
- --md-accent-fg-color: #2962FF;
-
- /* Code blocks */
- --md-code-bg-color: #F5F5F5;
- --md-code-fg-color: #212121;
-}
-
-[data-md-color-scheme="slate"] {
- /* Dark Mode */
- --md-default-bg-color: #0F1115;
- --md-default-fg-color: #E0E0E0;
-
- --md-primary-fg-color: #0F1115;
- --md-primary-fg-color--light: #212121;
- --md-primary-fg-color--dark: #000000;
-}
-
-/*
- ==========================================================================
- Typography
- ==========================================================================
-*/
-.md-typeset h2 {
- font-weight: 700;
- letter-spacing: -0.01em;
- margin-top: 2rem;
- margin-bottom: 0.75rem;
-}
-
-.md-typeset p {
- line-height: 1.6;
- margin-bottom: 1rem;
-}
-
-/* Links */
-.md-typeset a {
- color: var(--md-accent-fg-color);
- text-decoration: none;
- font-weight: 500;
- background-color: #F1F8F5;
-}
-
-/*
- ==========================================================================
- Admonitions
- ==========================================================================
-*/
-/* Tip */
-.md-typeset .admonition.tip .admonition-title {
- color: #00C853;
-}
-
-[data-md-color-scheme="slate"] .md-typeset .admonition.tip {
- border-color: #2E303E;
- border-left-color: #69F0AE;
- background-color: #0E1B14;
-}
-
-[data-md-color-scheme="slate"] .md-typeset .admonition.tip .admonition-title {
- color: #69F0AE;
-}
-
-/* Warning */
-.md-typeset .admonition.warning {
- border-color: #E0E0E0;
- border-left-color: #FFAB00;
- background-color: #FFF8E1;
-}
-
-.md-typeset .admonition.warning .admonition-title {
- color: #FFAB00;
-}
-
-[data-md-color-scheme="slate"] .md-typeset .admonition.warning {
- border-color: #2E303E;
- border-left-color: #FFD740;
- background-color: #1F1B0E;
-}
-
-[data-md-color-scheme="slate"] .md-typeset .admonition.warning .admonition-title {
- color: #FFD740;
-}
-
-/* Danger */
-.md-typeset .admonition.danger {
- border-color: #E0E0E0;
- border-left-color: #FF1744;
- background-color: #FFEBEE;
-}
-
-.md-typeset .admonition.danger .admonition-title {
- color: #FF1744;
-}
-
-[data-md-color-scheme="slate"] .md-typeset .admonition.danger {
- border-color: #2E303E;
- border-left-color: #FF5252;
- background-color: #241214;
-}
-
-[data-md-color-scheme="slate"] .md-typeset .admonition.danger .admonition-title {
- color: #FF5252;
-}
-
-/*
- ==========================================================================
- Code Blocks
- ==========================================================================
-*/
-.md-typeset pre {
- background-color: var(--md-code-bg-color);
- border: 1px solid rgba(0, 0, 0, 0.05);
- border-radius: 6px;
-}
-
-[data-md-color-scheme="slate"] .md-typeset pre {
- border-color: rgba(255, 255, 255, 0.05);
-}
-
-/*
- ==========================================================================
- Scrollbars
- ==========================================================================
-*/
-::-webkit-scrollbar {
- width: 6px;
- height: 6px;
-}
-
-::-webkit-scrollbar-thumb {
- background-color: rgba(0, 0, 0, 0.2);
- border-radius: 3px;
-}
-
-[data-md-color-scheme="slate"] ::-webkit-scrollbar-thumb {
- background-color: #2962FF;
-}
-
-/*
- ==========================================================================
- Footer Attribution - Keep MkDocs Credit Visible
- ==========================================================================
-*/
-.md-footer-meta__inner {
- display: flex;
- flex-wrap: wrap;
- justify-content: space-between;
- align-items: center;
-}
-
-.md-footer-copyright {
- opacity: 1 !important;
- color: var(--md-default-fg-color--light) !important;
-}
-
-.md-footer-copyright__highlight {
- opacity: 1 !important;
- color: var(--md-default-fg-color) !important;
- font-weight: 500 !important;
-}
-
-[data-md-color-scheme="slate"] .md-footer-copyright {
- color: rgba(255, 255, 255, 0.7) !important;
-}
-
-[data-md-color-scheme="slate"] .md-footer-copyright__highlight {
- color: rgba(255, 255, 255, 0.9) !important;
-}
-
-/*
- ==========================================================================
- Active Link Highlighting
- ==========================================================================
-*/
-/* Left Sidebar (Navigation) - Active Link */
-.md-nav__link--active {
- color: var(--md-accent-fg-color) !important;
- font-weight: bold;
-}
-
-/* Right Sidebar (Table of Contents) - Active Link */
-.md-nav__item--active > .md-nav__link {
- color: var(--md-accent-fg-color) !important;
- border-left: 2px solid var(--md-accent-fg-color);
- padding-left: 0.5rem;
-}
-
-/* Ensure nested items in TOC don't inherit the border unless active themselves */
-.md-nav__item .md-nav__item--active > .md-nav__link {
- border-left: 2px solid var(--md-accent-fg-color);
-}
-
-/*
- ==========================================================================
- Layout Optimization
- ==========================================================================
-*/
-
-/* Widen the overall grid */
-.md-grid {
- max-width: 1440px;
- margin-left: auto;
- margin-right: auto;
- padding-left: 0.5rem;
-}
-
-/* Narrow left sidebar to give content more room */
-.md-sidebar--primary {
- width: 11rem;
- padding-right: 0.25rem;
- padding-left: 0.25rem;
-}
-
-/* Right TOC sidebar */
-.md-sidebar--secondary {
- width: 11rem;
- padding-left: 0.5rem;
- padding-right: 0;
- margin-left: 0;
-}
-
-.md-sidebar--secondary .md-nav {
- width: 11rem;
-}
-
-/* Tighten TOC list spacing */
-.md-sidebar--secondary .md-nav__list {
- padding-bottom: 1.5rem;
- margin: 0;
-}
-
-.md-sidebar--secondary .md-nav__item {
- padding: 0;
- margin: 0;
-}
-
-.md-sidebar--secondary .md-nav__link {
- white-space: normal;
- word-break: break-word;
- overflow: visible;
- text-overflow: unset;
- padding-top: 0.15rem;
- padding-bottom: 0.15rem;
- line-height: 1.4;
- font-size: 0.7rem;
- margin: 0;
-}
-
-/* Nested TOC items (h3, h4) */
-.md-sidebar--secondary .md-nav__item .md-nav__item .md-nav__link {
- padding-left: 0.6rem;
- font-size: 0.68rem;
-}
-
-/* Remove extra gap between TOC title and first item */
-.md-sidebar--secondary .md-nav__title {
- margin-bottom: 0.25rem;
- padding-bottom: 0.25rem;
-}
-
-/* Give the main content area maximum available width */
-.md-content {
- max-width: none;
- padding-left: 1rem;
- padding-right: 1rem;
-}
-
-.md-content__inner {
- max-width: none;
- padding-left: 1rem;
- padding-right: 1rem;
- margin-left: 0;
- margin-right: 0;
-}
-
-.md-main__inner {
- margin-left: 0;
- margin-right: 0;
-}
-
-/* Ensure text content is left-aligned by default */
-.md-typeset {
- text-align: left;
-}
-
-/* Keep hero section centered */
-.md-typeset > div[align="center"] {
- text-align: center;
-}
diff --git a/docs/deep-dive.md b/docs/deep-dive.md
deleted file mode 100644
index 8c67ad49..00000000
--- a/docs/deep-dive.md
+++ /dev/null
@@ -1,208 +0,0 @@
----
-title: "Deep Dive"
-description: "Internals, advanced concepts, and extension points for contributors and power users."
-icon: "microscope"
----
-
-> Internals, advanced concepts, and extension points for contributors and power users.
-
-
-New to Semantica? Read the [Architecture](architecture) overview first for a higher-level picture.
-
-
----
-
-## Pipeline Internals
-
-Full data flow through a Semantica pipeline:
-
-```text
-Data Sources
- └─ Ingestion Layer (FileIngestor, WebIngestor, SnowflakeIngestor, StreamIngestor)
- └─ Parsing Layer (DocumentParser, DoclingParser, OCR)
- └─ Extraction (NER → Entity Linking → Validation)
- └─ Normalization
- └─ Conflict Resolution
- └─ Knowledge Graph Builder
- └─ Embedding Generator
- └─ Export Layer
-```
-
----
-
-## System Components
-
-### Ingestion Layer
-
-- **FileIngestor** — PDF, DOCX, HTML, JSON, CSV, TXT, Parquet (v0.5.0), XML (v0.5.0), archives
-- **WebIngestor** — URL crawling and scraping
-- **SnowflakeIngestor** — SQL databases and cloud warehouses
-- **StreamIngestor** — Kafka and real-time feeds
-
-### Parsing Layer
-
-- Text and metadata extraction from documents
-- OCR for scanned content
-- Layout analysis via Docling (tables, columns, headers)
-
-### Extraction Layer
-
-```text
-text → Tokenization → NER → Entity Linking → Entity Validation
-```
-
-Components: Named Entity Recognition, Relationship Extraction, Triplet Extraction, Coreference Resolution.
-
-### Normalization Layer
-
-Standardizes entity names, date formats, numbers, encodings, and language. Includes the v0.5.0 cp1252 encoding fix for Windows environments.
-
-### Conflict Resolution
-
-Multiple source facts that contradict each other are resolved using one of four strategies:
-
-| Strategy | Behavior |
-|----------|----------|
-| `voting` | Most common value wins |
-| `credibility_weighted` | Higher-credibility source wins |
-| `most_recent` | Latest timestamp wins |
-| `highest_confidence` | Highest extraction confidence wins |
-
-### Knowledge Graph Builder
-
-- Entity resolution across sources
-- Edge creation with typed relationships
-- Property assignment with confidence scores
-- Graph validation and quality checks
-
-### Embedding Generator
-
-- Text embeddings: Sentence-Transformers, FastEmbed, OpenAI, BGE
-- Graph embeddings: Node2Vec, GraphSAGE
-- Distance caching for Distance Intelligence (v0.5.0)
-
----
-
-## Advanced Concepts
-
-### Entity Resolution
-
-```python
-def resolve_entities(entities, threshold=0.85):
- clusters = []
- for entity in entities:
- matched = False
- for cluster in clusters:
- if similarity(entity, cluster.representative) > threshold:
- cluster.add(entity)
- matched = True
- break
- if not matched:
- clusters.append(EntityCluster(entity))
- return clusters
-```
-
-### Relationship Inference
-
-Semantica's reasoning engines derive implicit relationships:
-
-- **Transitive** — if A→B and B→C, infer A→C
-- **Temporal** — before/after/during from timestamped facts (Allen Interval Algebra)
-- **Causal** — IF/THEN rules via `Reasoner`
-- **Hierarchical** — subclass/instance inference via `OntologyReasoner`
-- **Datalog** — recursive rules with termination guarantee (v0.4.0)
-
-### Batch Processing for Large Datasets
-
-```python
-def process_large_dataset(sources, batch_size=100):
- for i in range(0, len(sources), batch_size):
- batch = sources[i : i + batch_size]
- result = semantica.build_knowledge_base(batch)
- save_result(result)
- del result
- gc.collect()
-```
-
----
-
-## Extension Points
-
-### Custom Plugin
-
-```python
-from semantica.core import Plugin
-
-class CustomPlugin(Plugin):
- def initialize(self):
- ...
-
- def process(self, data):
- return processed_data
-```
-
-### Custom Extractor
-
-```python
-from semantica.semantic_extract import BaseExtractor
-
-class DomainSpecificExtractor(BaseExtractor):
- def extract(self, text):
- # Domain-specific entity extraction logic
- return entities
-```
-
-### Custom Ingestor
-
-```python
-from semantica.ingest import BaseIngestor
-
-class CustomIngestor(BaseIngestor):
- def ingest(self, source):
- # Load and return document dicts
- return documents
-```
-
----
-
-## Internal APIs
-
-| API | Purpose |
-|-----|---------|
-| `Semantica.build_knowledge_base()` | Main orchestration entry point |
-| `GraphBuilder.build()` | Graph construction |
-| `ConflictResolver.resolve()` | Conflict resolution |
-| `EmbeddingGenerator.generate()` | Embedding generation |
-
-Extension hooks: plugin registration, custom extractor registration, custom exporter registration, event hooks.
-
----
-
-## Design Decisions
-
-**Why modular architecture?** Each component is independently testable and swappable. You can use `NERExtractor` alone without pulling in graph storage or pipelines.
-
-**Why built-in conflict resolution?** Multi-source data always has contradictions. Ignoring them produces low-quality graphs. Explicit strategies give you control over data quality.
-
-**Why W3C PROV-O for provenance?** It's an industry standard with broad tooling support. A custom format would make lineage data non-portable.
-
-**Why multiple reasoning engines?** Different problems need different reasoning: forward chaining for rule application, SPARQL for graph queries, abductive for hypothesis generation, Datalog for recursive rules.
-
----
-
-## See Also
-
-
-
- Every module with code examples.
-
-
- Framework orchestration internals.
-
-
- Pipeline DSL and execution model.
-
-
- How to extend the framework.
-
-
diff --git a/docs/docs.json b/docs/docs.json
index e40300bf..e9324bf4 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -68,9 +68,7 @@
{
"group": "Guides",
"pages": [
- "examples",
"architecture",
- "deep-dive",
"learning-more"
]
},
@@ -91,7 +89,6 @@
{
"group": "Vector Stores",
"pages": [
- "vector_store_usage",
"vector_stores/pgvector"
]
}
diff --git a/docs/examples.md b/docs/examples.md
deleted file mode 100644
index b9b8170f..00000000
--- a/docs/examples.md
+++ /dev/null
@@ -1,253 +0,0 @@
----
-title: "Examples"
-description: "Code examples organized by complexity — beginner through production."
-icon: "code"
----
-
-> Code examples organized by complexity. For interactive notebooks, see the [Cookbook](cookbook).
-
----
-
-## Beginner
-
-### Basic Knowledge Graph
-
-```python
-from semantica.ingest import FileIngestor
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder
-
-ingestor = FileIngestor()
-parser = DocumentParser()
-ner = NERExtractor()
-rel = RelationExtractor()
-
-sources = ingestor.ingest("data/sample.pdf")
-parsed = parser.parse(sources[0])
-
-entities = ner.extract(parsed)
-relationships = rel.extract(parsed, entities=entities)
-
-kg = GraphBuilder(merge_entities=True).build(
- entities=entities, relationships=relationships
-)
-print(f"{len(kg.nodes)} nodes, {len(kg.edges)} edges")
-```
-
-### Entity Extraction from Text
-
-```python
-from semantica.semantic_extract import NERExtractor
-
-ner = NERExtractor()
-entities = ner.extract("Apple Inc. was founded by Steve Jobs in 1976.")
-
-for entity in entities:
- print(f"{entity['text']}: {entity['type']}")
-# Apple Inc.: ORGANIZATION
-# Steve Jobs: PERSON
-# 1976: DATE
-```
-
-### Custom NER with LLM
-
-```python
-from semantica.semantic_extract import NERExtractor
-from semantica.llms import OpenAI
-
-llm = OpenAI(model="gpt-4o", api_key=os.getenv("OPENAI_API_KEY"))
-ner = NERExtractor(method="llm", llm_provider=llm, confidence_threshold=0.8)
-entities = ner.extract("Your document text here...")
-```
-
----
-
-## Intermediate
-
-### Multi-Source Integration
-
-```python
-from semantica.ingest import FileIngestor
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder
-
-ingestor = FileIngestor()
-parser = DocumentParser()
-ner = NERExtractor()
-rel = RelationExtractor()
-builder = GraphBuilder(merge_entities=True)
-
-all_entities, all_rels = [], []
-
-for path in ["source1.pdf", "source2.pdf", "source3.pdf"]:
- sources = ingestor.ingest(path)
- parsed = parser.parse(sources[0])
- all_entities.extend(ner.extract(parsed))
- all_rels.extend(rel.extract(parsed, entities=all_entities))
-
-kg = builder.build(entities=all_entities, relationships=all_rels)
-print(f"Unified graph: {len(kg.nodes)} nodes, {len(kg.edges)} edges")
-```
-
-### Conflict Detection and Resolution
-
-```python
-from semantica.conflicts import ConflictDetector, ConflictResolver
-
-detector = ConflictDetector()
-conflicts = detector.detect_conflicts(all_entities)
-
-resolver = ConflictResolver(default_strategy="voting")
-resolved = resolver.resolve_conflicts(conflicts)
-
-print(f"Detected {len(conflicts)} conflicts, resolved {len(resolved)}")
-```
-
-### Parquet and XML Ingestion (v0.5.0)
-
-```python
-from semantica.ingest import ParquetIngestor, XMLIngestor
-
-parquet_data = ParquetIngestor().ingest("data/records.parquet")
-xml_data = XMLIngestor(safe_mode=True).ingest("data/feed.xml")
-```
-
-### Persistent Storage — Neo4j
-
-```python
-from semantica.graph_store import GraphStore
-
-store = GraphStore(
- backend="neo4j",
- uri="bolt://localhost:7687",
- user="neo4j",
- password="password",
-)
-store.connect()
-
-apple = store.create_node(labels=["Company"], properties={"name": "Apple Inc."})
-tim = store.create_node(labels=["Person"], properties={"name": "Tim Cook"})
-store.create_relationship(
- start_node_id=tim["id"],
- end_node_id=apple["id"],
- rel_type="CEO_OF",
-)
-store.close()
-```
-
----
-
-## Advanced
-
-### GraphRAG with Reasoning
-
-```python
-from semantica.context import AgentContext
-from semantica.reasoning import Reasoner
-
-context = AgentContext(
- vector_store=vs,
- knowledge_graph=kg,
- graph_expansion=True,
- hybrid_alpha=0.7,
-)
-
-reasoner = Reasoner()
-reasoner.add_rule("IF Library(?x) AND Language(?y) THEN TechStackItem(?x)")
-inferred = reasoner.infer_facts(kg.get_all_triplets())
-
-for fact in inferred:
- kg.add_fact_from_string(fact)
-
-results = context.retrieve("What technologies are used in this project?")
-```
-
-### Temporal Knowledge Graph (v0.4.0)
-
-```python
-from semantica.kg import TemporalKnowledgeGraph
-
-tkg = TemporalKnowledgeGraph()
-tkg.add_temporal_fact("Apple", "CEO", "Tim Cook", valid_from="2011-08-24")
-tkg.add_temporal_fact("Apple", "CEO", "Steve Jobs", valid_from="1997-09-16", valid_to="2011-08-24")
-
-ceo_2005 = tkg.query_at("Apple", "CEO", timestamp="2005-01-01")
-```
-
-### Distance Intelligence (v0.5.0)
-
-```python
-from semantica.kg import DistanceCalculator
-
-calc = DistanceCalculator(kg)
-dist = calc.calculate("Apple Inc.", "Microsoft")
-
-print(f"Distance: {dist.score:.3f} — Band: {dist.band}")
-similar = calc.find_similar("Apple Inc.", radius=0.3)
-```
-
----
-
-## Production
-
-### Batch Processing (Large Datasets)
-
-```python
-from semantica.pipeline import Pipeline
-from semantica.ingest import FileIngestor
-from semantica.parse import DocumentParser
-from semantica.semantic_extract import NERExtractor
-from semantica.kg import GraphBuilder
-
-pipeline = Pipeline(workers=4)
-pipeline.add_step("ingest", FileIngestor())
-pipeline.add_step("parse", DocumentParser())
-pipeline.add_step("extract", NERExtractor(), parallel=True, batch_size=50)
-pipeline.add_step("build", GraphBuilder())
-
-result = pipeline.run("data/")
-print(f"Processed: {result.processed_count}, Failed: {result.failed_count}")
-```
-
-### Real-Time Streaming
-
-```python
-from semantica.ingest import StreamIngestor
-from semantica.semantic_extract import NERExtractor, RelationExtractor
-from semantica.kg import GraphBuilder
-
-stream = StreamIngestor(stream_uri="kafka://localhost:9092/topic")
-ner = NERExtractor()
-rel = RelationExtractor()
-builder = GraphBuilder()
-
-for batch in stream.stream(batch_size=100):
- all_entities, all_rels = [], []
- for item in batch:
- text = str(item)
- all_entities.extend(ner.extract(text))
- all_rels.extend(rel.extract(text, entities=all_entities))
- kg = builder.build(entities=all_entities, relationships=all_rels)
- print(f"Processed batch: {len(kg.nodes)} nodes")
-```
-
----
-
-## See Also
-
-
-
- Step-by-step first pipeline tutorial.
-
-
- Interactive Jupyter notebook tutorials.
-
-
- Domain-specific examples.
-
-
- Complete API documentation.
-
-
diff --git a/docs/netlify.toml b/docs/netlify.toml
deleted file mode 100644
index ed2871b0..00000000
--- a/docs/netlify.toml
+++ /dev/null
@@ -1,12 +0,0 @@
-[build]
- command = "pip install -r requirements-docs.txt && mkdocs build"
- publish = "site"
-
-[build.environment]
- PYTHON_VERSION = "3.11"
-
-[[redirects]]
- from = "/*"
- to = "/index.html"
- status = 200
-
diff --git a/docs/quickstart.md b/docs/quickstart.md
index 5d531a97..b8a7121d 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -428,7 +428,7 @@ pip install --upgrade semantica
Knowledge graphs, ontologies, reasoning engines — the mental model behind Semantica.
-
+
15+ copy-paste examples for healthcare, finance, legal, and cybersecurity.
diff --git a/docs/vector_store_usage.md b/docs/vector_store_usage.md
deleted file mode 100644
index bd2124fe..00000000
--- a/docs/vector_store_usage.md
+++ /dev/null
@@ -1,117 +0,0 @@
----
-title: "Vector Store: High-Performance Usage"
-description: "Parallel ingestion, batch processing, and performance tuning for the Semantica Vector Store."
-icon: "bolt"
----
-
-> High-performance batch ingestion with parallel embedding generation — 3–10× faster than sequential processing.
-
----
-
-## Key Features
-
-- **Parallel ingestion** — multi-threaded embedding generation and storage
-- **Batch processing** — minimizes overhead by grouping documents into chunks
-- **Unified API** — `add_documents` handles embedding generation and storage in one call
-
----
-
-## Quick Start: Parallel Ingestion
-
-```python
-from semantica.vector_store import VectorStore
-import time
-
-store = VectorStore(backend="faiss", dimension=768)
-
-documents = [f"This is document number {i} with some content." for i in range(1000)]
-metadata = [{"source": "generated", "id": i} for i in range(1000)]
-
-start = time.time()
-ids = store.add_documents(
- documents=documents,
- metadata=metadata,
- batch_size=64,
- parallel=True, # default: True
-)
-print(f"Ingested {len(ids)} documents in {time.time() - start:.2f}s")
-```
-
----
-
-## Performance Comparison
-
-**Old method (sequential loop)** — slower due to per-item overhead:
-
-```python
-for doc in documents:
- emb = embedder.generate(doc)
- store.store_vectors([emb], [{"text": doc}])
-```
-
-**New method (parallel batching)** — 3–10× faster:
-
-```python
-store.add_documents(documents, parallel=True)
-```
-
----
-
-## Configuration and Tuning
-
-### `max_workers`
-
-Number of concurrent threads for embedding generation.
-
-- **Default**: 6 (optimized for most systems)
-- Override only if you have very high core counts or specific throughput needs
-
-```python
-store = VectorStore(max_workers=16)
-```
-
-### `batch_size`
-
-Number of documents processed in a single chunk.
-
-- **Default**: 32
-- **Local models**: 32–64 works well
-- **API models (OpenAI, etc.)**: 100–200 reduces network latency overhead
-
-```python
-store.add_documents(documents, batch_size=100)
-```
-
----
-
-## Manual Batch Embedding
-
-If you need embeddings without immediately storing them:
-
-```python
-vectors = store.embed_batch(texts=documents[:100])
-print(f"Generated {len(vectors)} vectors")
-```
-
----
-
-## Best Practices
-
-
-- **Metadata consistency** — ensure your `metadata` list is the same length as `documents`.
-- **Error handling** — `add_documents` propagates exceptions if embedding fails; validate your data first.
-- **Memory usage** — very large `batch_size` combined with high `max_workers` increases RAM usage. Monitor system resources for large corpora.
-
-
----
-
-## See Also
-
-
-
- Full VectorStore API with all backends.
-
-
- Embedding providers and GPU acceleration.
-
-
diff --git a/docs_check.py b/docs_check.py
new file mode 100644
index 00000000..1af80201
--- /dev/null
+++ b/docs_check.py
@@ -0,0 +1,233 @@
+"""Docs integrity checker for PR #561."""
+import json
+import os
+import re
+import glob
+
+DOCS_DIR = "docs"
+
+results = {"pass": [], "fail": []}
+
+
+def ok(msg):
+ results["pass"].append(msg)
+ print(f" PASS {msg}")
+
+
+def fail(msg):
+ results["fail"].append(msg)
+ print(f" FAIL {msg}")
+
+
+# ── 1. docs.json valid JSON ───────────────────────────────────────────────────
+print("\n[1] docs.json validity")
+try:
+ with open(os.path.join(DOCS_DIR, "docs.json"), encoding="utf-8") as f:
+ cfg = json.load(f)
+ ok("docs.json is valid JSON")
+except Exception as e:
+ fail(f"docs.json parse error: {e}")
+ cfg = {}
+
+
+# ── 2. All nav pages exist on disk ────────────────────────────────────────────
+print("\n[2] Nav pages exist on disk")
+
+
+def collect_pages(obj):
+ pages = []
+ if isinstance(obj, dict):
+ if "pages" in obj:
+ for p in obj["pages"]:
+ if isinstance(p, str):
+ pages.append(p)
+ else:
+ pages.extend(collect_pages(p))
+ for v in obj.values():
+ if isinstance(v, (dict, list)):
+ pages.extend(collect_pages(v))
+ elif isinstance(obj, list):
+ for item in obj:
+ pages.extend(collect_pages(item))
+ return pages
+
+
+nav_pages = [p for p in set(collect_pages(cfg)) if not p.startswith("http")]
+missing_pages = []
+for p in sorted(nav_pages):
+ path = os.path.join(DOCS_DIR, p + ".md")
+ if not os.path.exists(path):
+ missing_pages.append(p)
+
+if missing_pages:
+ for m in missing_pages:
+ fail(f"Nav page missing: {m}")
+else:
+ ok(f"All {len(nav_pages)} nav pages exist on disk")
+
+
+# ── 3. Internal Card hrefs resolve (page-relative) ───────────────────────────
+print("\n[3] Internal Card hrefs")
+broken_hrefs = []
+
+for fpath in glob.glob(DOCS_DIR + "/**/*.md", recursive=True):
+ file_dir = os.path.dirname(fpath)
+ with open(fpath, encoding="utf-8") as f:
+ content = f.read()
+ for m in re.finditer(r'href=["\'](?!http)([^"\'#]+)["\']', content):
+ href = m.group(1).strip()
+ if not href:
+ continue
+ # Resolve relative to the file's directory (mirrors browser URL resolution)
+ resolved = os.path.normpath(os.path.join(file_dir, href))
+ target_md = resolved + ".md"
+ if not os.path.exists(target_md):
+ rel = fpath.replace("\\", "/")
+ broken_hrefs.append(f"{rel}: href '{href}' -> {target_md}")
+
+if broken_hrefs:
+ for b in broken_hrefs[:20]:
+ fail(f"Broken href: {b}")
+ if len(broken_hrefs) > 20:
+ fail(f"...and {len(broken_hrefs) - 20} more broken hrefs")
+else:
+ ok("All internal Card hrefs resolve to real files")
+
+
+# ── 4. No old repo URLs ───────────────────────────────────────────────────────
+print("\n[4] Repo URL consistency")
+old_patterns = ["Hawksight-AI/semantica", "semantica-dev/semantica"]
+old_url_hits = []
+for fpath in glob.glob(DOCS_DIR + "/**/*.md", recursive=True) + [
+ os.path.join(DOCS_DIR, "docs.json")
+]:
+ with open(fpath, encoding="utf-8") as f:
+ content = f.read()
+ for pat in old_patterns:
+ if pat in content:
+ old_url_hits.append(f"{fpath}: contains '{pat}'")
+
+if old_url_hits:
+ for h in old_url_hits:
+ fail(h)
+else:
+ ok("No old repo URLs found (Hawksight-AI, semantica-dev)")
+
+
+# ── 5. All reference .md files have frontmatter ───────────────────────────────
+print("\n[5] Reference page frontmatter")
+ref_pages = list(glob.glob(DOCS_DIR + "/reference/*.md"))
+no_frontmatter = []
+for fpath in ref_pages:
+ with open(fpath, encoding="utf-8") as f:
+ content = f.read()
+ if not content.startswith("---"):
+ no_frontmatter.append(os.path.basename(fpath))
+
+if no_frontmatter:
+ for f in no_frontmatter:
+ fail(f"Missing frontmatter: {f}")
+else:
+ ok(f"All {len(ref_pages)} reference pages have frontmatter")
+
+
+# ── 6. Code examples: no non-existent class names (exact word match) ──────────
+print("\n[6] Known-wrong class names")
+# (symbol, file, exclude_pattern) — exclude_pattern avoids substring false positives
+banned = [
+ ("BaseIngestor", "docs/architecture.md", None),
+ ("BaseExtractor", "docs/architecture.md", None),
+ ("BasePlugin", "docs/architecture.md", None),
+ (r"PluginRegistry\.register\(", "docs/architecture.md", r"register_plugin"),
+ ("start_explorer", "docs/reference/explorer.md", None),
+ (r"graph\.save\(", "docs/reference/explorer.md", None),
+ (r"\bDataNormalizer\b", "docs/reference/normalize.md", None),
+ (r"\bEntityResolver\b", "docs/reference/deduplication.md", None),
+ # ReasoningEngine: only flag as exact word, not as part of TemporalReasoningEngine
+ (r"(? 10:
+ fail(f"...and {len(py39_hits) - 10} more")
+else:
+ ok("No Python 3.9+ lowercase generic type hints in doc code blocks")
+
+
+# ── 8. Module table covers all 27 modules ─────────────────────────────────────
+print("\n[8] Module table coverage in index.md")
+expected_modules = [
+ "semantica.ingest", "semantica.parse", "semantica.split", "semantica.normalize",
+ "semantica.semantic_extract", "semantica.kg", "semantica.ontology", "semantica.reasoning",
+ "semantica.embeddings", "semantica.vector_store", "semantica.graph_store", "semantica.triplet_store",
+ "semantica.context", "semantica.provenance", "semantica.change_management",
+ "semantica.deduplication", "semantica.conflicts", "semantica.export", "semantica.visualization",
+ "semantica.pipeline", "semantica.seed", "semantica.llms", "semantica.mcp_server",
+ "semantica.explorer", "semantica.evals", "semantica.utils", "semantica.core",
+]
+index_path = os.path.join(DOCS_DIR, "index.md")
+with open(index_path, encoding="utf-8") as f:
+ index_content = f.read()
+
+missing_modules = [m for m in expected_modules if m not in index_content]
+if missing_modules:
+ for m in missing_modules:
+ fail(f"Module missing from index table: {m}")
+else:
+ ok(f"All {len(expected_modules)} modules present in index.md table")
+
+
+# ── Summary ──────────────────────────────────────────────────────────────────
+print(f"\n{'='*60}")
+print(f"Results: {len(results['pass'])} passed, {len(results['fail'])} failed")
+if results["fail"]:
+ print("STATUS: FAIL")
+ raise SystemExit(1)
+else:
+ print("STATUS: ALL CHECKS PASSED")
diff --git a/mkdocs.yml b/mkdocs.yml
deleted file mode 100644
index ab2234f8..00000000
--- a/mkdocs.yml
+++ /dev/null
@@ -1,161 +0,0 @@
-# Deployment Trigger: Public GitHub Pages
-site_name: Semantica
-site_description: Open Source Framework for Semantic Intelligence & Knowledge Engineering
-site_url: https://hawksight-ai.github.io/semantica/
-repo_url: https://github.com/Hawksight-AI/semantica
-repo_name: Hawksight-AI/semantica
-edit_uri: edit/main/docs/
-
-# Copyright
-copyright: Copyright © 2026 Hawksight AI
-
-# Theme Configuration
-theme:
- name: material
- palette:
- # Light mode
- - scheme: default
- primary: custom
- accent: custom
- toggle:
- icon: material/brightness-7
- name: Switch to dark mode
- # Dark mode
- - scheme: slate
- primary: custom
- accent: custom
- toggle:
- icon: material/brightness-4
- name: Switch to light mode
- features:
- - navigation.tabs
- - navigation.sections
- - navigation.expand
- - navigation.top
- - navigation.indexes
- - navigation.tracking
- - search.suggest
- - search.highlight
- - search.share
- - content.code.copy
- - content.code.annotate
- - content.tooltips
- icon:
- logo: material/brain
- repo: fontawesome/brands/github
-
-# Extensions
-markdown_extensions:
- - pymdownx.highlight:
- anchor_linenums: true
- line_spans: __span
- pygments_lang_class: true
- - pymdownx.inlinehilite
- - pymdownx.snippets:
- base_path: ["."]
- - pymdownx.superfences:
- custom_fences:
- - name: mermaid
- class: mermaid
- format: !!python/name:pymdownx.superfences.fence_code_format
- - pymdownx.emoji:
- emoji_index: !!python/name:material.extensions.emoji.twemoji
- emoji_generator: !!python/name:material.extensions.emoji.to_svg
- - pymdownx.tabbed:
- alternate_style: true
- - pymdownx.tasklist:
- custom_checkbox: true
- - admonition
- - pymdownx.details
- - attr_list
- - md_in_html
- - tables
- - toc:
- permalink: true
- toc_depth: 6
-
-# Plugins
-plugins:
- - search:
- lang: en
- - minify:
- minify_html: true
- - mkdocstrings:
- handlers:
- python:
- options:
- docstring_style: google
- show_source: true
- show_root_heading: true
- show_category_heading: true
- - mkdocs-jupyter:
- include_source: true
-
-# Custom CSS
-extra_css:
- - css/custom.css
-
-# Custom JavaScript
-extra_javascript: []
-
-# Navigation
-nav:
- - Home: index.md
- - Getting Started:
- - Overview: getting-started.md
- - installation.md
- - quickstart.md
- - Docs:
- - Change Management: reference/change_management.md
- - Conflicts: reference/conflicts.md
- - Context: reference/context.md
- - Core: reference/core.md
- - Deduplication: reference/deduplication.md
- - Embeddings: reference/embeddings.md
- - Evals: reference/evals.md
- - Export: reference/export.md
- - Graph Store: reference/graph_store.md
- - Ingest: reference/ingest.md
- - Knowledge Graph: reference/kg.md
- - LLMs: reference/llms.md
- - Normalize: reference/normalize.md
- - Ontology: reference/ontology.md
- - Parse: reference/parse.md
- - Pipeline: reference/pipeline.md
- - Provenance: reference/provenance.md
- - Reasoning: reference/reasoning.md
- - Seed: reference/seed.md
- - Semantic Extract: reference/semantic_extract.md
- - Split: reference/split.md
- - Triplet Store: reference/triplet_store.md
- - Utils: reference/utils.md
- - Vector Store: reference/vector_store.md
- - Visualization: reference/visualization.md
- - Guides:
- - concepts.md
- - modules.md
- - use-cases.md
- - examples.md
- - glossary.md
- - Integrations:
- - Agno: integrations/agno.md
- - Docling: integrations/docling.md
- - Snowflake: integrations/snowflake.md
- - Cookbook: cookbook.md
- - Resources:
- - community.md
- - contributing.md
- - faq.md
- - license.md
-
-# Extra
-extra:
- social:
- - icon: fontawesome/brands/github
- link: https://github.com/Hawksight-AI/semantica
- - icon: fontawesome/brands/python
- link: https://pypi.org/project/semantica/
- version:
- provider: mike
- generator: true
-
diff --git a/mkdocs_local.yml b/mkdocs_local.yml
deleted file mode 100644
index 26261fc3..00000000
--- a/mkdocs_local.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-INHERIT: mkdocs.yml
-plugins:
- - search:
- lang: en
- - minify:
- minify_html: true
- - mkdocstrings:
- handlers:
- python:
- options:
- docstring_style: google
- show_source: true
- show_root_heading: true
- show_category_heading: true
diff --git a/requirements-docs.txt b/requirements-docs.txt
deleted file mode 100644
index 7244f8d4..00000000
--- a/requirements-docs.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-mkdocs>=1.6.1
-mkdocs-material>=9.7.6
-mkdocs-minify-plugin>=0.7.0
-mkdocs-mermaid2-plugin>=1.2.3
-pymdown-extensions>=10.21.2
-
-mkdocstrings[python]>=0.24.0
-mkdocs-jupyter>=0.26.3
-
diff --git a/setup_docs.py b/setup_docs.py
deleted file mode 100644
index dd10e20c..00000000
--- a/setup_docs.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os
-import shutil
-
-# Create directories
-os.makedirs("docs/reference", exist_ok=True)
-os.makedirs("docs/cookbook", exist_ok=True)
-
-# Modules to generate docs for
-modules = {
- "core": "semantica.core",
- "ingest": "semantica.ingest",
- "parse": "semantica.parse",
- "normalize": "semantica.normalize",
- "semantic_extract": "semantica.semantic_extract",
- "kg": "semantica.kg",
- "embeddings": "semantica.embeddings",
- "vector_store": "semantica.vector_store",
- "triplet_store": "semantica.triplet_store",
- "ontology": "semantica.ontology",
- "reasoning": "semantica.reasoning",
- "pipeline": "semantica.pipeline",
- "export": "semantica.export",
- "visualization": "semantica.visualization",
- "utils": "semantica.utils"
-}
-
-# Generate reference markdown files
-for name, package in modules.items():
- content = f"# {name.replace('_', ' ').title()}\n\n::: {package}\n"
- with open(f"docs/reference/{name}.md", "w") as f:
- f.write(content)
- print(f"Created docs/reference/{name}.md")
-
-# Copy cookbook directory
-if os.path.exists("cookbook"):
- if os.path.exists("docs/cookbook"):
- shutil.rmtree("docs/cookbook")
- shutil.copytree("cookbook", "docs/cookbook")
- print("Copied cookbook to docs/cookbook")
-
-# Remove old files
-files_to_remove = ["docs/MODULES_DOCUMENTATION.md", "docs/cookbook.md", "docs/api.md"]
-for f in files_to_remove:
- if os.path.exists(f):
- os.remove(f)
- print(f"Removed {f}")