mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-12 04:01:35 +00:00
- Add 5 new Knowledge Graph Quality Assurance modules - Implement solutions for LinkedIn post problems (templates, seed data, deduplication, conflicts) - Add agentic analytics capabilities and autonomous AI features - Emphasize open source and free nature throughout README - Add comprehensive use cases and enterprise features - Update module count to 20 modules with 120+ submodules - Add GraphRAG integration and SLM support - Include critical enterprise challenges and solutions
3429 lines
159 KiB
Markdown
3429 lines
159 KiB
Markdown
# 🧩 Semantica Modules & Submodules
|
|
|
|
> **Complete reference guide for all Semantica toolkit modules with practical code examples**
|
|
|
|
---
|
|
|
|
## 📋 Table of Contents
|
|
|
|
1. [Core Modules](#core-modules)
|
|
2. [Data Processing](#data-processing)
|
|
3. [Semantic Intelligence](#semantic-intelligence)
|
|
4. [Storage & Retrieval](#storage--retrieval)
|
|
5. [AI & Reasoning](#ai--reasoning)
|
|
6. [Knowledge Graph Quality Assurance](#knowledge-graph-quality-assurance)
|
|
7. [Complete Module Index](#complete-module-index)
|
|
8. [Import Reference](#import-reference)
|
|
|
|
---
|
|
|
|
## 🏗️ Core Modules
|
|
|
|
### 1. **Core Engine** (`semantica.core`)
|
|
|
|
**Main Class:** `Semantica`
|
|
|
|
**Purpose:** Central orchestration, configuration, and pipeline management
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica import Semantica
|
|
from semantica.core import Config, PluginManager, Orchestrator, LifecycleManager
|
|
from semantica.core.orchestrator import PipelineCoordinator, TaskScheduler, ResourceManager
|
|
from semantica.core.config_manager import YAMLConfigParser, JSONConfigParser, EnvironmentConfig
|
|
from semantica.core.plugin_registry import PluginLoader, VersionCompatibility, DependencyResolver
|
|
from semantica.core.lifecycle import StartupHooks, ShutdownHooks, HealthChecker, GracefulDegradation
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Initialize Semantica with configuration
|
|
core = Semantica(
|
|
llm_provider="openai",
|
|
embedding_model="text-embedding-3-large",
|
|
vector_store="pinecone",
|
|
graph_db="neo4j"
|
|
)
|
|
|
|
# Core functionality
|
|
core.initialize() # Setup all modules
|
|
knowledge_base = core.build_knowledge_base(sources) # Process data
|
|
status = core.get_status() # Get system health
|
|
pipeline = core.create_pipeline() # Create processing pipeline
|
|
config = core.get_config() # Get current configuration
|
|
plugins = core.list_plugins() # List available plugins
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Orchestrator (`semantica.core.orchestrator`):**
|
|
```python
|
|
from semantica.core.orchestrator import PipelineCoordinator, TaskScheduler, ResourceManager
|
|
|
|
# Pipeline coordination
|
|
coordinator = PipelineCoordinator()
|
|
coordinator.schedule_pipeline(pipeline_config)
|
|
coordinator.monitor_progress(pipeline_id)
|
|
coordinator.handle_failures(pipeline_id)
|
|
|
|
# Task scheduling
|
|
scheduler = TaskScheduler()
|
|
scheduler.schedule_task(task, priority="high")
|
|
scheduler.get_queue_status()
|
|
scheduler.cancel_task(task_id)
|
|
|
|
# Resource management
|
|
resource_manager = ResourceManager()
|
|
resource_manager.allocate_resources(requirements)
|
|
resource_manager.monitor_usage()
|
|
resource_manager.release_resources(resource_id)
|
|
```
|
|
|
|
**Config Manager (`semantica.core.config_manager`):**
|
|
```python
|
|
from semantica.core.config_manager import YAMLConfigParser, JSONConfigParser, EnvironmentConfig
|
|
|
|
# YAML configuration
|
|
yaml_parser = YAMLConfigParser()
|
|
config = yaml_parser.load("config.yaml")
|
|
yaml_parser.validate(config, schema="config_schema.yaml")
|
|
yaml_parser.save(config, "output.yaml")
|
|
|
|
# JSON configuration
|
|
json_parser = JSONConfigParser()
|
|
config = json_parser.load("config.json")
|
|
json_parser.merge_configs(base_config, override_config)
|
|
|
|
# Environment configuration
|
|
env_config = EnvironmentConfig()
|
|
env_config.load_from_env()
|
|
env_config.set_defaults(defaults)
|
|
```
|
|
|
|
**Plugin Registry (`semantica.core.plugin_registry`):**
|
|
```python
|
|
from semantica.core.plugin_registry import PluginLoader, VersionCompatibility, DependencyResolver
|
|
|
|
# Plugin loading
|
|
loader = PluginLoader()
|
|
plugin = loader.load_plugin("custom_processor", version="1.2.0")
|
|
loader.register_plugin(plugin)
|
|
loader.unload_plugin(plugin_id)
|
|
|
|
# Version compatibility
|
|
version_checker = VersionCompatibility()
|
|
compatible = version_checker.check_compatibility(plugin, semantica_version)
|
|
version_checker.get_compatible_versions(plugin_name)
|
|
|
|
# Dependency resolution
|
|
resolver = DependencyResolver()
|
|
dependencies = resolver.resolve_dependencies(plugin)
|
|
resolver.install_dependencies(dependencies)
|
|
```
|
|
|
|
**Lifecycle (`semantica.core.lifecycle`):**
|
|
```python
|
|
from semantica.core.lifecycle import StartupHooks, ShutdownHooks, HealthChecker, GracefulDegradation
|
|
|
|
# Startup hooks
|
|
startup = StartupHooks()
|
|
startup.register_hook("database_init", init_database)
|
|
startup.register_hook("cache_warmup", warmup_cache)
|
|
startup.execute_hooks()
|
|
|
|
# Shutdown hooks
|
|
shutdown = ShutdownHooks()
|
|
shutdown.register_hook("cleanup_temp", cleanup_temp_files)
|
|
shutdown.register_hook("close_connections", close_db_connections)
|
|
shutdown.execute_hooks()
|
|
|
|
# Health checking
|
|
health = HealthChecker()
|
|
health.add_check("database", check_database_health)
|
|
health.add_check("memory", check_memory_usage)
|
|
status = health.run_checks()
|
|
|
|
# Graceful degradation
|
|
degradation = GracefulDegradation()
|
|
degradation.set_fallback_strategy("cache_only")
|
|
degradation.handle_service_failure(service_name)
|
|
```
|
|
|
|
### 2. **Pipeline Builder** (`semantica.pipeline`)
|
|
|
|
**Main Class:** `PipelineBuilder`
|
|
|
|
**Purpose:** Create and manage data processing pipelines
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.pipeline import PipelineBuilder, ExecutionEngine, FailureHandler
|
|
from semantica.pipeline.execution_engine import PipelineRunner, StepOrchestrator, ProgressTracker
|
|
from semantica.pipeline.failure_handler import RetryHandler, FallbackHandler, ErrorRecovery
|
|
from semantica.pipeline.parallelism_manager import ParallelExecutor, LoadBalancer, TaskDistributor
|
|
from semantica.pipeline.resource_scheduler import CPUScheduler, GPUScheduler, MemoryManager
|
|
from semantica.pipeline.pipeline_validator import DependencyChecker, CycleDetector, ConfigValidator
|
|
from semantica.pipeline.monitoring_hooks import MetricsCollector, AlertManager, StatusReporter
|
|
from semantica.pipeline.pipeline_templates import PrebuiltTemplates, CustomTemplates, TemplateManager
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Build custom pipeline
|
|
pipeline = PipelineBuilder() \
|
|
.add_step("ingest", {"source": "documents/"}) \
|
|
.add_step("parse", {"formats": ["pdf", "docx"]}) \
|
|
.add_step("extract", {"entities": True, "relations": True}) \
|
|
.add_step("embed", {"model": "text-embedding-3-large"}) \
|
|
.set_parallelism(4) \
|
|
.build()
|
|
|
|
# Execute pipeline
|
|
results = pipeline.run()
|
|
pipeline.pause() # Pause execution
|
|
pipeline.resume() # Resume execution
|
|
pipeline.stop() # Stop execution
|
|
status = pipeline.get_status() # Get current status
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Execution Engine (`semantica.pipeline.execution_engine`):**
|
|
```python
|
|
from semantica.pipeline.execution_engine import PipelineRunner, StepOrchestrator, ProgressTracker
|
|
|
|
# Pipeline execution
|
|
runner = PipelineRunner()
|
|
runner.execute_pipeline(pipeline_config)
|
|
runner.pause_pipeline(pipeline_id)
|
|
runner.resume_pipeline(pipeline_id)
|
|
runner.stop_pipeline(pipeline_id)
|
|
|
|
# Step orchestration
|
|
orchestrator = StepOrchestrator()
|
|
orchestrator.coordinate_steps(steps)
|
|
orchestrator.manage_dependencies(step_dependencies)
|
|
orchestrator.handle_step_completion(step_id, result)
|
|
|
|
# Progress tracking
|
|
tracker = ProgressTracker()
|
|
tracker.track_progress(pipeline_id)
|
|
tracker.get_completion_percentage()
|
|
tracker.estimate_remaining_time()
|
|
```
|
|
|
|
**Failure Handler (`semantica.pipeline.failure_handler`):**
|
|
```python
|
|
from semantica.pipeline.failure_handler import RetryHandler, FallbackHandler, ErrorRecovery
|
|
|
|
# Retry logic
|
|
retry_handler = RetryHandler(max_retries=3, backoff_factor=2.0)
|
|
retry_handler.retry_failed_step(step_id, error)
|
|
retry_handler.set_retry_policy(step_type, retry_policy)
|
|
|
|
# Fallback strategies
|
|
fallback = FallbackHandler()
|
|
fallback.set_fallback_strategy("cache_only")
|
|
fallback.handle_service_failure(service_name)
|
|
fallback.switch_to_backup(primary_failed)
|
|
|
|
# Error recovery
|
|
recovery = ErrorRecovery()
|
|
recovery.analyze_error(error)
|
|
recovery.suggest_recovery_actions(error)
|
|
recovery.execute_recovery(recovery_plan)
|
|
```
|
|
|
|
**Parallelism Manager (`semantica.pipeline.parallelism_manager`):**
|
|
```python
|
|
from semantica.pipeline.parallelism_manager import ParallelExecutor, LoadBalancer, TaskDistributor
|
|
|
|
# Parallel execution
|
|
executor = ParallelExecutor(max_workers=8)
|
|
executor.execute_parallel(tasks)
|
|
executor.set_parallelism_level(level=4)
|
|
executor.monitor_worker_health()
|
|
|
|
# Load balancing
|
|
balancer = LoadBalancer()
|
|
balancer.distribute_load(tasks, workers)
|
|
balancer.rebalance_workload()
|
|
balancer.get_worker_utilization()
|
|
|
|
# Task distribution
|
|
distributor = TaskDistributor()
|
|
distributor.distribute_tasks(tasks, workers)
|
|
distributor.collect_results(worker_results)
|
|
distributor.handle_worker_failure(worker_id)
|
|
```
|
|
|
|
**Resource Scheduler (`semantica.pipeline.resource_scheduler`):**
|
|
```python
|
|
from semantica.pipeline.resource_scheduler import CPUScheduler, GPUScheduler, MemoryManager
|
|
|
|
# CPU scheduling
|
|
cpu_scheduler = CPUScheduler()
|
|
cpu_scheduler.allocate_cpu(cores=4)
|
|
cpu_scheduler.set_cpu_affinity(process_id, cores)
|
|
cpu_scheduler.monitor_cpu_usage()
|
|
|
|
# GPU scheduling
|
|
gpu_scheduler = GPUScheduler()
|
|
gpu_scheduler.allocate_gpu(device_id=0)
|
|
gpu_scheduler.set_gpu_memory_limit(limit="8GB")
|
|
gpu_scheduler.monitor_gpu_usage()
|
|
|
|
# Memory management
|
|
memory_manager = MemoryManager()
|
|
memory_manager.allocate_memory(size="2GB")
|
|
memory_manager.optimize_memory_usage()
|
|
memory_manager.garbage_collect()
|
|
```
|
|
|
|
**Pipeline Validator (`semantica.pipeline.pipeline_validator`):**
|
|
```python
|
|
from semantica.pipeline.pipeline_validator import DependencyChecker, CycleDetector, ConfigValidator
|
|
|
|
# Dependency checking
|
|
dep_checker = DependencyChecker()
|
|
dep_checker.check_dependencies(pipeline_steps)
|
|
dep_checker.validate_dependency_graph(graph)
|
|
dep_checker.suggest_dependency_fixes(issues)
|
|
|
|
# Cycle detection
|
|
cycle_detector = CycleDetector()
|
|
has_cycles = cycle_detector.detect_cycles(pipeline_graph)
|
|
cycles = cycle_detector.find_cycles(pipeline_graph)
|
|
cycle_detector.suggest_cycle_breaks(cycles)
|
|
|
|
# Configuration validation
|
|
config_validator = ConfigValidator()
|
|
config_validator.validate_config(pipeline_config)
|
|
config_validator.check_required_fields(config)
|
|
config_validator.validate_data_types(config)
|
|
```
|
|
|
|
**Monitoring Hooks (`semantica.pipeline.monitoring_hooks`):**
|
|
```python
|
|
from semantica.pipeline.monitoring_hooks import MetricsCollector, AlertManager, StatusReporter
|
|
|
|
# Metrics collection
|
|
metrics = MetricsCollector()
|
|
metrics.collect_pipeline_metrics(pipeline_id)
|
|
metrics.record_step_duration(step_id, duration)
|
|
metrics.record_memory_usage(step_id, memory)
|
|
|
|
# Alert management
|
|
alerts = AlertManager()
|
|
alerts.set_alert_threshold("memory_usage", threshold=0.9)
|
|
alerts.send_alert("High memory usage detected")
|
|
alerts.configure_notifications(email="admin@example.com")
|
|
|
|
# Status reporting
|
|
reporter = StatusReporter()
|
|
reporter.generate_status_report(pipeline_id)
|
|
reporter.export_metrics(format="json")
|
|
reporter.create_dashboard_data()
|
|
```
|
|
|
|
**Pipeline Templates (`semantica.pipeline.pipeline_templates`):**
|
|
```python
|
|
from semantica.pipeline.pipeline_templates import PrebuiltTemplates, CustomTemplates, TemplateManager
|
|
|
|
# Prebuilt templates
|
|
templates = PrebuiltTemplates()
|
|
doc_processing = templates.get_template("document_processing")
|
|
web_scraping = templates.get_template("web_scraping")
|
|
knowledge_extraction = templates.get_template("knowledge_extraction")
|
|
|
|
# Custom templates
|
|
custom = CustomTemplates()
|
|
custom.create_template("my_pipeline", steps)
|
|
custom.save_template(template, "my_pipeline.json")
|
|
custom.load_template("my_pipeline.json")
|
|
|
|
# Template management
|
|
manager = TemplateManager()
|
|
manager.list_templates()
|
|
manager.validate_template(template)
|
|
manager.export_template(template_id, "export.json")
|
|
```
|
|
|
|
---
|
|
|
|
## 📊 Data Processing
|
|
|
|
### 3. **Data Ingestion** (`semantica.ingest`)
|
|
|
|
**Main Classes:** `FileIngestor`, `WebIngestor`, `FeedIngestor`
|
|
|
|
**Purpose:** Ingest data from various sources
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.ingest import FileIngestor, WebIngestor, FeedIngestor
|
|
from semantica.ingest.file import LocalFileHandler, S3Handler, GCSHandler, AzureHandler
|
|
from semantica.ingest.web import WebScraper, SitemapCrawler, JavaScriptRenderer
|
|
from semantica.ingest.feed import RSSParser, AtomParser, SocialMediaAPI
|
|
from semantica.ingest.stream import WebSocketHandler, MessageQueueHandler, KafkaStream
|
|
from semantica.ingest.repo import GitHandler, PackageManagerHandler
|
|
from semantica.ingest.email import IMAPHandler, ExchangeHandler, GmailAPIHandler
|
|
from semantica.ingest.db_export import DatabaseExporter, SQLQueryHandler, ETLProcessor
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# File ingestion
|
|
file_ingestor = FileIngestor()
|
|
files = file_ingestor.scan_directory("documents/", recursive=True)
|
|
formats = file_ingestor.detect_format("document.pdf")
|
|
metadata = file_ingestor.extract_metadata("document.pdf")
|
|
content = file_ingestor.ingest_file("document.pdf")
|
|
|
|
# Web ingestion
|
|
web_ingestor = WebIngestor(respect_robots=True, max_depth=3)
|
|
web_content = web_ingestor.crawl_site("https://example.com")
|
|
links = web_ingestor.extract_links(web_content)
|
|
sitemap = web_ingestor.parse_sitemap("https://example.com/sitemap.xml")
|
|
|
|
# Feed ingestion
|
|
feed_ingestor = FeedIngestor()
|
|
rss_data = feed_ingestor.parse_rss("https://example.com/feed.xml")
|
|
atom_data = feed_ingestor.parse_atom("https://example.com/atom.xml")
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**File Handler (`semantica.ingest.file`):**
|
|
```python
|
|
from semantica.ingest.file import LocalFileHandler, S3Handler, GCSHandler, AzureHandler
|
|
|
|
# Local file handling
|
|
local_handler = LocalFileHandler()
|
|
files = local_handler.scan_directory("documents/", patterns=["*.pdf", "*.docx"])
|
|
file_info = local_handler.get_file_info("document.pdf")
|
|
content = local_handler.read_file("document.pdf", encoding="utf-8")
|
|
|
|
# S3 handling
|
|
s3_handler = S3Handler(bucket="my-bucket", region="us-east-1")
|
|
s3_files = s3_handler.list_objects(prefix="documents/")
|
|
s3_content = s3_handler.download_object("documents/file.pdf")
|
|
s3_handler.upload_object("local_file.pdf", "documents/remote_file.pdf")
|
|
|
|
# Google Cloud Storage
|
|
gcs_handler = GCSHandler(project="my-project", bucket="my-bucket")
|
|
gcs_files = gcs_handler.list_blobs(prefix="documents/")
|
|
gcs_content = gcs_handler.download_blob("documents/file.pdf")
|
|
|
|
# Azure Blob Storage
|
|
azure_handler = AzureHandler(account_name="myaccount", container="documents")
|
|
azure_files = azure_handler.list_blobs()
|
|
azure_content = azure_handler.download_blob("file.pdf")
|
|
```
|
|
|
|
**Web Scraper (`semantica.ingest.web`):**
|
|
```python
|
|
from semantica.ingest.web import WebScraper, SitemapCrawler, JavaScriptRenderer
|
|
|
|
# Web scraping
|
|
scraper = WebScraper(respect_robots=True, delay=1.0)
|
|
content = scraper.scrape_url("https://example.com")
|
|
links = scraper.extract_links(content)
|
|
images = scraper.extract_images(content)
|
|
text = scraper.extract_text(content)
|
|
|
|
# Sitemap crawling
|
|
sitemap_crawler = SitemapCrawler()
|
|
urls = sitemap_crawler.parse_sitemap("https://example.com/sitemap.xml")
|
|
sitemap_crawler.crawl_sitemap_urls(urls, max_pages=100)
|
|
|
|
# JavaScript rendering
|
|
js_renderer = JavaScriptRenderer(headless=True)
|
|
rendered_content = js_renderer.render_page("https://spa-example.com")
|
|
js_renderer.wait_for_element("div.content", timeout=10)
|
|
```
|
|
|
|
**Feed Parser (`semantica.ingest.feed`):**
|
|
```python
|
|
from semantica.ingest.feed import RSSParser, AtomParser, SocialMediaAPI
|
|
|
|
# RSS parsing
|
|
rss_parser = RSSParser()
|
|
rss_feed = rss_parser.parse_rss("https://example.com/feed.xml")
|
|
entries = rss_parser.get_entries(rss_feed)
|
|
rss_parser.save_entries(entries, "rss_entries.json")
|
|
|
|
# Atom parsing
|
|
atom_parser = AtomParser()
|
|
atom_feed = atom_parser.parse_atom("https://example.com/atom.xml")
|
|
atom_entries = atom_parser.get_entries(atom_feed)
|
|
|
|
# Social media APIs
|
|
social_api = SocialMediaAPI(platform="twitter", api_key="your_key")
|
|
tweets = social_api.fetch_posts(hashtag="#semantica", count=100)
|
|
social_api.export_posts(tweets, format="json")
|
|
```
|
|
|
|
**Stream Handler (`semantica.ingest.stream`):**
|
|
```python
|
|
from semantica.ingest.stream import WebSocketHandler, MessageQueueHandler, KafkaStream
|
|
|
|
# WebSocket streaming
|
|
ws_handler = WebSocketHandler(url="wss://example.com/stream")
|
|
ws_handler.connect()
|
|
messages = ws_handler.listen_for_messages(callback=process_message)
|
|
ws_handler.send_message({"type": "subscribe", "channel": "updates"})
|
|
|
|
# Message queue handling
|
|
mq_handler = MessageQueueHandler(queue_type="rabbitmq", host="localhost")
|
|
mq_handler.connect()
|
|
mq_handler.consume_messages(queue="data_queue", callback=process_message)
|
|
mq_handler.publish_message("data_queue", {"data": "new_content"})
|
|
|
|
# Kafka streaming
|
|
kafka_stream = KafkaStream(bootstrap_servers=["localhost:9092"])
|
|
kafka_stream.subscribe_topics(["data_topic"])
|
|
kafka_stream.consume_messages(callback=process_kafka_message)
|
|
kafka_stream.produce_message("data_topic", {"key": "value"})
|
|
```
|
|
|
|
**Repository Handler (`semantica.ingest.repo`):**
|
|
```python
|
|
from semantica.ingest.repo import GitHandler, PackageManagerHandler
|
|
|
|
# Git repository handling
|
|
git_handler = GitHandler()
|
|
git_handler.clone_repository("https://github.com/user/repo.git", "local_repo")
|
|
commits = git_handler.get_commits(since="2023-01-01")
|
|
files = git_handler.get_changed_files(commit_hash="abc123")
|
|
git_handler.checkout_branch("feature-branch")
|
|
|
|
# Package manager handling
|
|
pkg_handler = PackageManagerHandler(manager="npm")
|
|
packages = pkg_handler.list_packages()
|
|
pkg_handler.install_package("package-name", version="1.0.0")
|
|
pkg_info = pkg_handler.get_package_info("package-name")
|
|
```
|
|
|
|
**Email Handler (`semantica.ingest.email`):**
|
|
```python
|
|
from semantica.ingest.email import IMAPHandler, ExchangeHandler, GmailAPIHandler
|
|
|
|
# IMAP handling
|
|
imap_handler = IMAPHandler(server="imap.gmail.com", port=993)
|
|
imap_handler.connect(username="user@example.com", password="password")
|
|
emails = imap_handler.fetch_emails(folder="INBOX", since="2023-01-01")
|
|
attachments = imap_handler.download_attachments(email_id=123)
|
|
|
|
# Exchange handling
|
|
exchange_handler = ExchangeHandler(server="outlook.office365.com")
|
|
exchange_handler.connect(username="user@example.com", password="password")
|
|
exchange_emails = exchange_handler.get_emails(folder="Inbox")
|
|
|
|
# Gmail API handling
|
|
gmail_handler = GmailAPIHandler(credentials_file="credentials.json")
|
|
gmail_emails = gmail_handler.list_messages(query="is:unread")
|
|
gmail_handler.download_attachments(message_id="msg123")
|
|
```
|
|
|
|
**Database Export (`semantica.ingest.db_export`):**
|
|
```python
|
|
from semantica.ingest.db_export import DatabaseExporter, SQLQueryHandler, ETLProcessor
|
|
|
|
# Database export
|
|
db_exporter = DatabaseExporter(connection_string="postgresql://user:pass@localhost/db")
|
|
tables = db_exporter.export_table("users", output_format="csv")
|
|
db_exporter.export_schema(output_file="schema.sql")
|
|
|
|
# SQL query handling
|
|
sql_handler = SQLQueryHandler(connection_string="postgresql://user:pass@localhost/db")
|
|
results = sql_handler.execute_query("SELECT * FROM users WHERE active = true")
|
|
sql_handler.export_query_results(results, "active_users.csv")
|
|
|
|
# ETL processing
|
|
etl_processor = ETLProcessor()
|
|
etl_processor.extract_from_source("database", config=db_config)
|
|
etl_processor.transform_data(transform_rules=transformation_rules)
|
|
etl_processor.load_to_destination("data_warehouse", config=dw_config)
|
|
```
|
|
|
|
### 4. **Document Parsing** (`semantica.parse`)
|
|
|
|
**Main Classes:** `PDFParser`, `DOCXParser`, `HTMLParser`, `ImageParser`
|
|
|
|
**Purpose:** Extract content from various document formats
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.parse import PDFParser, DOCXParser, HTMLParser, ImageParser
|
|
from semantica.parse.pdf import PDFTextExtractor, PDFTableExtractor, PDFImageExtractor, PDFAnnotationExtractor
|
|
from semantica.parse.docx import DOCXTextExtractor, DOCXStyleExtractor, DOCXTrackChangesExtractor
|
|
from semantica.parse.pptx import PPTXSlideExtractor, PPTXNotesExtractor, PPTXMediaExtractor
|
|
from semantica.parse.excel import ExcelDataExtractor, ExcelFormulaExtractor, ExcelChartExtractor
|
|
from semantica.parse.html import HTMLDOMParser, HTMLMetadataExtractor, HTMLFormExtractor
|
|
from semantica.parse.images import OCRProcessor, ObjectDetector, EXIFExtractor
|
|
from semantica.parse.tables import TableDetector, TableStructureAnalyzer, TableDataExtractor
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# PDF parsing
|
|
pdf_parser = PDFParser()
|
|
pdf_text = pdf_parser.extract_text("document.pdf")
|
|
pdf_tables = pdf_parser.extract_tables("document.pdf")
|
|
pdf_images = pdf_parser.extract_images("document.pdf")
|
|
pdf_annotations = pdf_parser.extract_annotations("document.pdf")
|
|
|
|
# DOCX parsing
|
|
docx_parser = DOCXParser()
|
|
docx_content = docx_parser.get_document_structure("document.docx")
|
|
track_changes = docx_parser.extract_track_changes("document.docx")
|
|
styles = docx_parser.extract_styles("document.docx")
|
|
|
|
# HTML parsing
|
|
html_parser = HTMLParser()
|
|
dom_tree = html_parser.parse_dom("https://example.com")
|
|
metadata = html_parser.extract_metadata(dom_tree)
|
|
forms = html_parser.extract_forms(dom_tree)
|
|
|
|
# Image parsing (OCR)
|
|
image_parser = ImageParser()
|
|
ocr_text = image_parser.ocr_text("image.png")
|
|
objects = image_parser.detect_objects("image.jpg")
|
|
exif_data = image_parser.extract_exif("image.jpg")
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**PDF Parser (`semantica.parse.pdf`):**
|
|
```python
|
|
from semantica.parse.pdf import PDFTextExtractor, PDFTableExtractor, PDFImageExtractor, PDFAnnotationExtractor
|
|
|
|
# Text extraction
|
|
text_extractor = PDFTextExtractor()
|
|
text = text_extractor.extract_text("document.pdf")
|
|
text_by_page = text_extractor.extract_text_by_page("document.pdf")
|
|
text_with_coordinates = text_extractor.extract_text_with_coordinates("document.pdf")
|
|
|
|
# Table extraction
|
|
table_extractor = PDFTableExtractor()
|
|
tables = table_extractor.extract_tables("document.pdf")
|
|
table_data = table_extractor.extract_table_data("document.pdf", page=1)
|
|
table_structure = table_extractor.analyze_table_structure("document.pdf")
|
|
|
|
# Image extraction
|
|
image_extractor = PDFImageExtractor()
|
|
images = image_extractor.extract_images("document.pdf")
|
|
image_metadata = image_extractor.get_image_metadata("document.pdf")
|
|
image_extractor.save_images("document.pdf", output_dir="extracted_images/")
|
|
|
|
# Annotation extraction
|
|
annotation_extractor = PDFAnnotationExtractor()
|
|
annotations = annotation_extractor.extract_annotations("document.pdf")
|
|
comments = annotation_extractor.extract_comments("document.pdf")
|
|
highlights = annotation_extractor.extract_highlights("document.pdf")
|
|
```
|
|
|
|
**DOCX Parser (`semantica.parse.docx`):**
|
|
```python
|
|
from semantica.parse.docx import DOCXTextExtractor, DOCXStyleExtractor, DOCXTrackChangesExtractor
|
|
|
|
# Text extraction
|
|
docx_text = DOCXTextExtractor()
|
|
text = docx_text.extract_text("document.docx")
|
|
paragraphs = docx_text.extract_paragraphs("document.docx")
|
|
headers_footers = docx_text.extract_headers_footers("document.docx")
|
|
|
|
# Style extraction
|
|
style_extractor = DOCXStyleExtractor()
|
|
styles = style_extractor.extract_styles("document.docx")
|
|
formatting = style_extractor.extract_formatting("document.docx")
|
|
tables = style_extractor.extract_table_styles("document.docx")
|
|
|
|
# Track changes extraction
|
|
track_changes = DOCXTrackChangesExtractor()
|
|
changes = track_changes.extract_changes("document.docx")
|
|
revisions = track_changes.extract_revisions("document.docx")
|
|
comments = track_changes.extract_comments("document.docx")
|
|
```
|
|
|
|
**PowerPoint Parser (`semantica.parse.pptx`):**
|
|
```python
|
|
from semantica.parse.pptx import PPTXSlideExtractor, PPTXNotesExtractor, PPTXMediaExtractor
|
|
|
|
# Slide extraction
|
|
slide_extractor = PPTXSlideExtractor()
|
|
slides = slide_extractor.extract_slides("presentation.pptx")
|
|
slide_text = slide_extractor.extract_slide_text("presentation.pptx", slide_number=1)
|
|
slide_layout = slide_extractor.extract_slide_layout("presentation.pptx")
|
|
|
|
# Notes extraction
|
|
notes_extractor = PPTXNotesExtractor()
|
|
notes = notes_extractor.extract_notes("presentation.pptx")
|
|
speaker_notes = notes_extractor.extract_speaker_notes("presentation.pptx")
|
|
|
|
# Media extraction
|
|
media_extractor = PPTXMediaExtractor()
|
|
images = media_extractor.extract_images("presentation.pptx")
|
|
videos = media_extractor.extract_videos("presentation.pptx")
|
|
audio = media_extractor.extract_audio("presentation.pptx")
|
|
```
|
|
|
|
**Excel Parser (`semantica.parse.excel`):**
|
|
```python
|
|
from semantica.parse.excel import ExcelDataExtractor, ExcelFormulaExtractor, ExcelChartExtractor
|
|
|
|
# Data extraction
|
|
excel_extractor = ExcelDataExtractor()
|
|
data = excel_extractor.extract_data("spreadsheet.xlsx", sheet_name="Sheet1")
|
|
all_sheets = excel_extractor.extract_all_sheets("spreadsheet.xlsx")
|
|
cell_values = excel_extractor.extract_cell_values("spreadsheet.xlsx", range="A1:C10")
|
|
|
|
# Formula extraction
|
|
formula_extractor = ExcelFormulaExtractor()
|
|
formulas = formula_extractor.extract_formulas("spreadsheet.xlsx")
|
|
formula_dependencies = formula_extractor.analyze_formula_dependencies("spreadsheet.xlsx")
|
|
|
|
# Chart extraction
|
|
chart_extractor = ExcelChartExtractor()
|
|
charts = chart_extractor.extract_charts("spreadsheet.xlsx")
|
|
chart_data = chart_extractor.extract_chart_data("spreadsheet.xlsx", chart_name="Chart1")
|
|
```
|
|
|
|
**HTML Parser (`semantica.parse.html`):**
|
|
```python
|
|
from semantica.parse.html import HTMLDOMParser, HTMLMetadataExtractor, HTMLFormExtractor
|
|
|
|
# DOM parsing
|
|
dom_parser = HTMLDOMParser()
|
|
dom_tree = dom_parser.parse_dom("https://example.com")
|
|
elements = dom_parser.find_elements(dom_tree, tag="div", class_name="content")
|
|
links = dom_parser.extract_links(dom_tree)
|
|
images = dom_parser.extract_images(dom_tree)
|
|
|
|
# Metadata extraction
|
|
metadata_extractor = HTMLMetadataExtractor()
|
|
title = metadata_extractor.extract_title(dom_tree)
|
|
description = metadata_extractor.extract_description(dom_tree)
|
|
keywords = metadata_extractor.extract_keywords(dom_tree)
|
|
og_data = metadata_extractor.extract_og_metadata(dom_tree)
|
|
|
|
# Form extraction
|
|
form_extractor = HTMLFormExtractor()
|
|
forms = form_extractor.extract_forms(dom_tree)
|
|
form_fields = form_extractor.extract_form_fields(dom_tree)
|
|
form_actions = form_extractor.extract_form_actions(dom_tree)
|
|
```
|
|
|
|
**Image Parser (`semantica.parse.images`):**
|
|
```python
|
|
from semantica.parse.images import OCRProcessor, ObjectDetector, EXIFExtractor
|
|
|
|
# OCR processing
|
|
ocr_processor = OCRProcessor()
|
|
text = ocr_processor.extract_text("image.png")
|
|
text_confidence = ocr_processor.extract_text_with_confidence("image.png")
|
|
text_by_region = ocr_processor.extract_text_by_region("image.png", regions=[(0,0,100,100)])
|
|
|
|
# Object detection
|
|
object_detector = ObjectDetector()
|
|
objects = object_detector.detect_objects("image.jpg")
|
|
faces = object_detector.detect_faces("image.jpg")
|
|
text_regions = object_detector.detect_text_regions("image.jpg")
|
|
|
|
# EXIF data extraction
|
|
exif_extractor = EXIFExtractor()
|
|
exif_data = exif_extractor.extract_exif("image.jpg")
|
|
camera_info = exif_extractor.extract_camera_info("image.jpg")
|
|
location_data = exif_extractor.extract_location("image.jpg")
|
|
```
|
|
|
|
**Table Parser (`semantica.parse.tables`):**
|
|
```python
|
|
from semantica.parse.tables import TableDetector, TableStructureAnalyzer, TableDataExtractor
|
|
|
|
# Table detection
|
|
table_detector = TableDetector()
|
|
tables = table_detector.detect_tables("document.pdf")
|
|
table_regions = table_detector.detect_table_regions("document.pdf")
|
|
|
|
# Structure analysis
|
|
structure_analyzer = TableStructureAnalyzer()
|
|
structure = structure_analyzer.analyze_structure("document.pdf", table_region)
|
|
headers = structure_analyzer.detect_headers("document.pdf", table_region)
|
|
rows_columns = structure_analyzer.detect_rows_columns("document.pdf", table_region)
|
|
|
|
# Data extraction
|
|
data_extractor = TableDataExtractor()
|
|
table_data = data_extractor.extract_data("document.pdf", table_region)
|
|
csv_data = data_extractor.extract_as_csv("document.pdf", table_region)
|
|
json_data = data_extractor.extract_as_json("document.pdf", table_region)
|
|
```
|
|
|
|
### 5. **Text Normalization** (`semantica.normalize`)
|
|
|
|
**Main Classes:** `TextCleaner`, `LanguageDetector`, `EntityNormalizer`
|
|
|
|
**Purpose:** Clean and normalize text data
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.normalize import TextCleaner, LanguageDetector, EntityNormalizer
|
|
from semantica.normalize.text_cleaner import HTMLRemover, WhitespaceNormalizer, SpecialCharRemover
|
|
from semantica.normalize.language_detector import MultiLanguageDetector, LanguageConfidence, LanguageSupport
|
|
from semantica.normalize.encoding_handler import UTF8Converter, EncodingValidator, EncodingDetector
|
|
from semantica.normalize.entity_normalizer import EntityCanonicalizer, AcronymExpander, EntityStandardizer
|
|
from semantica.normalize.date_normalizer import DateFormatter, DateParser, DateValidator
|
|
from semantica.normalize.number_normalizer import NumberFormatter, NumberParser, CurrencyNormalizer
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Text cleaning
|
|
cleaner = TextCleaner()
|
|
clean_text = cleaner.remove_html(html_content)
|
|
normalized = cleaner.normalize_whitespace(text)
|
|
cleaned = cleaner.remove_special_chars(text)
|
|
unicode_normalized = cleaner.normalize_unicode(text)
|
|
|
|
# Language detection
|
|
detector = LanguageDetector()
|
|
language = detector.detect("Hello world")
|
|
confidence = detector.get_confidence()
|
|
supported = detector.supported_languages()
|
|
multi_lang = detector.detect_multiple_languages(mixed_text)
|
|
|
|
# Entity normalization
|
|
normalizer = EntityNormalizer()
|
|
canonical = normalizer.canonicalize("Apple Inc.", "Apple")
|
|
expanded = normalizer.expand_acronyms("NASA")
|
|
standardized = normalizer.standardize_entities(text)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Text Cleaner (`semantica.normalize.text_cleaner`):**
|
|
```python
|
|
from semantica.normalize.text_cleaner import HTMLRemover, WhitespaceNormalizer, SpecialCharRemover
|
|
|
|
# HTML removal
|
|
html_remover = HTMLRemover()
|
|
clean_text = html_remover.remove_html("<p>Hello <b>world</b></p>")
|
|
clean_text = html_remover.remove_html_tags(html_content, keep_tags=["p", "br"])
|
|
clean_text = html_remover.remove_html_entities("& < >")
|
|
|
|
# Whitespace normalization
|
|
whitespace_normalizer = WhitespaceNormalizer()
|
|
normalized = whitespace_normalizer.normalize_spaces(" multiple spaces ")
|
|
normalized = whitespace_normalizer.normalize_line_breaks(text)
|
|
normalized = whitespace_normalizer.remove_extra_whitespace(text)
|
|
|
|
# Special character removal
|
|
char_remover = SpecialCharRemover()
|
|
cleaned = char_remover.remove_special_chars("Text with @#$% symbols")
|
|
cleaned = char_remover.remove_control_chars(text)
|
|
cleaned = char_remover.remove_non_printable(text)
|
|
```
|
|
|
|
**Language Detector (`semantica.normalize.language_detector`):**
|
|
```python
|
|
from semantica.normalize.language_detector import MultiLanguageDetector, LanguageConfidence, LanguageSupport
|
|
|
|
# Multi-language detection
|
|
multi_detector = MultiLanguageDetector()
|
|
languages = multi_detector.detect_languages("Hello world. Bonjour le monde.")
|
|
primary_lang = multi_detector.get_primary_language(text)
|
|
language_confidence = multi_detector.get_confidence_scores(text)
|
|
|
|
# Language confidence
|
|
confidence_calculator = LanguageConfidence()
|
|
confidence = confidence_calculator.calculate_confidence(text, "en")
|
|
confidence_scores = confidence_calculator.get_all_confidence_scores(text)
|
|
|
|
# Language support
|
|
language_support = LanguageSupport()
|
|
supported = language_support.get_supported_languages()
|
|
is_supported = language_support.is_language_supported("en")
|
|
language_info = language_support.get_language_info("en")
|
|
```
|
|
|
|
**Encoding Handler (`semantica.normalize.encoding_handler`):**
|
|
```python
|
|
from semantica.normalize.encoding_handler import UTF8Converter, EncodingValidator, EncodingDetector
|
|
|
|
# UTF-8 conversion
|
|
utf8_converter = UTF8Converter()
|
|
utf8_text = utf8_converter.convert_to_utf8(text, source_encoding="latin1")
|
|
utf8_text = utf8_converter.ensure_utf8(text)
|
|
utf8_text = utf8_converter.fix_encoding_issues(text)
|
|
|
|
# Encoding validation
|
|
encoding_validator = EncodingValidator()
|
|
is_valid = encoding_validator.validate_encoding(text, "utf-8")
|
|
validation_report = encoding_validator.validate_text_encoding(text)
|
|
encoding_validator.fix_encoding_errors(text)
|
|
|
|
# Encoding detection
|
|
encoding_detector = EncodingDetector()
|
|
detected_encoding = encoding_detector.detect_encoding(text)
|
|
confidence = encoding_detector.get_detection_confidence()
|
|
possible_encodings = encoding_detector.get_possible_encodings(text)
|
|
```
|
|
|
|
**Entity Normalizer (`semantica.normalize.entity_normalizer`):**
|
|
```python
|
|
from semantica.normalize.entity_normalizer import EntityCanonicalizer, AcronymExpander, EntityStandardizer
|
|
|
|
# Entity canonicalization
|
|
canonicalizer = EntityCanonicalizer()
|
|
canonical = canonicalizer.canonicalize("Apple Inc.", "Apple")
|
|
canonical = canonicalizer.canonicalize_entities(text)
|
|
canonical = canonicalizer.merge_duplicate_entities(entities)
|
|
|
|
# Acronym expansion
|
|
acronym_expander = AcronymExpander()
|
|
expanded = acronym_expander.expand_acronyms("NASA", context="space agency")
|
|
expanded = acronym_expander.expand_all_acronyms(text)
|
|
acronym_map = acronym_expander.build_acronym_dictionary(text)
|
|
|
|
# Entity standardization
|
|
entity_standardizer = EntityStandardizer()
|
|
standardized = entity_standardizer.standardize_entities(text)
|
|
standardized = entity_standardizer.normalize_entity_names(entities)
|
|
standardized = entity_standardizer.validate_entity_format(entities)
|
|
```
|
|
|
|
**Date Normalizer (`semantica.normalize.date_normalizer`):**
|
|
```python
|
|
from semantica.normalize.date_normalizer import DateFormatter, DateParser, DateValidator
|
|
|
|
# Date formatting
|
|
date_formatter = DateFormatter()
|
|
formatted = date_formatter.format_date("2023-01-15", format="%B %d, %Y")
|
|
formatted = date_formatter.normalize_date_format("15/01/2023")
|
|
formatted = date_formatter.standardize_dates(text)
|
|
|
|
# Date parsing
|
|
date_parser = DateParser()
|
|
parsed = date_parser.parse_date("January 15, 2023")
|
|
parsed = date_parser.parse_flexible_date("15th Jan 2023")
|
|
parsed = date_parser.extract_dates_from_text(text)
|
|
|
|
# Date validation
|
|
date_validator = DateValidator()
|
|
is_valid = date_validator.validate_date("2023-01-15")
|
|
is_valid = date_validator.validate_date_range(start_date, end_date)
|
|
validation_report = date_validator.validate_dates_in_text(text)
|
|
```
|
|
|
|
**Number Normalizer (`semantica.normalize.number_normalizer`):**
|
|
```python
|
|
from semantica.normalize.number_normalizer import NumberFormatter, NumberParser, CurrencyNormalizer
|
|
|
|
# Number formatting
|
|
number_formatter = NumberFormatter()
|
|
formatted = number_formatter.format_number(1234567.89, format="comma")
|
|
formatted = number_formatter.normalize_numbers(text)
|
|
formatted = number_formatter.standardize_decimal_separator(text)
|
|
|
|
# Number parsing
|
|
number_parser = NumberParser()
|
|
parsed = number_parser.parse_number("1,234,567.89")
|
|
parsed = number_parser.parse_flexible_number("1.23M")
|
|
parsed = number_parser.extract_numbers_from_text(text)
|
|
|
|
# Currency normalization
|
|
currency_normalizer = CurrencyNormalizer()
|
|
normalized = currency_normalizer.normalize_currency("$1,234.56", target_currency="USD")
|
|
normalized = currency_normalizer.convert_currency(amount, from_currency="EUR", to_currency="USD")
|
|
normalized = currency_normalizer.standardize_currency_format(text)
|
|
```
|
|
|
|
### 6. **Text Chunking** (`semantica.split`)
|
|
|
|
**Main Classes:** `SemanticChunker`, `StructuralChunker`, `TableChunker`
|
|
|
|
**Purpose:** Split documents into optimal chunks for processing
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.split import SemanticChunker, StructuralChunker, TableChunker
|
|
from semantica.split.sliding_window import SlidingWindowChunker, OverlapChunker, FixedSizeChunker
|
|
from semantica.split.semantic_chunker import TopicBasedChunker, SentenceChunker, ParagraphChunker
|
|
from semantica.split.structural_chunker import SectionChunker, HeaderChunker, DocumentChunker
|
|
from semantica.split.table_chunker import TablePreservingChunker, TableContextExtractor, TableAwareChunker
|
|
from semantica.split.provenance_tracker import ChunkProvenanceTracker, SourceTracker, MetadataTracker
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Semantic chunking
|
|
semantic_chunker = SemanticChunker()
|
|
chunks = semantic_chunker.split_by_meaning(long_text)
|
|
topics = semantic_chunker.detect_topics(text)
|
|
semantic_chunks = semantic_chunker.split_by_semantic_similarity(text)
|
|
|
|
# Structural chunking
|
|
structural_chunker = StructuralChunker()
|
|
sections = structural_chunker.split_by_sections(document)
|
|
headers = structural_chunker.identify_headers(document)
|
|
paragraphs = structural_chunker.split_by_paragraphs(document)
|
|
|
|
# Table-aware chunking
|
|
table_chunker = TableChunker()
|
|
table_chunks = table_chunker.preserve_tables(document)
|
|
context = table_chunker.extract_table_context(table)
|
|
table_aware_chunks = table_chunker.split_with_table_context(document)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Sliding Window (`semantica.split.sliding_window`):**
|
|
```python
|
|
from semantica.split.sliding_window import SlidingWindowChunker, OverlapChunker, FixedSizeChunker
|
|
|
|
# Sliding window chunking
|
|
sliding_chunker = SlidingWindowChunker(window_size=512, step_size=256)
|
|
chunks = sliding_chunker.chunk_text(long_text)
|
|
chunks = sliding_chunker.chunk_with_overlap(text, overlap_ratio=0.5)
|
|
|
|
# Overlap chunking
|
|
overlap_chunker = OverlapChunker(chunk_size=500, overlap_size=100)
|
|
chunks = overlap_chunker.chunk_text(text)
|
|
chunks = overlap_chunker.chunk_with_context(text, context_size=50)
|
|
|
|
# Fixed size chunking
|
|
fixed_chunker = FixedSizeChunker(chunk_size=1000)
|
|
chunks = fixed_chunker.chunk_text(text)
|
|
chunks = fixed_chunker.chunk_by_tokens(text, max_tokens=500)
|
|
```
|
|
|
|
**Semantic Chunker (`semantica.split.semantic_chunker`):**
|
|
```python
|
|
from semantica.split.semantic_chunker import TopicBasedChunker, SentenceChunker, ParagraphChunker
|
|
|
|
# Topic-based chunking
|
|
topic_chunker = TopicBasedChunker()
|
|
chunks = topic_chunker.split_by_topics(text)
|
|
topics = topic_chunker.detect_topic_boundaries(text)
|
|
chunks = topic_chunker.split_by_topic_shift(text, threshold=0.7)
|
|
|
|
# Sentence chunking
|
|
sentence_chunker = SentenceChunker()
|
|
chunks = sentence_chunker.split_by_sentences(text)
|
|
chunks = sentence_chunker.split_by_sentence_similarity(text)
|
|
chunks = sentence_chunker.split_by_semantic_coherence(text)
|
|
|
|
# Paragraph chunking
|
|
paragraph_chunker = ParagraphChunker()
|
|
chunks = paragraph_chunker.split_by_paragraphs(text)
|
|
chunks = paragraph_chunker.split_by_paragraph_similarity(text)
|
|
chunks = paragraph_chunker.split_by_paragraph_length(text, max_length=1000)
|
|
```
|
|
|
|
**Structural Chunker (`semantica.split.structural_chunker`):**
|
|
```python
|
|
from semantica.split.structural_chunker import SectionChunker, HeaderChunker, DocumentChunker
|
|
|
|
# Section-based chunking
|
|
section_chunker = SectionChunker()
|
|
chunks = section_chunker.split_by_sections(document)
|
|
sections = section_chunker.identify_sections(document)
|
|
chunks = section_chunker.split_by_section_hierarchy(document)
|
|
|
|
# Header-based chunking
|
|
header_chunker = HeaderChunker()
|
|
chunks = header_chunker.split_by_headers(document)
|
|
headers = header_chunker.extract_headers(document)
|
|
chunks = header_chunker.split_by_header_level(document, level=2)
|
|
|
|
# Document-aware chunking
|
|
document_chunker = DocumentChunker()
|
|
chunks = document_chunker.split_by_document_structure(document)
|
|
chunks = document_chunker.split_by_document_type(document, doc_type="research_paper")
|
|
chunks = document_chunker.split_by_document_sections(document)
|
|
```
|
|
|
|
**Table Chunker (`semantica.split.table_chunker`):**
|
|
```python
|
|
from semantica.split.table_chunker import TablePreservingChunker, TableContextExtractor, TableAwareChunker
|
|
|
|
# Table-preserving chunking
|
|
table_preserver = TablePreservingChunker()
|
|
chunks = table_preserver.split_preserve_tables(document)
|
|
chunks = table_preserver.split_around_tables(document, context_size=200)
|
|
chunks = table_preserver.split_table_aware(document)
|
|
|
|
# Table context extraction
|
|
context_extractor = TableContextExtractor()
|
|
context = context_extractor.extract_table_context(table)
|
|
context = context_extractor.extract_table_metadata(table)
|
|
context = context_extractor.extract_table_relationships(table)
|
|
|
|
# Table-aware chunking
|
|
table_aware = TableAwareChunker()
|
|
chunks = table_aware.split_with_table_context(document)
|
|
chunks = table_aware.split_table_centered(document)
|
|
chunks = table_aware.split_table_relationships(document)
|
|
```
|
|
|
|
**Provenance Tracker (`semantica.split.provenance_tracker`):**
|
|
```python
|
|
from semantica.split.provenance_tracker import ChunkProvenanceTracker, SourceTracker, MetadataTracker
|
|
|
|
# Chunk provenance tracking
|
|
provenance_tracker = ChunkProvenanceTracker()
|
|
chunks = provenance_tracker.track_chunk_sources(text, source="document.pdf")
|
|
chunks = provenance_tracker.track_chunk_metadata(chunks, metadata={"page": 1, "section": "intro"})
|
|
provenance = provenance_tracker.get_chunk_provenance(chunk_id)
|
|
|
|
# Source tracking
|
|
source_tracker = SourceTracker()
|
|
sources = source_tracker.track_sources(chunks)
|
|
sources = source_tracker.track_source_hierarchy(chunks)
|
|
sources = source_tracker.track_source_relationships(chunks)
|
|
|
|
# Metadata tracking
|
|
metadata_tracker = MetadataTracker()
|
|
metadata = metadata_tracker.track_metadata(chunks, metadata_schema)
|
|
metadata = metadata_tracker.track_metadata_changes(chunks, original_metadata)
|
|
metadata = metadata_tracker.track_metadata_provenance(chunks)
|
|
```
|
|
|
|
---
|
|
|
|
## 🧠 Semantic Intelligence
|
|
|
|
### 7. **Semantic Extraction** (`semantica.semantic_extract`)
|
|
|
|
**Main Classes:** `NERExtractor`, `RelationExtractor`, `TripleExtractor`
|
|
|
|
**Purpose:** Extract semantic information from text
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor
|
|
from semantica.semantic_extract.ner_extractor import NERModel, EntityClassifier, EntityLinker
|
|
from semantica.semantic_extract.relation_extractor import RelationModel, RelationClassifier, RelationValidator
|
|
from semantica.semantic_extract.event_detector import EventDetector, TemporalExtractor, EventClassifier
|
|
from semantica.semantic_extract.coref_resolver import CorefResolver, EntityLinker, MentionResolver
|
|
from semantica.semantic_extract.triple_extractor import TripleModel, TripleValidator, TripleFormatter
|
|
from semantica.semantic_extract.llm_enhancer import LLMEnhancer, ComplexExtractor, LLMValidator
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Named Entity Recognition
|
|
ner = NERExtractor()
|
|
entities = ner.extract_entities("Apple Inc. was founded by Steve Jobs in 1976")
|
|
classified = ner.classify_entities(entities)
|
|
linked = ner.link_entities(entities)
|
|
|
|
# Relation Extraction
|
|
rel_extractor = RelationExtractor()
|
|
relations = rel_extractor.find_relations("Apple Inc. was founded by Steve Jobs")
|
|
classified_rels = rel_extractor.classify_relations(relations)
|
|
validated_rels = rel_extractor.validate_relations(relations)
|
|
|
|
# Triple Extraction
|
|
triple_extractor = TripleExtractor()
|
|
triples = triple_extractor.extract_triples(text)
|
|
validated = triple_extractor.validate_triples(triples)
|
|
formatted = triple_extractor.format_triples(triples)
|
|
|
|
# Export triples
|
|
turtle = triple_extractor.to_turtle(triples)
|
|
jsonld = triple_extractor.to_jsonld(triples)
|
|
ntriples = triple_extractor.to_ntriples(triples)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**NER Extractor (`semantica.semantic_extract.ner_extractor`):**
|
|
```python
|
|
from semantica.semantic_extract.ner_extractor import NERModel, EntityClassifier, EntityLinker
|
|
|
|
# NER model
|
|
ner_model = NERModel(model_name="spacy", language="en")
|
|
entities = ner_model.extract_entities(text)
|
|
entities = ner_model.extract_entities_batch(texts)
|
|
entities = ner_model.extract_entities_with_confidence(text)
|
|
|
|
# Entity classification
|
|
entity_classifier = EntityClassifier()
|
|
classified = entity_classifier.classify_entities(entities)
|
|
classified = entity_classifier.classify_by_type(entities, entity_type="PERSON")
|
|
classified = entity_classifier.classify_by_domain(entities, domain="technology")
|
|
|
|
# Entity linking
|
|
entity_linker = EntityLinker()
|
|
linked = entity_linker.link_to_wikidata(entities)
|
|
linked = entity_linker.link_to_dbpedia(entities)
|
|
linked = entity_linker.link_to_custom_kb(entities, knowledge_base)
|
|
```
|
|
|
|
**Relation Extractor (`semantica.semantic_extract.relation_extractor`):**
|
|
```python
|
|
from semantica.semantic_extract.relation_extractor import RelationModel, RelationClassifier, RelationValidator
|
|
|
|
# Relation model
|
|
relation_model = RelationModel(model_name="rebel", language="en")
|
|
relations = relation_model.extract_relations(text)
|
|
relations = relation_model.extract_relations_batch(texts)
|
|
relations = relation_model.extract_relations_with_confidence(text)
|
|
|
|
# Relation classification
|
|
relation_classifier = RelationClassifier()
|
|
classified = relation_classifier.classify_relations(relations)
|
|
classified = relation_classifier.classify_by_type(relations, relation_type="founded_by")
|
|
classified = relation_classifier.classify_by_domain(relations, domain="business")
|
|
|
|
# Relation validation
|
|
relation_validator = RelationValidator()
|
|
validated = relation_validator.validate_relations(relations)
|
|
validated = relation_validator.validate_by_schema(relations, schema)
|
|
validated = relation_validator.validate_by_consistency(relations)
|
|
```
|
|
|
|
**Event Detector (`semantica.semantic_extract.event_detector`):**
|
|
```python
|
|
from semantica.semantic_extract.event_detector import EventDetector, TemporalExtractor, EventClassifier
|
|
|
|
# Event detection
|
|
event_detector = EventDetector()
|
|
events = event_detector.detect_events(text)
|
|
events = event_detector.detect_events_batch(texts)
|
|
events = event_detector.detect_events_with_confidence(text)
|
|
|
|
# Temporal extraction
|
|
temporal_extractor = TemporalExtractor()
|
|
temporal = temporal_extractor.extract_temporal_expressions(text)
|
|
temporal = temporal_extractor.extract_dates(text)
|
|
temporal = temporal_extractor.extract_time_expressions(text)
|
|
|
|
# Event classification
|
|
event_classifier = EventClassifier()
|
|
classified = event_classifier.classify_events(events)
|
|
classified = event_classifier.classify_by_type(events, event_type="founding")
|
|
classified = event_classifier.classify_by_domain(events, domain="business")
|
|
```
|
|
|
|
**Coreference Resolver (`semantica.semantic_extract.coref_resolver`):**
|
|
```python
|
|
from semantica.semantic_extract.coref_resolver import CorefResolver, EntityLinker, MentionResolver
|
|
|
|
# Coreference resolution
|
|
coref_resolver = CorefResolver()
|
|
resolved = coref_resolver.resolve_coreferences(text)
|
|
resolved = coref_resolver.resolve_coreferences_batch(texts)
|
|
resolved = coref_resolver.resolve_coreferences_with_confidence(text)
|
|
|
|
# Entity linking
|
|
entity_linker = EntityLinker()
|
|
linked = entity_linker.link_mentions(mentions)
|
|
linked = entity_linker.link_to_knowledge_base(mentions, kb)
|
|
linked = entity_linker.link_by_similarity(mentions, threshold=0.8)
|
|
|
|
# Mention resolution
|
|
mention_resolver = MentionResolver()
|
|
resolved = mention_resolver.resolve_mentions(mentions)
|
|
resolved = mention_resolver.resolve_pronouns(text)
|
|
resolved = mention_resolver.resolve_nominal_mentions(text)
|
|
```
|
|
|
|
**Triple Extractor (`semantica.semantic_extract.triple_extractor`):**
|
|
```python
|
|
from semantica.semantic_extract.triple_extractor import TripleModel, TripleValidator, TripleFormatter
|
|
|
|
# Triple extraction
|
|
triple_model = TripleModel()
|
|
triples = triple_model.extract_triples(text)
|
|
triples = triple_model.extract_triples_batch(texts)
|
|
triples = triple_model.extract_triples_with_confidence(text)
|
|
|
|
# Triple validation
|
|
triple_validator = TripleValidator()
|
|
validated = triple_validator.validate_triples(triples)
|
|
validated = triple_validator.validate_by_schema(triples, schema)
|
|
validated = triple_validator.validate_by_consistency(triples)
|
|
|
|
# Triple formatting
|
|
triple_formatter = TripleFormatter()
|
|
formatted = triple_formatter.format_as_rdf(triples)
|
|
formatted = triple_formatter.format_as_jsonld(triples)
|
|
formatted = triple_formatter.format_as_turtle(triples)
|
|
```
|
|
|
|
**LLM Enhancer (`semantica.semantic_extract.llm_enhancer`):**
|
|
```python
|
|
from semantica.semantic_extract.llm_enhancer import LLMEnhancer, ComplexExtractor, LLMValidator
|
|
|
|
# LLM enhancement
|
|
llm_enhancer = LLMEnhancer(model="gpt-4")
|
|
enhanced = llm_enhancer.enhance_extraction(text, extraction_type="entities")
|
|
enhanced = llm_enhancer.enhance_relations(text, relations)
|
|
enhanced = llm_enhancer.enhance_triples(text, triples)
|
|
|
|
# Complex extraction
|
|
complex_extractor = ComplexExtractor()
|
|
complex_entities = complex_extractor.extract_complex_entities(text)
|
|
complex_relations = complex_extractor.extract_complex_relations(text)
|
|
complex_events = complex_extractor.extract_complex_events(text)
|
|
|
|
# LLM validation
|
|
llm_validator = LLMValidator()
|
|
validated = llm_validator.validate_with_llm(extractions, text)
|
|
validated = llm_validator.validate_consistency(extractions)
|
|
validated = llm_validator.validate_completeness(extractions, text)
|
|
```
|
|
|
|
### 8. **Ontology Generation** (`semantica.ontology`)
|
|
|
|
**Main Class:** `OntologyGenerator`
|
|
|
|
**Purpose:** Generate ontologies from extracted data
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.ontology import OntologyGenerator
|
|
from semantica.ontology.class_inferrer import ClassInferrer, HierarchyBuilder, ClassValidator
|
|
from semantica.ontology.property_generator import PropertyInferrer, DataTypeInferrer, PropertyValidator
|
|
from semantica.ontology.owl_generator import OWLGenerator, RDFGenerator, TurtleGenerator
|
|
from semantica.ontology.base_mapper import SchemaMapper, FOAFMapper, DublinCoreMapper
|
|
from semantica.ontology.version_manager import VersionManager, MigrationManager, OntologyVersioner
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Initialize ontology generator
|
|
ontology_gen = OntologyGenerator(
|
|
base_ontologies=["schema.org", "foaf", "dublin_core"],
|
|
generate_classes=True,
|
|
generate_properties=True
|
|
)
|
|
|
|
# Generate ontology from documents
|
|
ontology = ontology_gen.generate_from_documents(documents)
|
|
ontology = ontology_gen.generate_from_entities(entities)
|
|
ontology = ontology_gen.generate_from_triples(triples)
|
|
|
|
# Export in various formats
|
|
owl_ontology = ontology.to_owl()
|
|
rdf_ontology = ontology.to_rdf()
|
|
turtle_ontology = ontology.to_turtle()
|
|
jsonld_ontology = ontology.to_jsonld()
|
|
|
|
# Save to triple store
|
|
ontology.save_to_triple_store("http://localhost:9999/blazegraph/sparql")
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Class Inferrer (`semantica.ontology.class_inferrer`):**
|
|
```python
|
|
from semantica.ontology.class_inferrer import ClassInferrer, HierarchyBuilder, ClassValidator
|
|
|
|
# Class inference
|
|
class_inferrer = ClassInferrer()
|
|
classes = class_inferrer.infer_classes(entities)
|
|
classes = class_inferrer.infer_classes_from_triples(triples)
|
|
classes = class_inferrer.infer_classes_from_text(text)
|
|
|
|
# Hierarchy building
|
|
hierarchy_builder = HierarchyBuilder()
|
|
hierarchy = hierarchy_builder.build_hierarchy(classes)
|
|
hierarchy = hierarchy_builder.build_is_a_hierarchy(classes)
|
|
hierarchy = hierarchy_builder.build_part_of_hierarchy(classes)
|
|
|
|
# Class validation
|
|
class_validator = ClassValidator()
|
|
validated = class_validator.validate_classes(classes)
|
|
validated = class_validator.validate_hierarchy(hierarchy)
|
|
validated = class_validator.validate_consistency(classes)
|
|
```
|
|
|
|
**Property Generator (`semantica.ontology.property_generator`):**
|
|
```python
|
|
from semantica.ontology.property_generator import PropertyInferrer, DataTypeInferrer, PropertyValidator
|
|
|
|
# Property inference
|
|
property_inferrer = PropertyInferrer()
|
|
properties = property_inferrer.infer_properties(entities)
|
|
properties = property_inferrer.infer_properties_from_triples(triples)
|
|
properties = property_inferrer.infer_properties_from_text(text)
|
|
|
|
# Data type inference
|
|
datatype_inferrer = DataTypeInferrer()
|
|
datatypes = datatype_inferrer.infer_datatypes(properties)
|
|
datatypes = datatype_inferrer.infer_datatypes_from_values(values)
|
|
datatypes = datatype_inferrer.infer_datatypes_from_schema(schema)
|
|
|
|
# Property validation
|
|
property_validator = PropertyValidator()
|
|
validated = property_validator.validate_properties(properties)
|
|
validated = property_validator.validate_datatypes(properties, datatypes)
|
|
validated = property_validator.validate_domain_range(properties)
|
|
```
|
|
|
|
**OWL Generator (`semantica.ontology.owl_generator`):**
|
|
```python
|
|
from semantica.ontology.owl_generator import OWLGenerator, RDFGenerator, TurtleGenerator
|
|
|
|
# OWL generation
|
|
owl_generator = OWLGenerator()
|
|
owl_ontology = owl_generator.generate_owl(classes, properties)
|
|
owl_ontology = owl_generator.generate_owl_from_triples(triples)
|
|
owl_ontology = owl_generator.generate_owl_with_axioms(classes, properties, axioms)
|
|
|
|
# RDF generation
|
|
rdf_generator = RDFGenerator()
|
|
rdf_ontology = rdf_generator.generate_rdf(classes, properties)
|
|
rdf_ontology = rdf_generator.generate_rdf_from_triples(triples)
|
|
rdf_ontology = rdf_generator.generate_rdf_with_namespaces(classes, properties, namespaces)
|
|
|
|
# Turtle generation
|
|
turtle_generator = TurtleGenerator()
|
|
turtle_ontology = turtle_generator.generate_turtle(classes, properties)
|
|
turtle_ontology = turtle_generator.generate_turtle_from_triples(triples)
|
|
turtle_ontology = turtle_generator.generate_turtle_with_prefixes(classes, properties, prefixes)
|
|
```
|
|
|
|
**Base Mapper (`semantica.ontology.base_mapper`):**
|
|
```python
|
|
from semantica.ontology.base_mapper import SchemaMapper, FOAFMapper, DublinCoreMapper
|
|
|
|
# Schema.org mapping
|
|
schema_mapper = SchemaMapper()
|
|
mapped = schema_mapper.map_to_schema_org(classes, properties)
|
|
mapped = schema_mapper.map_entities_to_schema_org(entities)
|
|
mapped = schema_mapper.map_relations_to_schema_org(relations)
|
|
|
|
# FOAF mapping
|
|
foaf_mapper = FOAFMapper()
|
|
mapped = foaf_mapper.map_to_foaf(classes, properties)
|
|
mapped = foaf_mapper.map_persons_to_foaf(entities)
|
|
mapped = foaf_mapper.map_organizations_to_foaf(entities)
|
|
|
|
# Dublin Core mapping
|
|
dc_mapper = DublinCoreMapper()
|
|
mapped = dc_mapper.map_to_dublin_core(classes, properties)
|
|
mapped = dc_mapper.map_documents_to_dublin_core(documents)
|
|
mapped = dc_mapper.map_metadata_to_dublin_core(metadata)
|
|
```
|
|
|
|
**Version Manager (`semantica.ontology.version_manager`):**
|
|
```python
|
|
from semantica.ontology.version_manager import VersionManager, MigrationManager, OntologyVersioner
|
|
|
|
# Version management
|
|
version_manager = VersionManager()
|
|
version = version_manager.create_version(ontology, version="1.0.0")
|
|
version = version_manager.get_version(ontology, version="1.0.0")
|
|
versions = version_manager.list_versions(ontology)
|
|
|
|
# Migration management
|
|
migration_manager = MigrationManager()
|
|
migrated = migration_manager.migrate_ontology(ontology, from_version="1.0.0", to_version="2.0.0")
|
|
migrated = migration_manager.migrate_triples(triples, migration_rules)
|
|
migrated = migration_manager.migrate_classes(classes, migration_rules)
|
|
|
|
# Ontology versioning
|
|
ontology_versioner = OntologyVersioner()
|
|
versioned = ontology_versioner.version_ontology(ontology)
|
|
versioned = ontology_versioner.compare_versions(ontology_v1, ontology_v2)
|
|
versioned = ontology_versioner.merge_versions(ontology_v1, ontology_v2)
|
|
```
|
|
|
|
### 9. **Knowledge Graph** (`semantica.kg`)
|
|
|
|
**Main Classes:** `GraphBuilder`, `EntityResolver`, `Deduplicator`
|
|
|
|
**Purpose:** Build and manage knowledge graphs
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.kg import GraphBuilder, EntityResolver, Deduplicator
|
|
from semantica.kg.graph_builder import NodeBuilder, EdgeBuilder, SubgraphBuilder
|
|
from semantica.kg.entity_resolver import IdentityResolver, EntityMerger, EntityMatcher
|
|
from semantica.kg.deduplicator import DuplicateFinder, EntityMerger, SimilarityCalculator
|
|
from semantica.kg.graph_analyzer import GraphAnalyzer, PathFinder, CentralityCalculator
|
|
from semantica.kg.graph_optimizer import GraphOptimizer, IndexBuilder, QueryOptimizer
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Build knowledge graph
|
|
graph_builder = GraphBuilder()
|
|
node = graph_builder.create_node("Apple Inc.", "Company")
|
|
edge = graph_builder.create_edge("Apple Inc.", "founded_by", "Steve Jobs")
|
|
subgraph = graph_builder.build_subgraph(entities)
|
|
graph = graph_builder.build_complete_graph(triples)
|
|
|
|
# Entity resolution
|
|
resolver = EntityResolver()
|
|
canonical = resolver.resolve_identity("Apple Inc.", "Apple")
|
|
merged = resolver.merge_entities(duplicate_entities)
|
|
resolved = resolver.resolve_all_entities(entities)
|
|
|
|
# Deduplication
|
|
deduplicator = Deduplicator()
|
|
duplicates = deduplicator.find_duplicates(entities)
|
|
merged = deduplicator.merge_duplicates(duplicates)
|
|
cleaned = deduplicator.clean_graph(graph)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Graph Builder (`semantica.kg.graph_builder`):**
|
|
```python
|
|
from semantica.kg.graph_builder import NodeBuilder, EdgeBuilder, SubgraphBuilder
|
|
|
|
# Node building
|
|
node_builder = NodeBuilder()
|
|
node = node_builder.create_node("Apple Inc.", "Company", properties={"founded": 1976})
|
|
node = node_builder.create_node_with_id("Apple Inc.", "Company", node_id="apple_inc")
|
|
nodes = node_builder.create_nodes_batch(entities)
|
|
|
|
# Edge building
|
|
edge_builder = EdgeBuilder()
|
|
edge = edge_builder.create_edge("Apple Inc.", "founded_by", "Steve Jobs")
|
|
edge = edge_builder.create_edge_with_properties("Apple Inc.", "founded_by", "Steve Jobs", {"year": 1976})
|
|
edges = edge_builder.create_edges_batch(relations)
|
|
|
|
# Subgraph building
|
|
subgraph_builder = SubgraphBuilder()
|
|
subgraph = subgraph_builder.build_subgraph(entities, relations)
|
|
subgraph = subgraph_builder.build_subgraph_by_type(entities, entity_type="Company")
|
|
subgraph = subgraph_builder.build_subgraph_by_relation(entities, relation_type="founded_by")
|
|
```
|
|
|
|
**Entity Resolver (`semantica.kg.entity_resolver`):**
|
|
```python
|
|
from semantica.kg.entity_resolver import IdentityResolver, EntityMerger, EntityMatcher
|
|
|
|
# Identity resolution
|
|
identity_resolver = IdentityResolver()
|
|
canonical = identity_resolver.resolve_identity("Apple Inc.", "Apple")
|
|
canonical = identity_resolver.resolve_identity_batch(entities)
|
|
canonical = identity_resolver.resolve_identity_with_confidence("Apple Inc.", "Apple", threshold=0.8)
|
|
|
|
# Entity merging
|
|
entity_merger = EntityMerger()
|
|
merged = entity_merger.merge_entities(duplicate_entities)
|
|
merged = entity_merger.merge_entities_with_strategy(duplicate_entities, strategy="highest_confidence")
|
|
merged = entity_merger.merge_entities_with_validation(duplicate_entities, validation_rules)
|
|
|
|
# Entity matching
|
|
entity_matcher = EntityMatcher()
|
|
matches = entity_matcher.find_matches("Apple Inc.", entities)
|
|
matches = entity_matcher.find_matches_with_similarity("Apple Inc.", entities, threshold=0.8)
|
|
matches = entity_matcher.find_matches_with_fuzzy("Apple Inc.", entities, fuzzy_threshold=0.7)
|
|
```
|
|
|
|
**Deduplicator (`semantica.kg.deduplicator`):**
|
|
```python
|
|
from semantica.kg.deduplicator import DuplicateFinder, EntityMerger, SimilarityCalculator
|
|
|
|
# Duplicate finding
|
|
duplicate_finder = DuplicateFinder()
|
|
duplicates = duplicate_finder.find_duplicates(entities)
|
|
duplicates = duplicate_finder.find_duplicates_by_similarity(entities, threshold=0.8)
|
|
duplicates = duplicate_finder.find_duplicates_by_fuzzy_matching(entities, fuzzy_threshold=0.7)
|
|
|
|
# Entity merging
|
|
entity_merger = EntityMerger()
|
|
merged = entity_merger.merge_duplicates(duplicates)
|
|
merged = entity_merger.merge_duplicates_with_strategy(duplicates, strategy="merge_properties")
|
|
merged = entity_merger.merge_duplicates_with_validation(duplicates, validation_rules)
|
|
|
|
# Similarity calculation
|
|
similarity_calculator = SimilarityCalculator()
|
|
similarity = similarity_calculator.calculate_similarity("Apple Inc.", "Apple")
|
|
similarity = similarity_calculator.calculate_similarity_batch(entities)
|
|
similarity = similarity_calculator.calculate_similarity_with_weights(entities, weights)
|
|
```
|
|
|
|
**Graph Analyzer (`semantica.kg.graph_analyzer`):**
|
|
```python
|
|
from semantica.kg.graph_analyzer import GraphAnalyzer, PathFinder, CentralityCalculator
|
|
|
|
# Graph analysis
|
|
graph_analyzer = GraphAnalyzer()
|
|
stats = graph_analyzer.analyze_graph(graph)
|
|
stats = graph_analyzer.analyze_connectivity(graph)
|
|
stats = graph_analyzer.analyze_centrality(graph)
|
|
|
|
# Path finding
|
|
path_finder = PathFinder()
|
|
paths = path_finder.find_paths("Apple Inc.", "Steve Jobs", graph)
|
|
paths = path_finder.find_shortest_paths("Apple Inc.", "Steve Jobs", graph)
|
|
paths = path_finder.find_all_paths("Apple Inc.", "Steve Jobs", graph, max_length=3)
|
|
|
|
# Centrality calculation
|
|
centrality_calculator = CentralityCalculator()
|
|
centrality = centrality_calculator.calculate_centrality(graph)
|
|
centrality = centrality_calculator.calculate_betweenness_centrality(graph)
|
|
centrality = centrality_calculator.calculate_eigenvector_centrality(graph)
|
|
```
|
|
|
|
**Graph Optimizer (`semantica.kg.graph_optimizer`):**
|
|
```python
|
|
from semantica.kg.graph_optimizer import GraphOptimizer, IndexBuilder, QueryOptimizer
|
|
|
|
# Graph optimization
|
|
graph_optimizer = GraphOptimizer()
|
|
optimized = graph_optimizer.optimize_graph(graph)
|
|
optimized = graph_optimizer.optimize_for_queries(graph, query_patterns)
|
|
optimized = graph_optimizer.optimize_for_storage(graph)
|
|
|
|
# Index building
|
|
index_builder = IndexBuilder()
|
|
index = index_builder.build_index(graph)
|
|
index = index_builder.build_property_index(graph, properties=["name", "type"])
|
|
index = index_builder.build_relationship_index(graph, relationships=["founded_by"])
|
|
|
|
# Query optimization
|
|
query_optimizer = QueryOptimizer()
|
|
optimized_query = query_optimizer.optimize_query(query, graph)
|
|
optimized_query = query_optimizer.optimize_query_with_index(query, graph, index)
|
|
optimized_query = query_optimizer.optimize_query_with_statistics(query, graph, statistics)
|
|
```
|
|
|
|
**Submodules:**
|
|
- `graph_builder` - Knowledge graph construction
|
|
- `entity_resolver` - Entity disambiguation and merging
|
|
- `deduplicator` - Duplicate detection and resolution
|
|
- `seed_manager` - Initial data loading
|
|
- `provenance_tracker` - Source tracking and confidence
|
|
- `conflict_detector` - Conflict identification and resolution
|
|
|
|
---
|
|
|
|
## 💾 Storage & Retrieval
|
|
|
|
### 10. **Vector Store** (`semantica.vector_store`)
|
|
|
|
**Main Classes:** `PineconeAdapter`, `FAISSAdapter`, `WeaviateAdapter`
|
|
|
|
**Purpose:** Store and search vector embeddings
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.vector_store import PineconeAdapter, FAISSAdapter, WeaviateAdapter
|
|
from semantica.vector_store.pinecone_adapter import PineconeIndex, PineconeQuery, PineconeMetadata
|
|
from semantica.vector_store.faiss_adapter import FAISSIndex, FAISSSearch, FAISSIndexBuilder
|
|
from semantica.vector_store.milvus_adapter import MilvusClient, MilvusCollection, MilvusSearch
|
|
from semantica.vector_store.weaviate_adapter import WeaviateClient, WeaviateSchema, WeaviateQuery
|
|
from semantica.vector_store.qdrant_adapter import QdrantClient, QdrantCollection, QdrantSearch
|
|
from semantica.vector_store.hybrid_search import HybridSearch, MetadataFilter, SearchRanker
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Pinecone integration
|
|
pinecone = PineconeAdapter()
|
|
pinecone.connect(api_key="your-key")
|
|
pinecone.create_index("semantica-index", dimension=1536)
|
|
pinecone.upsert_vectors(vectors, metadata)
|
|
results = pinecone.query_vectors(query_vector, top_k=10)
|
|
|
|
# FAISS integration
|
|
faiss = FAISSAdapter()
|
|
faiss.create_index("IVFFlat", dimension=1536)
|
|
faiss.add_vectors(vectors)
|
|
faiss.save_index("index.faiss")
|
|
similar = faiss.search_similar(query_vector, k=10)
|
|
|
|
# Weaviate integration
|
|
weaviate = WeaviateAdapter()
|
|
weaviate.connect(url="http://localhost:8080")
|
|
weaviate.create_schema(schema_definition)
|
|
weaviate.insert_objects(objects)
|
|
results = weaviate.query_objects(query, class_name="Document")
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Pinecone Adapter (`semantica.vector_store.pinecone_adapter`):**
|
|
```python
|
|
from semantica.vector_store.pinecone_adapter import PineconeIndex, PineconeQuery, PineconeMetadata
|
|
|
|
# Index management
|
|
pinecone_index = PineconeIndex()
|
|
pinecone_index.create_index("semantica-index", dimension=1536, metric="cosine")
|
|
pinecone_index.describe_index("semantica-index")
|
|
pinecone_index.delete_index("semantica-index")
|
|
|
|
# Vector operations
|
|
pinecone_index.upsert_vectors(vectors, metadata, namespace="documents")
|
|
pinecone_index.update_vectors(vectors, metadata)
|
|
pinecone_index.delete_vectors(ids)
|
|
|
|
# Query operations
|
|
pinecone_query = PineconeQuery()
|
|
results = pinecone_query.query_vectors(query_vector, top_k=10, include_metadata=True)
|
|
results = pinecone_query.query_with_filter(query_vector, filter={"category": "technology"})
|
|
results = pinecone_query.query_by_id(vector_id)
|
|
|
|
# Metadata operations
|
|
pinecone_metadata = PineconeMetadata()
|
|
metadata = pinecone_metadata.create_metadata({"text": "sample", "category": "tech"})
|
|
metadata = pinecone_metadata.validate_metadata(metadata)
|
|
metadata = pinecone_metadata.filter_metadata(metadata, {"category": "technology"})
|
|
```
|
|
|
|
**FAISS Adapter (`semantica.vector_store.faiss_adapter`):**
|
|
```python
|
|
from semantica.vector_store.faiss_adapter import FAISSIndex, FAISSSearch, FAISSIndexBuilder
|
|
|
|
# Index building
|
|
index_builder = FAISSIndexBuilder()
|
|
index = index_builder.build_index("IVFFlat", dimension=1536, nlist=100)
|
|
index = index_builder.build_index("HNSW", dimension=1536, M=16)
|
|
index = index_builder.build_index("PQ", dimension=1536, m=64)
|
|
|
|
# Index operations
|
|
faiss_index = FAISSIndex()
|
|
faiss_index.add_vectors(vectors)
|
|
faiss_index.add_vectors_with_ids(vectors, ids)
|
|
faiss_index.remove_vectors(ids)
|
|
faiss_index.save_index("index.faiss")
|
|
faiss_index.load_index("index.faiss")
|
|
|
|
# Search operations
|
|
faiss_search = FAISSSearch()
|
|
results = faiss_search.search_similar(query_vector, k=10)
|
|
results = faiss_search.search_with_ids(query_vector, ids, k=10)
|
|
results = faiss_search.search_range(query_vector, radius=0.5)
|
|
```
|
|
|
|
**Weaviate Adapter (`semantica.vector_store.weaviate_adapter`):**
|
|
```python
|
|
from semantica.vector_store.weaviate_adapter import WeaviateClient, WeaviateSchema, WeaviateQuery
|
|
|
|
# Client operations
|
|
weaviate_client = WeaviateClient()
|
|
weaviate_client.connect(url="http://localhost:8080")
|
|
weaviate_client.get_cluster_info()
|
|
weaviate_client.get_schema()
|
|
|
|
# Schema operations
|
|
weaviate_schema = WeaviateSchema()
|
|
weaviate_schema.create_class(class_definition)
|
|
weaviate_schema.update_class(class_name, class_definition)
|
|
weaviate_schema.delete_class(class_name)
|
|
|
|
# Query operations
|
|
weaviate_query = WeaviateQuery()
|
|
results = weaviate_query.query_objects(query, class_name="Document")
|
|
results = weaviate_query.query_with_filters(query, filters={"category": "technology"})
|
|
results = weaviate_query.query_similar(query_vector, class_name="Document", limit=10)
|
|
```
|
|
|
|
**Hybrid Search (`semantica.vector_store.hybrid_search`):**
|
|
```python
|
|
from semantica.vector_store.hybrid_search import HybridSearch, MetadataFilter, SearchRanker
|
|
|
|
# Hybrid search
|
|
hybrid_search = HybridSearch()
|
|
results = hybrid_search.search(query, vector_weight=0.7, metadata_weight=0.3)
|
|
results = hybrid_search.search_with_filters(query, filters={"category": "technology"})
|
|
results = hybrid_search.search_with_reranking(query, rerank_method="diversity")
|
|
|
|
# Metadata filtering
|
|
metadata_filter = MetadataFilter()
|
|
filtered = metadata_filter.filter_by_metadata(results, {"category": "technology"})
|
|
filtered = metadata_filter.filter_by_range(results, "date", start="2023-01-01", end="2023-12-31")
|
|
filtered = metadata_filter.filter_by_boolean(results, {"is_published": True})
|
|
|
|
# Search ranking
|
|
search_ranker = SearchRanker()
|
|
ranked = search_ranker.rank_by_relevance(results, query)
|
|
ranked = search_ranker.rank_by_diversity(results, diversity_threshold=0.8)
|
|
ranked = search_ranker.rank_by_hybrid_score(results, vector_score=0.6, metadata_score=0.4)
|
|
```
|
|
|
|
### 11. **Triple Store** (`semantica.triple_store`)
|
|
|
|
**Main Classes:** `BlazegraphAdapter`, `JenaAdapter`, `GraphDBAdapter`
|
|
|
|
**Purpose:** Store and query RDF triples
|
|
|
|
```python
|
|
from semantica.triple_store import BlazegraphAdapter, JenaAdapter
|
|
|
|
# Blazegraph integration
|
|
blazegraph = BlazegraphAdapter()
|
|
blazegraph.connect("http://localhost:9999/blazegraph")
|
|
blazegraph.bulk_load(triples)
|
|
|
|
# SPARQL queries
|
|
sparql_query = """
|
|
SELECT ?subject ?predicate ?object
|
|
WHERE { ?subject ?predicate ?object }
|
|
LIMIT 10
|
|
"""
|
|
results = blazegraph.execute_sparql(sparql_query)
|
|
|
|
# Jena integration
|
|
jena = JenaAdapter()
|
|
model = jena.create_model()
|
|
jena.add_triples(model, triples)
|
|
inferred = jena.run_inference(model)
|
|
```
|
|
|
|
**Submodules:**
|
|
- `blazegraph_adapter` - Blazegraph SPARQL endpoint
|
|
- `jena_adapter` - Apache Jena RDF framework
|
|
- `rdf4j_adapter` - Eclipse RDF4J
|
|
- `graphdb_adapter` - GraphDB with reasoning
|
|
- `virtuoso_adapter` - Virtuoso RDF store
|
|
|
|
### 12. **Embeddings** (`semantica.embeddings`)
|
|
|
|
**Main Class:** `SemanticEmbedder`
|
|
|
|
**Purpose:** Generate semantic embeddings for text and multimodal content
|
|
|
|
```python
|
|
from semantica.embeddings import SemanticEmbedder
|
|
|
|
# Initialize embedder
|
|
embedder = SemanticEmbedder(
|
|
model="text-embedding-3-large",
|
|
dimension=1536,
|
|
preserve_context=True
|
|
)
|
|
|
|
# Generate embeddings
|
|
text_embeddings = embedder.embed_text("Hello world")
|
|
sentence_embeddings = embedder.embed_sentence("This is a sentence")
|
|
document_embeddings = embedder.embed_document(long_document)
|
|
|
|
# Batch processing
|
|
batch_embeddings = embedder.batch_process(texts)
|
|
stats = embedder.get_embedding_stats()
|
|
```
|
|
|
|
**Submodules:**
|
|
- `text_embedder` - Text-based embeddings
|
|
- `image_embedder` - Image embeddings and vision models
|
|
- `audio_embedder` - Audio embeddings and speech recognition
|
|
- `multimodal_embedder` - Cross-modal embeddings
|
|
- `context_manager` - Context window management
|
|
- `pooling_strategies` - Various pooling strategies
|
|
|
|
---
|
|
|
|
## 🤖 AI & Reasoning
|
|
|
|
### 13. **RAG System** (`semantica.qa_rag`)
|
|
|
|
**Main Classes:** `RAGManager`, `SemanticChunker`, `AnswerBuilder`
|
|
|
|
**Purpose:** Question answering and retrieval-augmented generation
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.qa_rag import RAGManager, SemanticChunker, AnswerBuilder
|
|
from semantica.qa_rag.semantic_chunker import RAGChunker, ContextChunker, OverlapChunker
|
|
from semantica.qa_rag.prompt_templates import PromptTemplate, ContextTemplate, AnswerTemplate
|
|
from semantica.qa_rag.retrieval_policies import RetrievalPolicy, RankingPolicy, FilterPolicy
|
|
from semantica.qa_rag.answer_builder import AnswerBuilder, AttributionBuilder, ConfidenceCalculator
|
|
from semantica.qa_rag.provenance_tracker import SourceTracker, ConfidenceTracker, AttributionTracker
|
|
from semantica.qa_rag.conversation_manager import ConversationManager, ContextManager, HistoryManager
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Initialize RAG system
|
|
rag = RAGManager(
|
|
retriever="semantic",
|
|
generator="gpt-4",
|
|
chunk_size=512,
|
|
overlap=50
|
|
)
|
|
|
|
# Process question
|
|
question = "What are the key features of Semantica?"
|
|
answer = rag.process_question(question)
|
|
sources = rag.get_sources()
|
|
confidence = rag.get_confidence()
|
|
|
|
# Semantic chunking for RAG
|
|
chunker = SemanticChunker()
|
|
chunks = chunker.chunk_text(document, optimize_for_rag=True)
|
|
chunks = chunker.chunk_with_context(document, context_window=200)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Semantic Chunker (`semantica.qa_rag.semantic_chunker`):**
|
|
```python
|
|
from semantica.qa_rag.semantic_chunker import RAGChunker, ContextChunker, OverlapChunker
|
|
|
|
# RAG-optimized chunking
|
|
rag_chunker = RAGChunker()
|
|
chunks = rag_chunker.chunk_for_rag(document, chunk_size=512)
|
|
chunks = rag_chunker.chunk_with_semantic_boundaries(document)
|
|
chunks = rag_chunker.chunk_with_entity_preservation(document)
|
|
|
|
# Context-aware chunking
|
|
context_chunker = ContextChunker()
|
|
chunks = context_chunker.chunk_with_context(document, context_size=100)
|
|
chunks = context_chunker.chunk_with_overlap_context(document, overlap=50)
|
|
chunks = context_chunker.chunk_with_semantic_context(document)
|
|
|
|
# Overlap chunking
|
|
overlap_chunker = OverlapChunker()
|
|
chunks = overlap_chunker.chunk_with_overlap(document, overlap_ratio=0.2)
|
|
chunks = overlap_chunker.chunk_with_sliding_window(document, window_size=512, step=256)
|
|
chunks = overlap_chunker.chunk_with_adaptive_overlap(document)
|
|
```
|
|
|
|
**Prompt Templates (`semantica.qa_rag.prompt_templates`):**
|
|
```python
|
|
from semantica.qa_rag.prompt_templates import PromptTemplate, ContextTemplate, AnswerTemplate
|
|
|
|
# Prompt template management
|
|
prompt_template = PromptTemplate()
|
|
template = prompt_template.create_template("qa_template", question="{question}", context="{context}")
|
|
template = prompt_template.create_template("summarization", text="{text}", length="{length}")
|
|
template = prompt_template.create_template("classification", text="{text}", categories="{categories}")
|
|
|
|
# Context templates
|
|
context_template = ContextTemplate()
|
|
context = context_template.format_context(retrieved_chunks, question)
|
|
context = context_template.format_context_with_metadata(retrieved_chunks, question, metadata)
|
|
context = context_template.format_context_with_ranking(retrieved_chunks, question, rankings)
|
|
|
|
# Answer templates
|
|
answer_template = AnswerTemplate()
|
|
answer = answer_template.format_answer(response, sources)
|
|
answer = answer_template.format_answer_with_attribution(response, sources, attributions)
|
|
answer = answer_template.format_answer_with_confidence(response, sources, confidence_scores)
|
|
```
|
|
|
|
**Retrieval Policies (`semantica.qa_rag.retrieval_policies`):**
|
|
```python
|
|
from semantica.qa_rag.retrieval_policies import RetrievalPolicy, RankingPolicy, FilterPolicy
|
|
|
|
# Retrieval policy management
|
|
retrieval_policy = RetrievalPolicy()
|
|
results = retrieval_policy.retrieve(query, top_k=10)
|
|
results = retrieval_policy.retrieve_with_filters(query, filters={"category": "technology"})
|
|
results = retrieval_policy.retrieve_with_reranking(query, rerank_method="diversity")
|
|
|
|
# Ranking policies
|
|
ranking_policy = RankingPolicy()
|
|
ranked = ranking_policy.rank_by_relevance(results, query)
|
|
ranked = ranking_policy.rank_by_diversity(results, diversity_threshold=0.8)
|
|
ranked = ranking_policy.rank_by_hybrid_score(results, vector_weight=0.7, metadata_weight=0.3)
|
|
|
|
# Filter policies
|
|
filter_policy = FilterPolicy()
|
|
filtered = filter_policy.filter_by_metadata(results, {"category": "technology"})
|
|
filtered = filter_policy.filter_by_date_range(results, start_date="2023-01-01", end_date="2023-12-31")
|
|
filtered = filter_policy.filter_by_confidence(results, min_confidence=0.8)
|
|
```
|
|
|
|
**Answer Builder (`semantica.qa_rag.answer_builder`):**
|
|
```python
|
|
from semantica.qa_rag.answer_builder import AnswerBuilder, AttributionBuilder, ConfidenceCalculator
|
|
|
|
# Answer construction
|
|
answer_builder = AnswerBuilder()
|
|
answer = answer_builder.build_answer(query, retrieved_chunks, llm_response)
|
|
answer = answer_builder.build_answer_with_sources(query, retrieved_chunks, llm_response, sources)
|
|
answer = answer_builder.build_answer_with_confidence(query, retrieved_chunks, llm_response, confidence)
|
|
|
|
# Attribution building
|
|
attribution_builder = AttributionBuilder()
|
|
attributions = attribution_builder.build_attributions(answer, sources)
|
|
attributions = attribution_builder.build_attributions_with_confidence(answer, sources, confidence_scores)
|
|
attributions = attribution_builder.build_attributions_with_metadata(answer, sources, metadata)
|
|
|
|
# Confidence calculation
|
|
confidence_calculator = ConfidenceCalculator()
|
|
confidence = confidence_calculator.calculate_confidence(answer, sources)
|
|
confidence = confidence_calculator.calculate_confidence_with_llm(answer, sources, llm_confidence)
|
|
confidence = confidence_calculator.calculate_confidence_with_retrieval(answer, sources, retrieval_scores)
|
|
```
|
|
|
|
**Conversation Manager (`semantica.qa_rag.conversation_manager`):**
|
|
```python
|
|
from semantica.qa_rag.conversation_manager import ConversationManager, ContextManager, HistoryManager
|
|
|
|
# Conversation management
|
|
conversation_manager = ConversationManager()
|
|
conversation = conversation_manager.start_conversation()
|
|
conversation = conversation_manager.add_turn(conversation, question, answer)
|
|
conversation = conversation_manager.get_conversation_history(conversation_id)
|
|
|
|
# Context management
|
|
context_manager = ContextManager()
|
|
context = context_manager.build_context(conversation_history)
|
|
context = context_manager.build_context_with_entities(conversation_history, entities)
|
|
context = context_manager.build_context_with_topics(conversation_history, topics)
|
|
|
|
# History management
|
|
history_manager = HistoryManager()
|
|
history = history_manager.save_conversation(conversation)
|
|
history = history_manager.load_conversation(conversation_id)
|
|
history = history_manager.search_conversations(query, filters={"user_id": "user123"})
|
|
```
|
|
|
|
### 14. **Reasoning Engine** (`semantica.reasoning`)
|
|
|
|
**Main Classes:** `InferenceEngine`, `SPARQLReasoner`, `AbductiveReasoner`
|
|
|
|
**Purpose:** Logical reasoning and inference
|
|
|
|
```python
|
|
from semantica.reasoning import InferenceEngine, SPARQLReasoner
|
|
|
|
# Rule-based inference
|
|
inference = InferenceEngine()
|
|
inference.add_rule("IF ?x is_a Company AND ?x founded_by ?y THEN ?y is_a Person")
|
|
inference.forward_chain()
|
|
inference.backward_chain()
|
|
|
|
# SPARQL reasoning
|
|
sparql_reasoner = SPARQLReasoner()
|
|
expanded_query = sparql_reasoner.expand_query(sparql_query)
|
|
inferred_results = sparql_reasoner.infer_results(query_results)
|
|
```
|
|
|
|
**Submodules:**
|
|
- `inference_engine` - Rule-based inference
|
|
- `sparql_reasoner` - SPARQL-based reasoning
|
|
- `rete_engine` - Rete algorithm implementation
|
|
- `abductive_reasoner` - Abductive reasoning
|
|
- `deductive_reasoner` - Deductive reasoning
|
|
- `explanation_generator` - Reasoning explanations
|
|
|
|
### 15. **Multi-Agent System** (`semantica.agents`)
|
|
|
|
**Main Classes:** `AgentManager`, `OrchestrationEngine`, `MultiAgentManager`
|
|
|
|
**Purpose:** Multi-agent coordination and workflows
|
|
|
|
```python
|
|
from semantica.agents import AgentManager, OrchestrationEngine
|
|
|
|
# Agent management
|
|
agent_manager = AgentManager()
|
|
agent = agent_manager.register_agent("data_processor", capabilities=["parse", "extract"])
|
|
agent_manager.start_agent(agent)
|
|
|
|
# Multi-agent orchestration
|
|
orchestrator = OrchestrationEngine()
|
|
workflow = orchestrator.coordinate_agents([
|
|
"ingestion_agent",
|
|
"parsing_agent",
|
|
"extraction_agent",
|
|
"embedding_agent"
|
|
])
|
|
results = orchestrator.distribute_tasks(workflow, tasks)
|
|
```
|
|
|
|
**Submodules:**
|
|
- `agent_manager` - Agent lifecycle management
|
|
- `orchestration_engine` - Multi-agent coordination
|
|
- `tool_registry` - Tool registration and discovery
|
|
- `cost_tracker` - Cost monitoring and optimization
|
|
- `sandbox_manager` - Agent sandboxing and security
|
|
- `workflow_engine` - Workflow definition and execution
|
|
|
|
---
|
|
|
|
## 🚀 Quick Start Examples
|
|
|
|
### Complete Pipeline Example
|
|
|
|
```python
|
|
from semantica import Semantica
|
|
from semantica.pipeline import PipelineBuilder
|
|
|
|
# Initialize Semantica
|
|
core = Semantica(
|
|
llm_provider="openai",
|
|
embedding_model="text-embedding-3-large",
|
|
vector_store="pinecone",
|
|
graph_db="neo4j"
|
|
)
|
|
|
|
# Build processing pipeline
|
|
pipeline = PipelineBuilder() \
|
|
.add_step("ingest", {"source": "documents/", "formats": ["pdf", "docx"]}) \
|
|
.add_step("parse", {"extract_tables": True, "extract_images": True}) \
|
|
.add_step("normalize", {"clean_text": True, "detect_language": True}) \
|
|
.add_step("chunk", {"strategy": "semantic", "size": 512}) \
|
|
.add_step("extract", {"entities": True, "relations": True, "triples": True}) \
|
|
.add_step("embed", {"model": "text-embedding-3-large"}) \
|
|
.add_step("store", {"vector_store": "pinecone", "triple_store": "neo4j"}) \
|
|
.set_parallelism(4) \
|
|
.build()
|
|
|
|
# Execute pipeline
|
|
results = pipeline.run()
|
|
|
|
# Query results
|
|
knowledge_base = core.build_knowledge_base("documents/")
|
|
answer = knowledge_base.query("What are the main topics?")
|
|
```
|
|
|
|
|
|
---
|
|
|
|
## 📚 Additional Resources
|
|
|
|
- **Documentation**: [https://semantica.readthedocs.io/](https://semantica.readthedocs.io/)
|
|
- **API Reference**: [https://semantica.readthedocs.io/api/](https://semantica.readthedocs.io/api/)
|
|
- **Examples Repository**: [https://github.com/semantica/examples](https://github.com/semantica/examples)
|
|
- **Community**: [https://discord.gg/semantica](https://discord.gg/semantica)
|
|
|
|
---
|
|
|
|
## 🔧 Knowledge Graph Quality Assurance
|
|
|
|
> **Addressing the fundamental challenges in building production-ready Knowledge Graphs**
|
|
|
|
### 16. **Template System** (`semantica.templates`)
|
|
|
|
**Main Classes:** `SchemaTemplate`, `EntityTemplate`, `RelationshipTemplate`
|
|
|
|
**Purpose:** Enforce fixed, predefined schemas for consistent and predictable Knowledge Graph structure
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.templates import SchemaTemplate, EntityTemplate, RelationshipTemplate
|
|
from semantica.templates.schema_manager import SchemaManager, SchemaValidator, SchemaEnforcer
|
|
from semantica.templates.entity_templates import EntityTemplateManager, EntityValidator, EntityEnforcer
|
|
from semantica.templates.relationship_templates import RelationshipTemplateManager, RelationshipValidator
|
|
from semantica.templates.template_loader import TemplateLoader, TemplateParser, TemplateCompiler
|
|
from semantica.templates.constraint_enforcer import ConstraintEnforcer, ValidationEngine, ComplianceChecker
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Define fixed schema template
|
|
schema_template = SchemaTemplate(
|
|
name="business_knowledge_graph",
|
|
entities=["Company", "Person", "Product", "Department", "Project"],
|
|
relationships=["founded_by", "works_for", "manages", "belongs_to", "reports_to"],
|
|
constraints={
|
|
"Company": {"required_props": ["name", "industry", "founded_year"]},
|
|
"Person": {"required_props": ["name", "title", "department"]},
|
|
"founded_by": {"domain": "Company", "range": "Person"}
|
|
}
|
|
)
|
|
|
|
# Enforce template compliance
|
|
schema_template.enforce_schema(extracted_entities)
|
|
schema_template.validate_relationships(extracted_relations)
|
|
schema_template.apply_constraints(knowledge_graph)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Schema Manager (`semantica.templates.schema_manager`):**
|
|
```python
|
|
from semantica.templates.schema_manager import SchemaManager, SchemaValidator, SchemaEnforcer
|
|
|
|
# Schema management
|
|
schema_manager = SchemaManager()
|
|
schema = schema_manager.load_schema("business_schema.yaml")
|
|
schema = schema_manager.create_schema_from_template(template)
|
|
schema = schema_manager.update_schema(schema, updates)
|
|
|
|
# Schema validation
|
|
schema_validator = SchemaValidator()
|
|
is_valid = schema_validator.validate_entities(entities, schema)
|
|
is_valid = schema_validator.validate_relationships(relationships, schema)
|
|
is_valid = schema_validator.validate_properties(properties, schema)
|
|
|
|
# Schema enforcement
|
|
schema_enforcer = SchemaEnforcer()
|
|
enforced = schema_enforcer.enforce_entity_schema(entities, schema)
|
|
enforced = schema_enforcer.enforce_relationship_schema(relationships, schema)
|
|
enforced = schema_enforcer.enforce_property_schema(properties, schema)
|
|
```
|
|
|
|
**Entity Templates (`semantica.templates.entity_templates`):**
|
|
```python
|
|
from semantica.templates.entity_templates import EntityTemplateManager, EntityValidator, EntityEnforcer
|
|
|
|
# Entity template management
|
|
entity_template_manager = EntityTemplateManager()
|
|
template = entity_template_manager.create_template("Company", required_props=["name", "industry"])
|
|
template = entity_template_manager.load_template("person_template.yaml")
|
|
template = entity_template_manager.update_template(template, new_constraints)
|
|
|
|
# Entity validation
|
|
entity_validator = EntityValidator()
|
|
is_valid = entity_validator.validate_entity(entity, template)
|
|
is_valid = entity_validator.validate_entity_batch(entities, template)
|
|
is_valid = entity_validator.validate_required_properties(entity, template)
|
|
|
|
# Entity enforcement
|
|
entity_enforcer = EntityEnforcer()
|
|
enforced = entity_enforcer.enforce_template(entity, template)
|
|
enforced = entity_enforcer.add_missing_properties(entity, template)
|
|
enforced = entity_enforcer.normalize_entity(entity, template)
|
|
```
|
|
|
|
### 17. **Seed Data System** (`semantica.seed`)
|
|
|
|
**Main Classes:** `SeedDataManager`, `KnowledgeSeeder`, `DataIntegrator`
|
|
|
|
**Purpose:** Initialize Knowledge Graph with pre-existing, verified data to build on foundation of truth
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.seed import SeedDataManager, KnowledgeSeeder, DataIntegrator
|
|
from semantica.seed.data_loader import CSVLoader, JSONLoader, DatabaseLoader, APILoader
|
|
from semantica.seed.entity_seeder import EntitySeeder, RelationshipSeeder, PropertySeeder
|
|
from semantica.seed.verification import DataVerifier, ConsistencyChecker, TruthValidator
|
|
from semantica.seed.integration import DataIntegrator, ConflictResolver, MergeStrategy
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Initialize with seed data
|
|
seed_manager = SeedDataManager()
|
|
seed_manager.load_products("products.csv")
|
|
seed_manager.load_departments("departments.json")
|
|
seed_manager.load_employees("employees_db")
|
|
|
|
# Seed knowledge graph
|
|
knowledge_seeder = KnowledgeSeeder()
|
|
seeded_graph = knowledge_seeder.seed_entities(seed_data)
|
|
seeded_graph = knowledge_seeder.seed_relationships(seed_data)
|
|
seeded_graph = knowledge_seeder.seed_properties(seed_data)
|
|
|
|
# Integrate with extracted data
|
|
integrator = DataIntegrator()
|
|
integrated = integrator.integrate_seed_with_extracted(seed_data, extracted_data)
|
|
integrated = integrator.merge_verified_data(seed_data, extracted_data)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Data Loader (`semantica.seed.data_loader`):**
|
|
```python
|
|
from semantica.seed.data_loader import CSVLoader, JSONLoader, DatabaseLoader, APILoader
|
|
|
|
# CSV data loading
|
|
csv_loader = CSVLoader()
|
|
products = csv_loader.load_entities("products.csv", entity_type="Product")
|
|
departments = csv_loader.load_entities("departments.csv", entity_type="Department")
|
|
|
|
# JSON data loading
|
|
json_loader = JSONLoader()
|
|
employees = json_loader.load_entities("employees.json", entity_type="Person")
|
|
companies = json_loader.load_entities("companies.json", entity_type="Company")
|
|
|
|
# Database loading
|
|
db_loader = DatabaseLoader(connection_string="postgresql://...")
|
|
customers = db_loader.load_entities("customers", entity_type="Person")
|
|
orders = db_loader.load_relationships("orders", relationship_type="placed_by")
|
|
|
|
# API loading
|
|
api_loader = APILoader(api_key="your_key")
|
|
external_data = api_loader.load_from_api("https://api.example.com/entities")
|
|
```
|
|
|
|
**Entity Seeder (`semantica.seed.entity_seeder`):**
|
|
```python
|
|
from semantica.seed.entity_seeder import EntitySeeder, RelationshipSeeder, PropertySeeder
|
|
|
|
# Entity seeding
|
|
entity_seeder = EntitySeeder()
|
|
seeded = entity_seeder.seed_entities(products, entity_type="Product")
|
|
seeded = entity_seeder.seed_with_verification(employees, verification_rules)
|
|
seeded = entity_seeder.seed_with_metadata(companies, metadata={"source": "verified"})
|
|
|
|
# Relationship seeding
|
|
relationship_seeder = RelationshipSeeder()
|
|
seeded = relationship_seeder.seed_relationships(org_chart, relationship_type="reports_to")
|
|
seeded = relationship_seeder.seed_with_hierarchy(relationships, hierarchy_rules)
|
|
|
|
# Property seeding
|
|
property_seeder = PropertySeeder()
|
|
seeded = property_seeder.seed_properties(entities, property_mappings)
|
|
seeded = property_seeder.seed_with_validation(properties, validation_rules)
|
|
```
|
|
|
|
### 18. **Advanced Deduplication** (`semantica.deduplication`)
|
|
|
|
**Main Classes:** `DuplicateDetector`, `EntityMerger`, `SimilarityEngine`
|
|
|
|
**Purpose:** Identify and merge duplicate entities like "First Quarter Sales" and "Q1 Sales Report"
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.deduplication import DuplicateDetector, EntityMerger, SimilarityEngine
|
|
from semantica.deduplication.similarity_calculator import SemanticSimilarity, FuzzyMatcher, PhoneticMatcher
|
|
from semantica.deduplication.merge_strategies import MergeStrategy, PropertyMerger, RelationshipMerger
|
|
from semantica.deduplication.conflict_resolver import ConflictResolver, MergeConflictHandler, ResolutionStrategy
|
|
from semantica.deduplication.quality_assessor import QualityAssessor, MergeQualityChecker, ConfidenceCalculator
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Detect duplicates
|
|
duplicate_detector = DuplicateDetector()
|
|
duplicates = duplicate_detector.find_semantic_duplicates(entities)
|
|
duplicates = duplicate_detector.find_fuzzy_duplicates(entities, threshold=0.8)
|
|
duplicates = duplicate_detector.find_phonetic_duplicates(entities)
|
|
|
|
# Merge duplicates
|
|
entity_merger = EntityMerger()
|
|
merged = entity_merger.merge_duplicates(duplicates)
|
|
merged = entity_merger.merge_with_strategy(duplicates, strategy="highest_confidence")
|
|
merged = entity_merger.merge_with_validation(duplicates, validation_rules)
|
|
|
|
# Calculate similarity
|
|
similarity_engine = SimilarityEngine()
|
|
similarity = similarity_engine.calculate_semantic_similarity("First Quarter Sales", "Q1 Sales Report")
|
|
similarity = similarity_engine.calculate_fuzzy_similarity("Apple Inc.", "Apple Corporation")
|
|
similarity = similarity_engine.calculate_phonetic_similarity("Smith", "Smyth")
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Similarity Calculator (`semantica.deduplication.similarity_calculator`):**
|
|
```python
|
|
from semantica.deduplication.similarity_calculator import SemanticSimilarity, FuzzyMatcher, PhoneticMatcher
|
|
|
|
# Semantic similarity
|
|
semantic_similarity = SemanticSimilarity()
|
|
score = semantic_similarity.calculate("First Quarter Sales", "Q1 Sales Report")
|
|
score = semantic_similarity.calculate_batch(entity_pairs)
|
|
score = semantic_similarity.calculate_with_embeddings(entity1, entity2)
|
|
|
|
# Fuzzy matching
|
|
fuzzy_matcher = FuzzyMatcher()
|
|
matches = fuzzy_matcher.find_matches("Apple Inc.", entities, threshold=0.8)
|
|
matches = fuzzy_matcher.find_partial_matches("Apple", entities, threshold=0.6)
|
|
matches = fuzzy_matcher.find_approximate_matches("Q1", entities, threshold=0.7)
|
|
|
|
# Phonetic matching
|
|
phonetic_matcher = PhoneticMatcher()
|
|
matches = phonetic_matcher.find_phonetic_matches("Smith", entities)
|
|
matches = phonetic_matcher.find_soundex_matches("Johnson", entities)
|
|
matches = phonetic_matcher.find_metaphone_matches("Knight", entities)
|
|
```
|
|
|
|
**Merge Strategies (`semantica.deduplication.merge_strategies`):**
|
|
```python
|
|
from semantica.deduplication.merge_strategies import MergeStrategy, PropertyMerger, RelationshipMerger
|
|
|
|
# Merge strategy management
|
|
merge_strategy = MergeStrategy()
|
|
merged = merge_strategy.merge_by_confidence(duplicates)
|
|
merged = merge_strategy.merge_by_completeness(duplicates)
|
|
merged = merge_strategy.merge_by_authority(duplicates, authority_sources)
|
|
|
|
# Property merging
|
|
property_merger = PropertyMerger()
|
|
merged = property_merger.merge_properties(duplicate_entities)
|
|
merged = property_merger.merge_with_priority(duplicate_entities, priority_rules)
|
|
merged = property_merger.merge_with_validation(duplicate_entities, validation_rules)
|
|
|
|
# Relationship merging
|
|
relationship_merger = RelationshipMerger()
|
|
merged = relationship_merger.merge_relationships(duplicate_entities)
|
|
merged = relationship_merger.merge_with_deduplication(duplicate_entities)
|
|
merged = relationship_merger.merge_with_consistency_check(duplicate_entities)
|
|
```
|
|
|
|
### 19. **Conflict Detection & Source Tracking** (`semantica.conflicts`)
|
|
|
|
**Main Classes:** `ConflictDetector`, `SourceTracker`, `DisagreementResolver`
|
|
|
|
**Purpose:** Flag when sources disagree and track exact document origins for investigation
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.conflicts import ConflictDetector, SourceTracker, DisagreementResolver
|
|
from semantica.conflicts.conflict_analyzer import ConflictAnalyzer, DisagreementDetector, InconsistencyFinder
|
|
from semantica.conflicts.source_tracker import SourceTracker, DocumentTracker, ProvenanceTracker
|
|
from semantica.conflicts.resolution_strategies import ResolutionStrategy, VotingResolver, AuthorityResolver
|
|
from semantica.conflicts.reporting import ConflictReporter, DisagreementReporter, InvestigationGuide
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Detect conflicts
|
|
conflict_detector = ConflictDetector()
|
|
conflicts = conflict_detector.detect_value_conflicts(entities, "sales_figure")
|
|
conflicts = conflict_detector.detect_property_conflicts(entities, "founded_year")
|
|
conflicts = conflict_detector.detect_relationship_conflicts(relationships)
|
|
|
|
# Track sources
|
|
source_tracker = SourceTracker()
|
|
sources = source_tracker.track_entity_sources(entity, "Apple Inc.")
|
|
sources = source_tracker.track_property_sources(property, "sales_figure", "$10M")
|
|
sources = source_tracker.track_relationship_sources(relationship, "founded_by")
|
|
|
|
# Resolve disagreements
|
|
disagreement_resolver = DisagreementResolver()
|
|
resolved = disagreement_resolver.resolve_by_voting(conflicts)
|
|
resolved = disagreement_resolver.resolve_by_authority(conflicts, authority_sources)
|
|
resolved = disagreement_resolver.flag_for_investigation(conflicts)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Conflict Analyzer (`semantica.conflicts.conflict_analyzer`):**
|
|
```python
|
|
from semantica.conflicts.conflict_analyzer import ConflictAnalyzer, DisagreementDetector, InconsistencyFinder
|
|
|
|
# Conflict analysis
|
|
conflict_analyzer = ConflictAnalyzer()
|
|
conflicts = conflict_analyzer.analyze_value_conflicts(entities, property_name="sales_figure")
|
|
conflicts = conflict_analyzer.analyze_type_conflicts(entities, property_name="founded_year")
|
|
conflicts = conflict_analyzer.analyze_relationship_conflicts(relationships)
|
|
|
|
# Disagreement detection
|
|
disagreement_detector = DisagreementDetector()
|
|
disagreements = disagreement_detector.detect_value_disagreements(entities, "sales_figure")
|
|
disagreements = disagreement_detector.detect_factual_disagreements(entities, "founded_year")
|
|
disagreements = disagreement_detector.detect_categorical_disagreements(entities, "industry")
|
|
|
|
# Inconsistency finding
|
|
inconsistency_finder = InconsistencyFinder()
|
|
inconsistencies = inconsistency_finder.find_logical_inconsistencies(knowledge_graph)
|
|
inconsistencies = inconsistency_finder.find_temporal_inconsistencies(entities, "founded_year")
|
|
inconsistencies = inconsistency_finder.find_hierarchical_inconsistencies(relationships)
|
|
```
|
|
|
|
**Source Tracker (`semantica.conflicts.source_tracker`):**
|
|
```python
|
|
from semantica.conflicts.source_tracker import SourceTracker, DocumentTracker, ProvenanceTracker
|
|
|
|
# Source tracking
|
|
source_tracker = SourceTracker()
|
|
sources = source_tracker.track_entity_sources(entity, "Apple Inc.")
|
|
sources = source_tracker.track_property_sources(property, "sales_figure", "$10M")
|
|
sources = source_tracker.track_relationship_sources(relationship, "founded_by")
|
|
|
|
# Document tracking
|
|
document_tracker = DocumentTracker()
|
|
documents = document_tracker.get_source_documents(entity, "Apple Inc.")
|
|
documents = document_tracker.get_document_sections(entity, "Apple Inc.", "sales_figure")
|
|
documents = document_tracker.get_document_context(entity, "Apple Inc.", context_size=200)
|
|
|
|
# Provenance tracking
|
|
provenance_tracker = ProvenanceTracker()
|
|
provenance = provenance_tracker.get_entity_provenance(entity, "Apple Inc.")
|
|
provenance = provenance_tracker.get_property_provenance(property, "sales_figure")
|
|
provenance = provenance_tracker.get_relationship_provenance(relationship, "founded_by")
|
|
```
|
|
|
|
**Conflict Reporter (`semantica.conflicts.reporting`):**
|
|
```python
|
|
from semantica.conflicts.reporting import ConflictReporter, DisagreementReporter, InvestigationGuide
|
|
|
|
# Conflict reporting
|
|
conflict_reporter = ConflictReporter()
|
|
report = conflict_reporter.generate_conflict_report(conflicts)
|
|
report = conflict_reporter.generate_detailed_report(conflicts, include_sources=True)
|
|
report = conflict_reporter.generate_summary_report(conflicts)
|
|
|
|
# Disagreement reporting
|
|
disagreement_reporter = DisagreementReporter()
|
|
report = disagreement_reporter.report_value_disagreements(disagreements, "sales_figure")
|
|
report = disagreement_reporter.report_factual_disagreements(disagreements, "founded_year")
|
|
report = disagreement_reporter.report_categorical_disagreements(disagreements, "industry")
|
|
|
|
# Investigation guide
|
|
investigation_guide = InvestigationGuide()
|
|
guide = investigation_guide.create_investigation_plan(conflicts)
|
|
guide = investigation_guide.suggest_investigation_steps(conflicts)
|
|
guide = investigation_guide.prioritize_investigations(conflicts, priority_criteria)
|
|
```
|
|
|
|
### 20. **Knowledge Graph Quality Assurance** (`semantica.kg_qa`)
|
|
|
|
**Main Classes:** `KGQualityAssessor`, `ConsistencyChecker`, `CompletenessValidator`
|
|
|
|
**Purpose:** Comprehensive quality assurance for production-ready Knowledge Graphs
|
|
|
|
#### **Imports:**
|
|
```python
|
|
from semantica.kg_qa import KGQualityAssessor, ConsistencyChecker, CompletenessValidator
|
|
from semantica.kg_qa.quality_metrics import QualityMetrics, CompletenessMetrics, ConsistencyMetrics
|
|
from semantica.kg_qa.validation_engine import ValidationEngine, RuleValidator, ConstraintValidator
|
|
from semantica.kg_qa.reporting import QualityReporter, IssueTracker, ImprovementSuggestions
|
|
from semantica.kg_qa.automated_fixes import AutomatedFixer, AutoMerger, AutoResolver
|
|
```
|
|
|
|
#### **Main Functions:**
|
|
```python
|
|
# Assess knowledge graph quality
|
|
kg_qa = KGQualityAssessor()
|
|
quality_score = kg_qa.assess_overall_quality(knowledge_graph)
|
|
quality_report = kg_qa.generate_quality_report(knowledge_graph)
|
|
quality_issues = kg_qa.identify_quality_issues(knowledge_graph)
|
|
|
|
# Check consistency
|
|
consistency_checker = ConsistencyChecker()
|
|
is_consistent = consistency_checker.check_logical_consistency(knowledge_graph)
|
|
is_consistent = consistency_checker.check_temporal_consistency(knowledge_graph)
|
|
is_consistent = consistency_checker.check_hierarchical_consistency(knowledge_graph)
|
|
|
|
# Validate completeness
|
|
completeness_validator = CompletenessValidator()
|
|
is_complete = completeness_validator.validate_entity_completeness(entities, schema)
|
|
is_complete = completeness_validator.validate_relationship_completeness(relationships, schema)
|
|
is_complete = completeness_validator.validate_property_completeness(properties, schema)
|
|
```
|
|
|
|
#### **Submodules with Functions:**
|
|
|
|
**Quality Metrics (`semantica.kg_qa.quality_metrics`):**
|
|
```python
|
|
from semantica.kg_qa.quality_metrics import QualityMetrics, CompletenessMetrics, ConsistencyMetrics
|
|
|
|
# Quality metrics calculation
|
|
quality_metrics = QualityMetrics()
|
|
score = quality_metrics.calculate_overall_score(knowledge_graph)
|
|
score = quality_metrics.calculate_entity_quality(entities)
|
|
score = quality_metrics.calculate_relationship_quality(relationships)
|
|
|
|
# Completeness metrics
|
|
completeness_metrics = CompletenessMetrics()
|
|
score = completeness_metrics.calculate_entity_completeness(entities, schema)
|
|
score = completeness_metrics.calculate_property_completeness(properties, schema)
|
|
score = completeness_metrics.calculate_relationship_completeness(relationships, schema)
|
|
|
|
# Consistency metrics
|
|
consistency_metrics = ConsistencyMetrics()
|
|
score = consistency_metrics.calculate_logical_consistency(knowledge_graph)
|
|
score = consistency_metrics.calculate_temporal_consistency(knowledge_graph)
|
|
score = consistency_metrics.calculate_hierarchical_consistency(knowledge_graph)
|
|
```
|
|
|
|
**Automated Fixes (`semantica.kg_qa.automated_fixes`):**
|
|
```python
|
|
from semantica.kg_qa.automated_fixes import AutomatedFixer, AutoMerger, AutoResolver
|
|
|
|
# Automated fixing
|
|
automated_fixer = AutomatedFixer()
|
|
fixed = automated_fixer.fix_duplicates(knowledge_graph)
|
|
fixed = automated_fixer.fix_inconsistencies(knowledge_graph)
|
|
fixed = automated_fixer.fix_missing_properties(knowledge_graph)
|
|
|
|
# Auto merging
|
|
auto_merger = AutoMerger()
|
|
merged = auto_merger.merge_duplicate_entities(knowledge_graph)
|
|
merged = auto_merger.merge_duplicate_relationships(knowledge_graph)
|
|
merged = auto_merger.merge_conflicting_properties(knowledge_graph)
|
|
|
|
# Auto resolving
|
|
auto_resolver = AutoResolver()
|
|
resolved = auto_resolver.resolve_conflicts(knowledge_graph)
|
|
resolved = auto_resolver.resolve_disagreements(knowledge_graph)
|
|
resolved = auto_resolver.resolve_inconsistencies(knowledge_graph)
|
|
```
|
|
|
|
---
|
|
|
|
## 📚 Complete Module Index
|
|
|
|
### All 20 Main Modules with Submodules
|
|
|
|
| # | Module | Package | Main Classes | Submodules Count |
|
|
|---|--------|---------|--------------|------------------|
|
|
| 1 | **Core Engine** | `semantica.core` | `Semantica`, `Config`, `PluginManager` | 4 |
|
|
| 2 | **Pipeline Builder** | `semantica.pipeline` | `PipelineBuilder`, `ExecutionEngine` | 7 |
|
|
| 3 | **Data Ingestion** | `semantica.ingest` | `FileIngestor`, `WebIngestor`, `FeedIngestor` | 7 |
|
|
| 4 | **Document Parsing** | `semantica.parse` | `PDFParser`, `DOCXParser`, `HTMLParser` | 9 |
|
|
| 5 | **Text Normalization** | `semantica.normalize` | `TextCleaner`, `LanguageDetector` | 6 |
|
|
| 6 | **Text Chunking** | `semantica.split` | `SemanticChunker`, `StructuralChunker` | 5 |
|
|
| 7 | **Semantic Extraction** | `semantica.semantic_extract` | `NERExtractor`, `RelationExtractor` | 6 |
|
|
| 8 | **Ontology Generation** | `semantica.ontology` | `OntologyGenerator`, `ClassInferrer` | 6 |
|
|
| 9 | **Knowledge Graph** | `semantica.kg` | `GraphBuilder`, `EntityResolver` | 7 |
|
|
| 10 | **Vector Store** | `semantica.vector_store` | `PineconeAdapter`, `FAISSAdapter` | 6 |
|
|
| 11 | **Triple Store** | `semantica.triple_store` | `BlazegraphAdapter`, `JenaAdapter` | 5 |
|
|
| 12 | **Embeddings** | `semantica.embeddings` | `SemanticEmbedder`, `TextEmbedder` | 6 |
|
|
| 13 | **RAG System** | `semantica.qa_rag` | `RAGManager`, `SemanticChunker` | 7 |
|
|
| 14 | **Reasoning Engine** | `semantica.reasoning` | `InferenceEngine`, `SPARQLReasoner` | 7 |
|
|
| 15 | **Multi-Agent System** | `semantica.agents` | `AgentManager`, `OrchestrationEngine` | 8 |
|
|
| 16 | **Template System** | `semantica.templates` | `SchemaTemplate`, `EntityTemplate` | 5 |
|
|
| 17 | **Seed Data System** | `semantica.seed` | `SeedDataManager`, `KnowledgeSeeder` | 4 |
|
|
| 18 | **Advanced Deduplication** | `semantica.deduplication` | `DuplicateDetector`, `EntityMerger` | 5 |
|
|
| 19 | **Conflict Detection** | `semantica.conflicts` | `ConflictDetector`, `SourceTracker` | 4 |
|
|
| 20 | **KG Quality Assurance** | `semantica.kg_qa` | `KGQualityAssessor`, `ConsistencyChecker` | 5 |
|
|
|
|
**Total: 20 Main Modules, 120+ Submodules**
|
|
|
|
---
|
|
|
|
## 🔧 Complete Functions Reference
|
|
|
|
### Module-by-Module Function Tables
|
|
|
|
#### 1. **Core Engine Functions** (`semantica.core`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `Semantica.initialize()` | core | Setup all modules and connections | None | Status |
|
|
| `Semantica.build_knowledge_base()` | core | Process data sources into knowledge base | sources: List[str] | KnowledgeBase |
|
|
| `Semantica.get_status()` | core | Get system health and metrics | None | Dict |
|
|
| `Semantica.create_pipeline()` | core | Create processing pipeline | config: Dict | Pipeline |
|
|
| `Semantica.get_config()` | core | Get current configuration | None | Config |
|
|
| `Semantica.list_plugins()` | core | List available plugins | None | List[Plugin] |
|
|
| `Config.validate()` | config_manager | Validate configuration against schema | schema: str | bool |
|
|
| `PluginManager.load_plugin()` | plugin_registry | Dynamically load plugin modules | name: str, version: str | Plugin |
|
|
| `PluginManager.list_plugins()` | plugin_registry | Show available plugins and versions | None | List[Plugin] |
|
|
| `Orchestrator.schedule_pipeline()` | orchestrator | Schedule pipeline execution | pipeline_config: Dict | PipelineID |
|
|
| `Orchestrator.monitor_progress()` | orchestrator | Monitor pipeline progress | pipeline_id: str | Progress |
|
|
| `LifecycleManager.startup()` | lifecycle | Execute startup hooks | None | Status |
|
|
| `LifecycleManager.shutdown()` | lifecycle | Execute shutdown hooks | None | Status |
|
|
|
|
#### 2. **Pipeline Builder Functions** (`semantica.pipeline`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `PipelineBuilder.add_step()` | pipeline | Add processing step to pipeline | name: str, config: Dict | PipelineBuilder |
|
|
| `PipelineBuilder.set_parallelism()` | pipeline | Configure parallel execution | level: int | PipelineBuilder |
|
|
| `PipelineBuilder.build()` | pipeline | Build the pipeline | None | Pipeline |
|
|
| `Pipeline.run()` | execution_engine | Execute complete pipeline | None | Results |
|
|
| `Pipeline.pause()` | execution_engine | Pause pipeline execution | None | Status |
|
|
| `Pipeline.resume()` | execution_engine | Resume paused pipeline | None | Status |
|
|
| `Pipeline.stop()` | execution_engine | Stop pipeline execution | None | Status |
|
|
| `ExecutionEngine.execute_pipeline()` | execution_engine | Execute pipeline with config | config: Dict | Results |
|
|
| `FailureHandler.retry_step()` | failure_handler | Retry failed step | step_id: str, error: Exception | Status |
|
|
| `FailureHandler.handle_error()` | failure_handler | Handle execution errors | error: Exception | RecoveryPlan |
|
|
| `ParallelExecutor.execute_parallel()` | parallelism_manager | Execute tasks in parallel | tasks: List[Task] | Results |
|
|
| `ResourceScheduler.allocate_cpu()` | resource_scheduler | Allocate CPU resources | cores: int | ResourceID |
|
|
| `ResourceScheduler.allocate_gpu()` | resource_scheduler | Allocate GPU resources | device_id: int | ResourceID |
|
|
| `PipelineValidator.validate_pipeline()` | pipeline_validator | Validate pipeline configuration | config: Dict | ValidationResult |
|
|
|
|
#### 3. **Data Ingestion Functions** (`semantica.ingest`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `FileIngestor.scan_directory()` | file | Recursively scan directory for files | path: str, recursive: bool | List[File] |
|
|
| `FileIngestor.detect_format()` | file | Auto-detect file type and encoding | file_path: str | FileFormat |
|
|
| `WebIngestor.crawl_site()` | web | Crawl website with depth and rate limiting | url: str, max_depth: int | WebContent |
|
|
| `WebIngestor.extract_links()` | web | Extract and follow hyperlinks | content: WebContent | List[Link] |
|
|
| `FeedIngestor.parse_rss()` | feed | Parse RSS/Atom feeds with metadata | feed_url: str | FeedData |
|
|
| `StreamIngestor.connect()` | stream | Establish real-time data connection | config: Dict | StreamConnection |
|
|
| `RepoIngestor.clone_repo()` | repo | Clone and track repository changes | repo_url: str | Repository |
|
|
| `EmailIngestor.connect_imap()` | email | Connect to email server | server: str, credentials: Dict | EmailConnection |
|
|
| `DBIngestor.export_table()` | db_export | Export database table to structured format | table: str, query: str | StructuredData |
|
|
| `IngestManager.resume_from_token()` | ingest | Resume interrupted ingestion | token: str | Status |
|
|
| `IngestManager.get_progress()` | ingest | Monitor ingestion progress | None | Progress |
|
|
| `ConnectorRegistry.register()` | ingest | Register custom data connectors | connector: Connector | Status |
|
|
|
|
#### 4. **Document Parsing Functions** (`semantica.parse`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `PDFParser.extract_text()` | pdf | Extract text with positioning and formatting | file_path: str | TextContent |
|
|
| `PDFParser.extract_tables()` | pdf | Extract tables using Camelot/Tabula | file_path: str | List[Table] |
|
|
| `PDFParser.extract_images()` | pdf | Extract embedded images and figures | file_path: str | List[Image] |
|
|
| `DOCXParser.get_document_structure()` | docx | Extract document outline and sections | file_path: str | DocumentStructure |
|
|
| `DOCXParser.extract_track_changes()` | docx | Extract revision history | file_path: str | TrackChanges |
|
|
| `PPTXParser.extract_slides()` | pptx | Extract slide content and speaker notes | file_path: str | List[Slide] |
|
|
| `ExcelParser.read_sheet()` | excel | Read specific worksheet with data types | file_path: str, sheet: str | Worksheet |
|
|
| `ExcelParser.extract_charts()` | excel | Extract chart data and metadata | file_path: str | List[Chart] |
|
|
| `HTMLParser.parse_dom()` | html | Parse HTML into structured DOM tree | url: str | DOMTree |
|
|
| `HTMLParser.extract_metadata()` | html | Extract meta tags and structured data | dom: DOMTree | Metadata |
|
|
| `ImageParser.ocr_text()` | images | Perform OCR using Tesseract/Google Vision | image_path: str | TextContent |
|
|
| `ImageParser.detect_objects()` | images | Detect objects and faces in images | image_path: str | List[Object] |
|
|
| `TableParser.detect_structure()` | tables | Detect table boundaries and headers | content: str | TableStructure |
|
|
| `TableParser.extract_cells()` | tables | Extract individual cell data | table: Table | List[Cell] |
|
|
| `ParserRegistry.get_parser()` | parse | Get appropriate parser for file type | file_type: str | Parser |
|
|
| `ParserRegistry.supported_formats()` | parse | List all supported file formats | None | List[str] |
|
|
|
|
#### 5. **Text Normalization Functions** (`semantica.normalize`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `TextCleaner.remove_html()` | text_cleaner | Strip HTML tags and preserve text content | html: str | str |
|
|
| `TextCleaner.normalize_whitespace()` | text_cleaner | Standardize spacing and line breaks | text: str | str |
|
|
| `TextCleaner.remove_special_chars()` | text_cleaner | Clean special characters and symbols | text: str | str |
|
|
| `LanguageDetector.detect()` | language_detector | Identify text language with confidence score | text: str | Language |
|
|
| `LanguageDetector.supported_languages()` | language_detector | List all supported languages | None | List[str] |
|
|
| `EncodingHandler.normalize()` | encoding_handler | Convert to UTF-8 and validate encoding | text: bytes | str |
|
|
| `EncodingHandler.detect_encoding()` | encoding_handler | Auto-detect file encoding | file_path: str | str |
|
|
| `EntityNormalizer.canonicalize()` | entity_normalizer | Standardize entity names and aliases | entity: str, alias: str | str |
|
|
| `EntityNormalizer.expand_acronyms()` | entity_normalizer | Expand abbreviations and acronyms | text: str | str |
|
|
| `DateNormalizer.parse_date()` | date_normalizer | Parse various date formats to ISO standard | date_str: str | datetime |
|
|
| `DateNormalizer.resolve_relative()` | date_normalizer | Convert relative dates to absolute | date_str: str | datetime |
|
|
| `NumberNormalizer.standardize()` | number_normalizer | Convert numbers to standard format | number: str | str |
|
|
| `NumberNormalizer.convert_units()` | number_normalizer | Convert between measurement units | value: float, from_unit: str, to_unit: str | float |
|
|
| `NormalizationPipeline.run()` | normalize | Execute complete normalization pipeline | text: str | NormalizedText |
|
|
| `NormalizationPipeline.get_stats()` | normalize | Return normalization statistics | None | Dict |
|
|
|
|
#### 6. **Text Chunking Functions** (`semantica.split`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `SlidingWindowChunker.split()` | sliding_window | Create fixed-size chunks with overlap | text: str, size: int, overlap: int | List[Chunk] |
|
|
| `SlidingWindowChunker.set_window_size()` | sliding_window | Configure chunk size and overlap | size: int, overlap: int | None |
|
|
| `SemanticChunker.split_by_meaning()` | semantic_chunker | Split based on semantic boundaries | text: str | List[Chunk] |
|
|
| `SemanticChunker.detect_topics()` | semantic_chunker | Identify topic changes for splitting | text: str | List[Topic] |
|
|
| `StructuralChunker.split_by_sections()` | structural_chunker | Split on document structure | document: Document | List[Chunk] |
|
|
| `StructuralChunker.identify_headers()` | structural_chunker | Detect section headers and levels | document: Document | List[Header] |
|
|
| `TableChunker.preserve_tables()` | table_chunker | Keep tables intact during splitting | document: Document | List[Chunk] |
|
|
| `TableChunker.extract_table_context()` | table_chunker | Extract surrounding context for tables | table: Table | str |
|
|
| `ProvenanceTracker.track_source()` | provenance_tracker | Track original source and position | chunk: Chunk | Provenance |
|
|
| `ProvenanceTracker.get_provenance()` | provenance_tracker | Retrieve chunk source information | chunk_id: str | Provenance |
|
|
| `ChunkValidator.validate_chunk()` | chunk_validator | Validate chunk quality and size | chunk: Chunk | ValidationResult |
|
|
| `ChunkValidator.detect_overlaps()` | chunk_validator | Find overlapping chunks | chunks: List[Chunk] | List[Overlap] |
|
|
| `SplitManager.run_strategy()` | split | Execute chosen splitting strategy | text: str, strategy: str | List[Chunk] |
|
|
| `SplitManager.get_chunk_stats()` | split | Return chunking statistics | None | Dict |
|
|
|
|
#### 7. **Semantic Extraction Functions** (`semantica.semantic_extract`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `NERExtractor.extract_entities()` | ner_extractor | Extract named entities with types and confidence | text: str | List[Entity] |
|
|
| `NERExtractor.classify_entities()` | ner_extractor | Classify entities into predefined categories | entities: List[Entity] | List[ClassifiedEntity] |
|
|
| `RelationExtractor.find_relations()` | relation_extractor | Detect relationships between entities | text: str | List[Relation] |
|
|
| `RelationExtractor.classify_relations()` | relation_extractor | Classify relation types and directions | relations: List[Relation] | List[ClassifiedRelation] |
|
|
| `EventDetector.detect_events()` | event_detector | Identify events and their participants | text: str | List[Event] |
|
|
| `EventDetector.extract_temporal()` | event_detector | Extract temporal information for events | events: List[Event] | List[TemporalInfo] |
|
|
| `CorefResolver.resolve_references()` | coref_resolver | Resolve co-references and pronouns | text: str | List[Resolution] |
|
|
| `CorefResolver.link_entities()` | coref_resolver | Link entities across document sections | entities: List[Entity] | List[Link] |
|
|
| `TripleExtractor.extract_triples()` | triple_extractor | Extract RDF-style triples | text: str | List[Triple] |
|
|
| `TripleExtractor.validate_triples()` | triple_extractor | Validate triple structure and consistency | triples: List[Triple] | List[ValidatedTriple] |
|
|
| `LLMEnhancer.enhance_extraction()` | llm_enhancer | Use LLM for complex extraction tasks | text: str, task: str | EnhancedResults |
|
|
| `LLMEnhancer.detect_patterns()` | llm_enhancer | Identify complex patterns and relationships | text: str | List[Pattern] |
|
|
| `ExtractionValidator.validate_quality()` | extraction_validator | Assess extraction quality | results: ExtractionResults | QualityScore |
|
|
| `ExtractionValidator.filter_by_confidence()` | extraction_validator | Filter results by confidence score | results: List[Result], threshold: float | List[Result] |
|
|
| `ExtractionPipeline.run()` | semantic_extract | Execute complete extraction pipeline | text: str | ExtractionResults |
|
|
|
|
#### 8. **Ontology Generation Functions** (`semantica.ontology`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `ClassInferrer.infer_classes()` | class_inferrer | Automatically discover entity classes | entities: List[Entity] | List[Class] |
|
|
| `ClassInferrer.build_hierarchy()` | class_inferrer | Build class inheritance hierarchy | classes: List[Class] | Hierarchy |
|
|
| `ClassInferrer.analyze_relationships()` | class_inferrer | Analyze class relationships and dependencies | classes: List[Class] | List[Relationship] |
|
|
| `PropertyGenerator.infer_properties()` | property_generator | Infer object and data properties | classes: List[Class] | List[Property] |
|
|
| `PropertyGenerator.detect_data_types()` | property_generator | Detect property data types and constraints | properties: List[Property] | List[DataType] |
|
|
| `PropertyGenerator.analyze_cardinality()` | property_generator | Analyze property cardinality | properties: List[Property] | List[Cardinality] |
|
|
| `OWLGenerator.generate_owl()` | owl_generator | Generate OWL ontology in RDF/XML format | ontology: Ontology | str |
|
|
| `OWLGenerator.serialize_rdf()` | owl_generator | Serialize to various RDF formats | ontology: Ontology, format: str | str |
|
|
| `BaseMapper.map_to_schema_org()` | base_mapper | Map entities to schema.org vocabulary | entities: List[Entity] | List[Mapping] |
|
|
| `BaseMapper.map_to_foaf()` | base_mapper | Map to FOAF ontology | entities: List[Entity] | List[Mapping] |
|
|
| `BaseMapper.map_to_dublin_core()` | base_mapper | Map to Dublin Core metadata standards | entities: List[Entity] | List[Mapping] |
|
|
| `VersionManager.create_version()` | version_manager | Create new ontology version | ontology: Ontology | Version |
|
|
| `VersionManager.track_changes()` | version_manager | Track changes between versions | old_version: Version, new_version: Version | List[Change] |
|
|
| `VersionManager.migrate_ontology()` | version_manager | Support ontology migration and updates | old_ontology: Ontology, new_schema: Schema | Ontology |
|
|
| `OntologyValidator.validate_schema()` | ontology_validator | Validate ontology schema consistency | ontology: Ontology | ValidationResult |
|
|
| `OntologyValidator.check_constraints()` | ontology_validator | Check ontology constraint violations | ontology: Ontology | List[Violation] |
|
|
| `DomainOntologies.get_finance_ontology()` | domain_ontologies | Get pre-built financial ontology | None | Ontology |
|
|
| `DomainOntologies.get_healthcare_ontology()` | domain_ontologies | Get pre-built healthcare ontology | None | Ontology |
|
|
| `OntologyManager.build_ontology()` | ontology | Build complete ontology from extracted data | data: ExtractedData | Ontology |
|
|
| `OntologyManager.export_ontology()` | ontology | Export ontology in various formats | ontology: Ontology, format: str | str |
|
|
|
|
#### 9. **Knowledge Graph Functions** (`semantica.kg`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `GraphBuilder.create_node()` | graph_builder | Create knowledge graph node | id: str, type: str, properties: Dict | Node |
|
|
| `GraphBuilder.create_edge()` | graph_builder | Create relationship edge between nodes | from_node: str, to_node: str, relation: str | Edge |
|
|
| `GraphBuilder.build_subgraph()` | graph_builder | Build subgraph from specific entities | entities: List[Entity] | SubGraph |
|
|
| `GraphBuilder.merge_graphs()` | graph_builder | Merge multiple knowledge graphs | graphs: List[Graph] | Graph |
|
|
| `EntityResolver.resolve_identity()` | entity_resolver | Resolve entity identity across sources | entity1: Entity, entity2: Entity | Resolution |
|
|
| `EntityResolver.merge_entities()` | entity_resolver | Merge duplicate entities | entities: List[Entity] | MergedEntity |
|
|
| `EntityResolver.get_canonical()` | entity_resolver | Get canonical entity representation | entity: Entity | Entity |
|
|
| `Deduplicator.find_duplicates()` | deduplicator | Find duplicate entities | entities: List[Entity] | List[Duplicate] |
|
|
| `Deduplicator.merge_duplicates()` | deduplicator | Merge duplicate entities | duplicates: List[Duplicate] | List[MergedEntity] |
|
|
| `Deduplicator.validate_merge()` | deduplicator | Validate merge operation | merge: MergeOperation | ValidationResult |
|
|
| `SeedManager.load_seed_data()` | seed_manager | Load initial seed data | data_source: str | SeedData |
|
|
| `SeedManager.validate_seed_data()` | seed_manager | Validate seed data quality | seed_data: SeedData | ValidationResult |
|
|
| `SeedManager.update_seed_data()` | seed_manager | Update existing seed data | seed_data: SeedData | Status |
|
|
| `ProvenanceTracker.track_source()` | provenance_tracker | Track information source | information: Information | Provenance |
|
|
| `ProvenanceTracker.get_provenance()` | provenance_tracker | Retrieve provenance information | info_id: str | Provenance |
|
|
| `ProvenanceTracker.calculate_confidence()` | provenance_tracker | Calculate confidence scores | provenance: Provenance | float |
|
|
| `ConflictDetector.detect_conflicts()` | conflict_detector | Detect conflicts between sources | sources: List[Source] | List[Conflict] |
|
|
| `ConflictDetector.classify_severity()` | conflict_detector | Classify conflict severity | conflict: Conflict | Severity |
|
|
| `ConflictDetector.create_resolution_workflow()` | conflict_detector | Create resolution workflow | conflicts: List[Conflict] | Workflow |
|
|
| `GraphValidator.validate_consistency()` | graph_validator | Validate graph consistency | graph: Graph | ValidationResult |
|
|
| `GraphValidator.check_schema_compliance()` | graph_validator | Check schema compliance | graph: Graph, schema: Schema | ComplianceResult |
|
|
| `GraphValidator.calculate_quality_metrics()` | graph_validator | Calculate quality metrics | graph: Graph | QualityMetrics |
|
|
| `GraphAnalyzer.calculate_centrality()` | graph_analyzer | Calculate node centrality | graph: Graph | CentralityScores |
|
|
| `GraphAnalyzer.detect_communities()` | graph_analyzer | Detect community structures | graph: Graph | List[Community] |
|
|
| `GraphAnalyzer.analyze_connectivity()` | graph_analyzer | Analyze graph connectivity | graph: Graph | ConnectivityMetrics |
|
|
| `KnowledgeGraphManager.build_graph()` | kg | Build complete knowledge graph | data: ProcessedData | KnowledgeGraph |
|
|
| `KnowledgeGraphManager.export_graph()` | kg | Export graph in various formats | graph: Graph, format: str | str |
|
|
| `KnowledgeGraphManager.visualize_graph()` | kg | Generate graph visualizations | graph: Graph | Visualization |
|
|
|
|
#### 10. **Vector Store Functions** (`semantica.vector_store`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `PineconeAdapter.connect()` | pinecone_adapter | Connect to Pinecone service | api_key: str | Connection |
|
|
| `PineconeAdapter.create_index()` | pinecone_adapter | Create new vector index | name: str, dimension: int | Index |
|
|
| `PineconeAdapter.upsert_vectors()` | pinecone_adapter | Insert or update vectors | vectors: List[Vector], metadata: Dict | Status |
|
|
| `PineconeAdapter.query_vectors()` | pinecone_adapter | Query similar vectors | query_vector: Vector, top_k: int | List[Result] |
|
|
| `FAISSAdapter.create_index()` | faiss_adapter | Create FAISS index | index_type: str, dimension: int | Index |
|
|
| `FAISSAdapter.add_vectors()` | faiss_adapter | Add vectors to index | vectors: List[Vector] | Status |
|
|
| `FAISSAdapter.search_similar()` | faiss_adapter | Search for similar vectors | query_vector: Vector, k: int | List[Result] |
|
|
| `FAISSAdapter.save_index()` | faiss_adapter | Save index to disk | file_path: str | Status |
|
|
| `MilvusAdapter.create_collection()` | milvus_adapter | Create Milvus collection | name: str, schema: Schema | Collection |
|
|
| `MilvusAdapter.insert_vectors()` | milvus_adapter | Insert vectors into collection | vectors: List[Vector] | Status |
|
|
| `MilvusAdapter.search_vectors()` | milvus_adapter | Search vectors in collection | query_vector: Vector, top_k: int | List[Result] |
|
|
| `WeaviateAdapter.create_schema()` | weaviate_adapter | Create Weaviate schema | schema: Schema | Status |
|
|
| `WeaviateAdapter.add_objects()` | weaviate_adapter | Add objects to Weaviate | objects: List[Object] | Status |
|
|
| `WeaviateAdapter.graphql_query()` | weaviate_adapter | Execute GraphQL queries | query: str | QueryResult |
|
|
| `QdrantAdapter.create_collection()` | qdrant_adapter | Create Qdrant collection | name: str, config: Dict | Collection |
|
|
| `QdrantAdapter.upsert_points()` | qdrant_adapter | Insert or update points | points: List[Point] | Status |
|
|
| `QdrantAdapter.search_points()` | qdrant_adapter | Search points with filters | query: Vector, filters: Dict | List[Result] |
|
|
| `NamespaceManager.create_namespace()` | namespace_manager | Create isolated namespace | name: str | Namespace |
|
|
| `NamespaceManager.set_access_control()` | namespace_manager | Set namespace permissions | namespace: str, permissions: Dict | Status |
|
|
| `NamespaceManager.list_namespaces()` | namespace_manager | List available namespaces | None | List[Namespace] |
|
|
| `MetadataStore.index_metadata()` | metadata_store | Index metadata for search | metadata: Dict | Status |
|
|
| `MetadataStore.filter_by_metadata()` | metadata_store | Filter results by metadata | filters: Dict | List[Result] |
|
|
| `MetadataStore.search_metadata()` | metadata_store | Search metadata content | query: str | List[Result] |
|
|
| `HybridSearch.combine_results()` | hybrid_search | Combine vector and metadata results | vector_results: List, metadata_results: List | List[Result] |
|
|
| `HybridSearch.rank_results()` | hybrid_search | Rank results using multiple criteria | results: List[Result] | List[RankedResult] |
|
|
| `HybridSearch.fuse_results()` | hybrid_search | Fuse results from different sources | results: List[List[Result]] | List[FusedResult] |
|
|
| `IndexOptimizer.optimize_index()` | index_optimizer | Optimize index performance | index: Index | OptimizedIndex |
|
|
| `IndexOptimizer.rebuild_index()` | index_optimizer | Rebuild index for better performance | index: Index | Index |
|
|
| `IndexOptimizer.get_performance_metrics()` | index_optimizer | Get index performance metrics | index: Index | Metrics |
|
|
| `VectorStoreManager.get_store_info()` | vector_store | Get store information | None | StoreInfo |
|
|
| `VectorStoreManager.backup_store()` | vector_store | Create store backup | backup_path: str | Status |
|
|
| `VectorStoreManager.restore_store()` | vector_store | Restore from backup | backup_path: str | Status |
|
|
|
|
#### 11. **Triple Store Functions** (`semantica.triple_store`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `BlazegraphAdapter.connect()` | blazegraph_adapter | Connect to Blazegraph instance | endpoint: str | Connection |
|
|
| `BlazegraphAdapter.execute_sparql()` | blazegraph_adapter | Execute SPARQL queries | query: str | QueryResult |
|
|
| `BlazegraphAdapter.bulk_load()` | blazegraph_adapter | Load triples in bulk | triples: List[Triple] | Status |
|
|
| `JenaAdapter.create_model()` | jena_adapter | Create and manage RDF models | None | Model |
|
|
| `JenaAdapter.add_triples()` | jena_adapter | Add triples to model | model: Model, triples: List[Triple] | Status |
|
|
| `JenaAdapter.run_inference()` | jena_adapter | Execute inference rules | model: Model | InferredModel |
|
|
| `RDF4JAdapter.create_repository()` | rdf4j_adapter | Create and configure repositories | config: Dict | Repository |
|
|
| `RDF4JAdapter.begin_transaction()` | rdf4j_adapter | Start transaction for batch operations | None | Transaction |
|
|
| `GraphDBAdapter.enable_reasoning()` | graphdb_adapter | Enable reasoning capabilities | config: Dict | Status |
|
|
| `GraphDBAdapter.visualize_graph()` | graphdb_adapter | Generate graph visualizations | query: str | Visualization |
|
|
| `VirtuosoAdapter.connect_cluster()` | virtuoso_adapter | Connect to Virtuoso cluster | cluster_config: Dict | Connection |
|
|
| `VirtuosoAdapter.optimize_queries()` | virtuoso_adapter | Optimize query performance | queries: List[str] | OptimizedQueries |
|
|
| `TripleManager.add_triple()` | triple_manager | Add single triple to store | triple: Triple | Status |
|
|
| `TripleManager.add_triples()` | triple_manager | Add multiple triples | triples: List[Triple] | Status |
|
|
| `TripleManager.delete_triple()` | triple_manager | Delete specific triple | triple: Triple | Status |
|
|
| `TripleManager.update_triple()` | triple_manager | Update existing triple | old_triple: Triple, new_triple: Triple | Status |
|
|
| `QueryEngine.execute_sparql()` | query_engine | Execute SPARQL queries | query: str | QueryResult |
|
|
| `QueryEngine.optimize_query()` | query_engine | Optimize query for performance | query: str | OptimizedQuery |
|
|
| `QueryEngine.format_results()` | query_engine | Format query results | results: QueryResult, format: str | FormattedResults |
|
|
| `BulkLoader.load_file()` | bulk_loader | Load triples from file | file_path: str | Status |
|
|
| `BulkLoader.create_indexes()` | bulk_loader | Create database indexes | None | Status |
|
|
| `BulkLoader.monitor_progress()` | bulk_loader | Monitor loading progress | None | Progress |
|
|
| `TripleStoreManager.get_store_info()` | triple_store | Get store statistics and status | None | StoreInfo |
|
|
| `TripleStoreManager.backup_store()` | triple_store | Create backup of store | backup_path: str | Status |
|
|
| `TripleStoreManager.restore_store()` | triple_store | Restore from backup | backup_path: str | Status |
|
|
|
|
#### 12. **Embeddings Functions** (`semantica.embeddings`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `TextEmbedder.embed_text()` | text_embedder | Generate text embeddings | text: str | Vector |
|
|
| `TextEmbedder.embed_sentence()` | text_embedder | Generate sentence-level embeddings | sentence: str | Vector |
|
|
| `TextEmbedder.embed_document()` | text_embedder | Generate document-level embeddings | document: str | Vector |
|
|
| `ImageEmbedder.embed_image()` | image_embedder | Generate image embeddings | image_path: str | Vector |
|
|
| `ImageEmbedder.extract_features()` | image_embedder | Extract visual features | image_path: str | Features |
|
|
| `ImageEmbedder.embed_batch()` | image_embedder | Process multiple images | image_paths: List[str] | List[Vector] |
|
|
| `AudioEmbedder.embed_audio()` | audio_embedder | Generate audio embeddings | audio_path: str | Vector |
|
|
| `AudioEmbedder.extract_audio_features()` | audio_embedder | Extract audio features | audio_path: str | Features |
|
|
| `MultimodalEmbedder.fuse_embeddings()` | multimodal_embedder | Fuse multiple modality embeddings | embeddings: List[Vector] | FusedVector |
|
|
| `MultimodalEmbedder.align_modalities()` | multimodal_embedder | Align different modality representations | modalities: List[Vector] | AlignedVectors |
|
|
| `ContextManager.set_window_size()` | context_manager | Set context window size | size: int | None |
|
|
| `ContextManager.apply_sliding_window()` | context_manager | Apply sliding window approach | text: str | List[Window] |
|
|
| `ContextManager.manage_attention()` | context_manager | Manage attention mechanisms | config: Dict | AttentionWeights |
|
|
| `PoolingStrategies.mean_pooling()` | pooling_strategies | Apply mean pooling strategy | vectors: List[Vector] | Vector |
|
|
| `PoolingStrategies.max_pooling()` | pooling_strategies | Apply max pooling strategy | vectors: List[Vector] | Vector |
|
|
| `PoolingStrategies.attention_pooling()` | pooling_strategies | Apply attention-based pooling | vectors: List[Vector], weights: List[float] | Vector |
|
|
| `ProviderAdapter.connect_openai()` | provider_adapter | Connect to OpenAI embedding API | api_key: str | Connection |
|
|
| `ProviderAdapter.connect_bge()` | provider_adapter | Connect to BGE embedding service | endpoint: str | Connection |
|
|
| `ProviderAdapter.connect_llama()` | provider_adapter | Connect to Llama embedding model | model_path: str | Connection |
|
|
| `ProviderAdapter.load_custom_model()` | provider_adapter | Load custom embedding model | model_config: Dict | Model |
|
|
| `EmbeddingOptimizer.optimize_dimensions()` | embedding_optimizer | Optimize embedding dimensions | vectors: List[Vector], target_dim: int | OptimizedVectors |
|
|
| `EmbeddingOptimizer.apply_clustering()` | embedding_optimizer | Apply clustering to embeddings | vectors: List[Vector] | ClusterResults |
|
|
| `EmbeddingOptimizer.calculate_similarity()` | embedding_optimizer | Calculate embedding similarities | vector1: Vector, vector2: Vector | float |
|
|
| `SemanticEmbedder.generate_embeddings()` | embeddings | Generate embeddings for input | input_data: Any | List[Vector] |
|
|
| `SemanticEmbedder.batch_process()` | embeddings | Process multiple inputs in batch | inputs: List[Any] | List[Vector] |
|
|
| `SemanticEmbedder.get_embedding_stats()` | embeddings | Get embedding statistics | None | Stats |
|
|
|
|
#### 13. **RAG System Functions** (`semantica.qa_rag`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `RAGManager.process_question()` | qa_rag | Process user question | question: str | Answer |
|
|
| `RAGManager.get_answer()` | qa_rag | Get RAG-generated answer | question: str | Answer |
|
|
| `RAGManager.evaluate_performance()` | qa_rag | Evaluate RAG performance | test_data: List[Question] | PerformanceMetrics |
|
|
| `SemanticChunker.chunk_text()` | semantic_chunker | Create semantic chunks with context | text: str | List[Chunk] |
|
|
| `SemanticChunker.optimize_chunks()` | semantic_chunker | Optimize chunk size and overlap | chunks: List[Chunk] | List[OptimizedChunk] |
|
|
| `SemanticChunker.merge_chunks()` | semantic_chunker | Merge related chunks when needed | chunks: List[Chunk] | List[MergedChunk] |
|
|
| `PromptTemplates.get_template()` | prompt_templates | Get RAG prompt template | template_name: str | Template |
|
|
| `PromptTemplates.format_question()` | prompt_templates | Format question for retrieval | question: str | FormattedQuestion |
|
|
| `PromptTemplates.inject_context()` | prompt_templates | Inject retrieved context into prompt | question: str, context: str | Prompt |
|
|
| `RetrievalPolicies.set_strategy()` | retrieval_policies | Set retrieval strategy | strategy: str | None |
|
|
| `RetrievalPolicies.rank_results()` | retrieval_policies | Rank retrieval results | results: List[Result] | List[RankedResult] |
|
|
| `RetrievalPolicies.filter_results()` | retrieval_policies | Filter results by criteria | results: List[Result], criteria: Dict | List[FilteredResult] |
|
|
| `AnswerBuilder.construct_answer()` | answer_builder | Construct answer from retrieved context | context: List[Chunk], question: str | Answer |
|
|
| `AnswerBuilder.integrate_context()` | answer_builder | Integrate multiple context sources | contexts: List[Context] | IntegratedContext |
|
|
| `AnswerBuilder.attribute_sources()` | answer_builder | Attribute answer to source documents | answer: Answer | AttributedAnswer |
|
|
| `ProvenanceTracker.track_sources()` | provenance_tracker | Track information sources | answer: Answer | List[Source] |
|
|
| `ProvenanceTracker.calculate_confidence()` | provenance_tracker | Calculate answer confidence | answer: Answer | float |
|
|
| `ProvenanceTracker.link_evidence()` | provenance_tracker | Link answer to supporting evidence | answer: Answer | List[Evidence] |
|
|
| `AnswerValidator.validate_answer()` | answer_validator | Validate answer accuracy | answer: Answer | ValidationResult |
|
|
| `AnswerValidator.fact_check()` | answer_validator | Perform fact checking | answer: Answer | FactCheckResult |
|
|
| `AnswerValidator.verify_consistency()` | answer_validator | Verify answer consistency | answer: Answer | ConsistencyResult |
|
|
| `RAGOptimizer.optimize_retrieval()` | rag_optimizer | Optimize retrieval performance | config: Dict | OptimizedConfig |
|
|
| `RAGOptimizer.enhance_queries()` | rag_optimizer | Enhance user queries | query: str | EnhancedQuery |
|
|
| `RAGOptimizer.improve_ranking()` | rag_optimizer | Improve result ranking | results: List[Result] | List[ImprovedResult] |
|
|
| `ConversationManager.start_conversation()` | conversation_manager | Start new conversation | None | Conversation |
|
|
| `ConversationManager.add_context()` | conversation_manager | Add context to conversation | conversation: Conversation, context: str | None |
|
|
| `ConversationManager.get_history()` | conversation_manager | Get conversation history | conversation: Conversation | List[Message] |
|
|
|
|
#### 14. **Reasoning Engine Functions** (`semantica.reasoning`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `InferenceEngine.add_rule()` | inference_engine | Add inference rule to engine | rule: Rule | Status |
|
|
| `InferenceEngine.execute_rules()` | inference_engine | Execute inference rules | None | List[Inference] |
|
|
| `InferenceEngine.forward_chain()` | inference_engine | Perform forward chaining | None | List[Inference] |
|
|
| `InferenceEngine.backward_chain()` | inference_engine | Perform backward chaining | goal: Goal | List[Inference] |
|
|
| `InferenceEngine.resolve_conflicts()` | inference_engine | Resolve rule conflicts | conflicts: List[Conflict] | Resolution |
|
|
| `SPARQLReasoner.expand_query()` | sparql_reasoner | Expand SPARQL query with reasoning | query: str | ExpandedQuery |
|
|
| `SPARQLReasoner.infer_results()` | sparql_reasoner | Infer additional results | query_results: QueryResult | InferredResults |
|
|
| `SPARQLReasoner.apply_reasoning()` | sparql_reasoner | Apply reasoning to query results | query: str, results: QueryResult | ReasonedResults |
|
|
| `ReteEngine.compile_rules()` | rete_engine | Compile rules into Rete network | rules: List[Rule] | ReteNetwork |
|
|
| `ReteEngine.match_patterns()` | rete_engine | Match patterns using Rete algorithm | facts: List[Fact] | List[Match] |
|
|
| `ReteEngine.execute_matches()` | rete_engine | Execute matched rules | matches: List[Match] | List[Inference] |
|
|
| `AbductiveReasoner.generate_hypotheses()` | abductive_reasoner | Generate explanatory hypotheses | observations: List[Observation] | List[Hypothesis] |
|
|
| `AbductiveReasoner.find_explanations()` | abductive_reasoner | Find explanations for observations | observations: List[Observation] | List[Explanation] |
|
|
| `AbductiveReasoner.rank_hypotheses()` | abductive_reasoner | Rank hypotheses by plausibility | hypotheses: List[Hypothesis] | List[RankedHypothesis] |
|
|
| `DeductiveReasoner.apply_logic()` | deductive_reasoner | Apply logical inference rules | premises: List[Premise] | List[Conclusion] |
|
|
| `DeductiveReasoner.prove_theorem()` | deductive_reasoner | Prove logical theorems | theorem: Theorem | Proof |
|
|
| `DeductiveReasoner.validate_argument()` | deductive_reasoner | Validate logical arguments | argument: Argument | ValidationResult |
|
|
| `RuleManager.define_rule()` | rule_manager | Define new inference rule | rule_definition: str | Rule |
|
|
| `RuleManager.validate_rule()` | rule_manager | Validate rule syntax and logic | rule: Rule | ValidationResult |
|
|
| `RuleManager.track_execution()` | rule_manager | Track rule execution history | rule: Rule | ExecutionHistory |
|
|
| `ReasoningValidator.validate_reasoning()` | reasoning_validator | Validate reasoning process | reasoning: Reasoning | ValidationResult |
|
|
| `ReasoningValidator.check_consistency()` | reasoning_validator | Check reasoning consistency | reasoning: Reasoning | ConsistencyResult |
|
|
| `ReasoningValidator.detect_errors()` | reasoning_validator | Detect reasoning errors | reasoning: Reasoning | List[Error] |
|
|
| `ExplanationGenerator.generate_explanation()` | explanation_generator | Generate reasoning explanation | reasoning: Reasoning | Explanation |
|
|
| `ExplanationGenerator.show_reasoning_path()` | explanation_generator | Show reasoning path | reasoning: Reasoning | ReasoningPath |
|
|
| `ExplanationGenerator.justify_conclusion()` | explanation_generator | Justify reasoning conclusion | conclusion: Conclusion | Justification |
|
|
| `ReasoningManager.run_reasoning()` | reasoning | Run complete reasoning process | input_data: Any | ReasoningResult |
|
|
| `ReasoningManager.get_reasoning_results()` | reasoning | Get reasoning results | reasoning_id: str | ReasoningResult |
|
|
| `ReasoningManager.export_reasoning()` | reasoning | Export reasoning process | reasoning: Reasoning, format: str | str |
|
|
|
|
#### 15. **Multi-Agent System Functions** (`semantica.agents`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `AgentManager.register_agent()` | agent_manager | Register new agent | agent_config: Dict | Agent |
|
|
| `AgentManager.start_agent()` | agent_manager | Start agent execution | agent: Agent | Status |
|
|
| `AgentManager.stop_agent()` | agent_manager | Stop agent execution | agent_id: str | Status |
|
|
| `AgentManager.monitor_agent()` | agent_manager | Monitor agent status | agent_id: str | AgentStatus |
|
|
| `OrchestrationEngine.coordinate_agents()` | orchestration_engine | Coordinate multiple agents | agents: List[Agent] | Coordination |
|
|
| `OrchestrationEngine.distribute_tasks()` | orchestration_engine | Distribute tasks among agents | tasks: List[Task], agents: List[Agent] | TaskDistribution |
|
|
| `OrchestrationEngine.manage_workflows()` | orchestration_engine | Manage agent workflows | workflow: Workflow | WorkflowStatus |
|
|
| `ToolRegistry.register_tool()` | tool_registry | Register tool for agent use | tool: Tool | Status |
|
|
| `ToolRegistry.discover_tools()` | tool_registry | Discover available tools | None | List[Tool] |
|
|
| `ToolRegistry.get_tool()` | tool_registry | Get specific tool | tool_name: str | Tool |
|
|
| `CostTracker.monitor_costs()` | cost_tracker | Monitor agent execution costs | agent_id: str | CostMetrics |
|
|
| `CostTracker.set_budget()` | cost_tracker | Set cost budget limits | budget: float | Status |
|
|
| `CostTracker.optimize_resources()` | cost_tracker | Optimize resource usage | usage_data: Dict | OptimizationPlan |
|
|
| `SandboxManager.create_sandbox()` | sandbox_manager | Create agent sandbox | config: Dict | Sandbox |
|
|
| `SandboxManager.isolate_agent()` | sandbox_manager | Isolate agent execution | agent: Agent | IsolationStatus |
|
|
| `SandboxManager.set_resource_limits()` | sandbox_manager | Set resource limits | limits: Dict | Status |
|
|
| `WorkflowEngine.define_workflow()` | workflow_engine | Define agent workflow | workflow_definition: Dict | Workflow |
|
|
| `WorkflowEngine.execute_workflow()` | workflow_engine | Execute defined workflow | workflow: Workflow | WorkflowResult |
|
|
| `WorkflowEngine.monitor_progress()` | workflow_engine | Monitor workflow progress | workflow_id: str | Progress |
|
|
| `AgentCommunication.send_message()` | agent_communication | Send message between agents | from_agent: str, to_agent: str, message: Message | Status |
|
|
| `AgentCommunication.route_message()` | agent_communication | Route message to appropriate agent | message: Message | RoutingResult |
|
|
| `AgentCommunication.manage_protocols()` | agent_communication | Manage communication protocols | protocols: List[Protocol] | Status |
|
|
| `PolicyEnforcer.enforce_policy()` | policy_enforcer | Enforce access policies | agent: Agent, resource: Resource | EnforcementResult |
|
|
| `PolicyEnforcer.check_compliance()` | policy_enforcer | Check policy compliance | agent: Agent | ComplianceResult |
|
|
| `PolicyEnforcer.set_permissions()` | policy_enforcer | Set agent permissions | agent: Agent, permissions: List[Permission] | Status |
|
|
| `AgentAnalytics.analyze_performance()` | agent_analytics | Analyze agent performance | agent_id: str | PerformanceMetrics |
|
|
| `AgentAnalytics.analyze_behavior()` | agent_analytics | Analyze agent behavior patterns | agent_id: str | BehaviorAnalysis |
|
|
| `AgentAnalytics.optimize_agents()` | agent_analytics | Optimize agent performance | agents: List[Agent] | OptimizationPlan |
|
|
| `MultiAgentManager.create_team()` | multi_agent_manager | Create agent team | team_config: Dict | Team |
|
|
| `MultiAgentManager.orchestrate_workflow()` | multi_agent_manager | Orchestrate team workflow | team: Team, workflow: Workflow | WorkflowResult |
|
|
| `MultiAgentManager.get_team_status()` | multi_agent_manager | Get team execution status | team_id: str | TeamStatus |
|
|
|
|
#### 16. **Domain Specialization Functions** (`semantica.domains`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `FinanceSpecialist.analyze_financial_data()` | finance | Analyze financial documents and data | data: FinancialData | Analysis |
|
|
| `FinanceSpecialist.extract_financial_entities()` | finance | Extract financial entities and metrics | text: str | List[FinancialEntity] |
|
|
| `FinanceSpecialist.calculate_ratios()` | finance | Calculate financial ratios | data: FinancialData | List[Ratio] |
|
|
| `HealthcareSpecialist.process_medical_records()` | healthcare | Process medical records and documents | records: MedicalRecords | ProcessedRecords |
|
|
| `HealthcareSpecialist.extract_medical_entities()` | healthcare | Extract medical entities and concepts | text: str | List[MedicalEntity] |
|
|
| `HealthcareSpecialist.analyze_drug_interactions()` | healthcare | Analyze drug interaction patterns | drugs: List[Drug] | List[Interaction] |
|
|
| `LegalSpecialist.analyze_legal_documents()` | legal | Analyze legal documents and contracts | documents: LegalDocuments | Analysis |
|
|
| `LegalSpecialist.extract_legal_entities()` | legal | Extract legal entities and clauses | text: str | List[LegalEntity] |
|
|
| `LegalSpecialist.identify_risks()` | legal | Identify legal risks and compliance issues | document: LegalDocument | List[Risk] |
|
|
| `ScientificSpecialist.process_research_papers()` | scientific | Process scientific research papers | papers: ResearchPapers | ProcessedPapers |
|
|
| `ScientificSpecialist.extract_scientific_entities()` | scientific | Extract scientific entities and concepts | text: str | List[ScientificEntity] |
|
|
| `ScientificSpecialist.analyze_citations()` | scientific | Analyze citation networks and patterns | papers: List[Paper] | CitationAnalysis |
|
|
| `DomainManager.register_domain()` | domain_manager | Register new domain specialization | domain_config: Dict | Domain |
|
|
| `DomainManager.get_domain_processor()` | domain_manager | Get domain-specific processor | domain: str | Processor |
|
|
| `DomainManager.list_domains()` | domain_manager | List available domains | None | List[Domain] |
|
|
| `DomainValidator.validate_domain_data()` | domain_validator | Validate domain-specific data | data: Any, domain: str | ValidationResult |
|
|
| `DomainValidator.check_compliance()` | domain_validator | Check domain compliance | data: Any, domain: str | ComplianceResult |
|
|
| `DomainOptimizer.optimize_for_domain()` | domain_optimizer | Optimize processing for specific domain | config: Dict, domain: str | OptimizedConfig |
|
|
| `DomainOptimizer.adapt_models()` | domain_optimizer | Adapt models for domain requirements | models: List[Model], domain: str | AdaptedModels |
|
|
|
|
#### 17. **User Interface Functions** (`semantica.ui`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `WebInterface.start_server()` | web_interface | Start web interface server | config: Dict | Server |
|
|
| `WebInterface.create_dashboard()` | web_interface | Create interactive dashboard | dashboard_config: Dict | Dashboard |
|
|
| `WebInterface.add_widget()` | web_interface | Add widget to dashboard | widget: Widget | Status |
|
|
| `CLIInterface.create_command()` | cli_interface | Create CLI command | command_config: Dict | Command |
|
|
| `CLIInterface.add_subcommand()` | cli_interface | Add subcommand to CLI | subcommand: SubCommand | Status |
|
|
| `CLIInterface.setup_help()` | cli_interface | Setup command help and documentation | command: Command | Status |
|
|
| `APIInterface.create_endpoint()` | api_interface | Create REST API endpoint | endpoint_config: Dict | Endpoint |
|
|
| `APIInterface.add_middleware()` | api_interface | Add middleware to API | middleware: Middleware | Status |
|
|
| `APIInterface.generate_docs()` | api_interface | Generate API documentation | None | Documentation |
|
|
| `VisualizationEngine.create_chart()` | visualization_engine | Create data visualization chart | chart_config: Dict | Chart |
|
|
| `VisualizationEngine.create_graph()` | visualization_engine | Create knowledge graph visualization | graph: Graph | GraphViz |
|
|
| `VisualizationEngine.export_visualization()` | visualization_engine | Export visualization to file | visualization: Visualization, format: str | Status |
|
|
| `UIThemeManager.set_theme()` | ui_theme_manager | Set UI theme and styling | theme: Theme | Status |
|
|
| `UIThemeManager.customize_colors()` | ui_theme_manager | Customize color scheme | colors: ColorScheme | Status |
|
|
| `UIThemeManager.apply_responsive_design()` | ui_theme_manager | Apply responsive design | breakpoints: List[Breakpoint] | Status |
|
|
| `UserManager.create_user()` | user_manager | Create new user account | user_data: Dict | User |
|
|
| `UserManager.authenticate_user()` | user_manager | Authenticate user login | credentials: Credentials | AuthResult |
|
|
| `UserManager.set_permissions()` | user_manager | Set user permissions | user: User, permissions: List[Permission] | Status |
|
|
| `SessionManager.create_session()` | session_manager | Create user session | user: User | Session |
|
|
| `SessionManager.validate_session()` | session_manager | Validate session token | token: str | ValidationResult |
|
|
| `SessionManager.refresh_session()` | session_manager | Refresh session token | session: Session | NewSession |
|
|
| `UIComponentManager.register_component()` | ui_component_manager | Register UI component | component: Component | Status |
|
|
| `UIComponentManager.get_component()` | ui_component_manager | Get component by name | name: str | Component |
|
|
| `UIComponentManager.render_component()` | ui_component_manager | Render component with data | component: Component, data: Any | RenderedComponent |
|
|
|
|
#### 18. **Operations Functions** (`semantica.ops`)
|
|
|
|
| Function | Module | Description | Parameters | Returns |
|
|
|----------|--------|-------------|------------|---------|
|
|
| `DeploymentManager.deploy_service()` | deployment_manager | Deploy service to production | service_config: Dict | Deployment |
|
|
| `DeploymentManager.rollback_deployment()` | deployment_manager | Rollback to previous version | deployment_id: str | Status |
|
|
| `DeploymentManager.scale_service()` | deployment_manager | Scale service instances | service: Service, instances: int | Status |
|
|
| `MonitoringManager.setup_monitoring()` | monitoring_manager | Setup system monitoring | config: Dict | Monitoring |
|
|
| `MonitoringManager.create_alert()` | monitoring_manager | Create monitoring alert | alert_config: Dict | Alert |
|
|
| `MonitoringManager.get_metrics()` | monitoring_manager | Get system metrics | time_range: TimeRange | Metrics |
|
|
| `LoggingManager.configure_logging()` | logging_manager | Configure logging system | config: Dict | Status |
|
|
| `LoggingManager.create_log_handler()` | logging_manager | Create custom log handler | handler_config: Dict | LogHandler |
|
|
| `LoggingManager.analyze_logs()` | logging_manager | Analyze log patterns | logs: List[Log] | LogAnalysis |
|
|
| `BackupManager.create_backup()` | backup_manager | Create system backup | backup_config: Dict | Backup |
|
|
| `BackupManager.restore_backup()` | backup_manager | Restore from backup | backup_id: str | Status |
|
|
| `BackupManager.schedule_backup()` | backup_manager | Schedule automatic backups | schedule: Schedule | Status |
|
|
| `SecurityManager.audit_security()` | security_manager | Perform security audit | None | AuditResult |
|
|
| `SecurityManager.scan_vulnerabilities()` | security_manager | Scan for security vulnerabilities | None | VulnerabilityReport |
|
|
| `SecurityManager.update_policies()` | security_manager | Update security policies | policies: List[Policy] | Status |
|
|
| `PerformanceManager.optimize_performance()` | performance_manager | Optimize system performance | config: Dict | OptimizationResult |
|
|
| `PerformanceManager.benchmark_system()` | performance_manager | Benchmark system performance | None | BenchmarkResult |
|
|
| `PerformanceManager.profile_application()` | performance_manager | Profile application performance | app: Application | ProfileResult |
|
|
| `ResourceManager.allocate_resources()` | resource_manager | Allocate system resources | resource_config: Dict | ResourceAllocation |
|
|
| `ResourceManager.monitor_usage()` | resource_manager | Monitor resource usage | None | UsageMetrics |
|
|
| `ResourceManager.optimize_allocation()` | resource_manager | Optimize resource allocation | usage_data: UsageData | OptimizationPlan |
|
|
| `OpsManager.deploy_infrastructure()` | ops_manager | Deploy infrastructure components | infra_config: Dict | Infrastructure |
|
|
| `OpsManager.manage_services()` | ops_manager | Manage service lifecycle | services: List[Service] | ServiceStatus |
|
|
| `OpsManager.get_operational_status()` | ops_manager | Get operational status | None | OperationalStatus |
|
|
|
|
| `CLIManager.handle_errors()` | cli_manager | Handle CLI errors | error: Exception | ErrorResponse |
|
|
|
|
---
|
|
|
|
## 📊 Function Statistics
|
|
|
|
| Module | Total Functions | Core Functions | Utility Functions | Management Functions |
|
|
|--------|----------------|----------------|-------------------|---------------------|
|
|
| **Core Engine** | 13 | 6 | 4 | 3 |
|
|
| **Pipeline Builder** | 13 | 7 | 3 | 3 |
|
|
| **Data Ingestion** | 12 | 8 | 2 | 2 |
|
|
| **Document Parsing** | 16 | 12 | 2 | 2 |
|
|
| **Text Normalization** | 14 | 10 | 2 | 2 |
|
|
| **Text Chunking** | 14 | 8 | 4 | 2 |
|
|
| **Semantic Extraction** | 15 | 10 | 3 | 2 |
|
|
| **Ontology Generation** | 20 | 12 | 4 | 4 |
|
|
| **Knowledge Graph** | 25 | 15 | 6 | 4 |
|
|
| **Vector Store** | 25 | 15 | 6 | 4 |
|
|
| **Triple Store** | 25 | 15 | 6 | 4 |
|
|
| **Embeddings** | 20 | 12 | 4 | 4 |
|
|
| **RAG System** | 20 | 12 | 4 | 4 |
|
|
| **Reasoning Engine** | 25 | 15 | 6 | 4 |
|
|
| **Multi-Agent System** | 25 | 15 | 6 | 4 |
|
|
|
|
**Total: 20 Modules, 400+ Functions**
|
|
|
|
---
|
|
|
|
## 📥 Import Reference
|
|
|
|
### Complete Import Guide
|
|
|
|
```python
|
|
# =============================================================================
|
|
# CORE MODULES
|
|
# =============================================================================
|
|
|
|
# Main Semantica class
|
|
from semantica import Semantica
|
|
from semantica.core import Config, PluginManager, Orchestrator, LifecycleManager
|
|
|
|
# Pipeline management
|
|
from semantica.pipeline import (
|
|
PipelineBuilder, ExecutionEngine, FailureHandler,
|
|
ParallelismManager, ResourceScheduler, PipelineValidator,
|
|
MonitoringHooks, PipelineTemplates, PipelineManager
|
|
)
|
|
|
|
# =============================================================================
|
|
# DATA PROCESSING MODULES
|
|
# =============================================================================
|
|
|
|
# Data ingestion
|
|
from semantica.ingest import (
|
|
FileIngestor, WebIngestor, FeedIngestor, StreamIngestor,
|
|
RepoIngestor, EmailIngestor, DBIngestor, IngestManager,
|
|
ConnectorRegistry
|
|
)
|
|
|
|
# Document parsing
|
|
from semantica.parse import (
|
|
PDFParser, DOCXParser, PPTXParser, ExcelParser, HTMLParser,
|
|
JSONLParser, CSVParser, LaTeXParser, ImageParser, TableParser,
|
|
ParserRegistry
|
|
)
|
|
|
|
# Text normalization
|
|
from semantica.normalize import (
|
|
TextCleaner, LanguageDetector, EncodingHandler, EntityNormalizer,
|
|
DateNormalizer, NumberNormalizer, NormalizationPipeline
|
|
)
|
|
|
|
# Text chunking
|
|
from semantica.split import (
|
|
SlidingWindowChunker, SemanticChunker, StructuralChunker,
|
|
TableChunker, ProvenanceTracker, ChunkValidator, SplitManager
|
|
)
|
|
|
|
# =============================================================================
|
|
# SEMANTIC INTELLIGENCE MODULES
|
|
# =============================================================================
|
|
|
|
# Semantic extraction
|
|
from semantica.semantic_extract import (
|
|
NERExtractor, RelationExtractor, EventDetector, CorefResolver,
|
|
TripleExtractor, LLMEnhancer, ExtractionValidator, ExtractionPipeline
|
|
)
|
|
|
|
# Ontology generation
|
|
from semantica.ontology import (
|
|
OntologyGenerator, ClassInferrer, PropertyGenerator, OWLGenerator,
|
|
BaseMapper, VersionManager, OntologyValidator, DomainOntologies,
|
|
OntologyManager
|
|
)
|
|
|
|
# Knowledge graph
|
|
from semantica.kg import (
|
|
GraphBuilder, EntityResolver, Deduplicator, SeedManager,
|
|
ProvenanceTracker, ConflictDetector, GraphValidator, GraphAnalyzer,
|
|
KnowledgeGraphManager
|
|
)
|
|
|
|
# =============================================================================
|
|
# STORAGE & RETRIEVAL MODULES
|
|
# =============================================================================
|
|
|
|
# Vector stores
|
|
from semantica.vector_store import (
|
|
PineconeAdapter, FAISSAdapter, MilvusAdapter, WeaviateAdapter,
|
|
QdrantAdapter, NamespaceManager, MetadataStore, HybridSearch,
|
|
IndexOptimizer, VectorStoreManager
|
|
)
|
|
|
|
# Triple stores
|
|
from semantica.triple_store import (
|
|
BlazegraphAdapter, JenaAdapter, RDF4JAdapter, GraphDBAdapter,
|
|
VirtuosoAdapter, TripleManager, QueryEngine, BulkLoader,
|
|
TripleStoreManager
|
|
)
|
|
|
|
# Embeddings
|
|
from semantica.embeddings import (
|
|
SemanticEmbedder, TextEmbedder, ImageEmbedder, AudioEmbedder,
|
|
MultimodalEmbedder, ContextManager, PoolingStrategies,
|
|
ProviderAdapter, EmbeddingOptimizer
|
|
)
|
|
|
|
# =============================================================================
|
|
# AI & REASONING MODULES
|
|
# =============================================================================
|
|
|
|
# RAG system
|
|
from semantica.qa_rag import (
|
|
RAGManager, SemanticChunker, PromptTemplates, RetrievalPolicies,
|
|
AnswerBuilder, ProvenanceTracker, AnswerValidator, RAGOptimizer,
|
|
ConversationManager
|
|
)
|
|
|
|
# Reasoning engine
|
|
from semantica.reasoning import (
|
|
InferenceEngine, SPARQLReasoner, ReteEngine, AbductiveReasoner,
|
|
DeductiveReasoner, RuleManager, ReasoningValidator, ExplanationGenerator,
|
|
ReasoningManager
|
|
)
|
|
|
|
# Multi-agent system
|
|
from semantica.agents import (
|
|
AgentManager, OrchestrationEngine, ToolRegistry, CostTracker,
|
|
SandboxManager, WorkflowEngine, AgentCommunication, PolicyEnforcer,
|
|
AgentAnalytics, MultiAgentManager
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# UTILITY MODULES
|
|
# =============================================================================
|
|
|
|
# Additional utilities
|
|
from semantica.utils import (
|
|
DataValidator, SchemaManager, TemplateManager, SeedManager,
|
|
SemanticDeduplicator, ConflictDetector, MultiProviderConfig,
|
|
AnalyticsDashboard, BusinessIntelligenceDashboard,
|
|
HealthcareProcessor, EnterpriseDeployment
|
|
)
|
|
|
|
# =============================================================================
|
|
# QUICK IMPORTS FOR COMMON USE CASES
|
|
# =============================================================================
|
|
|
|
# Basic usage
|
|
from semantica import Semantica
|
|
from semantica.processors import DocumentProcessor, WebProcessor, FeedProcessor
|
|
from semantica.context import ContextEngineer
|
|
from semantica.embeddings import SemanticEmbedder
|
|
from semantica.graph import KnowledgeGraphBuilder
|
|
from semantica.query import SPARQLQueryGenerator
|
|
from semantica.pipelines import ResearchPipeline, BusinessIntelligenceDashboard
|
|
from semantica.healthcare import HealthcareProcessor
|
|
from semantica.deployment import EnterpriseDeployment
|
|
from semantica.analytics import AnalyticsDashboard
|
|
|
|
# Advanced usage
|
|
from semantica.config import MultiProviderConfig
|
|
```
|
|
|
|
---
|
|
|
|
## 🔧 Module Dependencies
|
|
|
|
### Core Dependencies
|
|
```python
|
|
# Required for all modules
|
|
semantica[core] >= 1.0.0
|
|
|
|
# Optional dependencies by module
|
|
semantica[pdf] # PDF parsing
|
|
semantica[web] # Web scraping
|
|
semantica[feeds] # RSS/Atom feeds
|
|
semantica[office] # Office documents
|
|
semantica[scientific] # Scientific formats
|
|
semantica[all] # All dependencies
|
|
```
|
|
|
|
### External Dependencies
|
|
```python
|
|
# Vector stores
|
|
pinecone-client >= 2.0.0
|
|
faiss-cpu >= 1.7.0
|
|
weaviate-client >= 3.0.0
|
|
|
|
# Triple stores
|
|
rdflib >= 6.0.0
|
|
sparqlwrapper >= 2.0.0
|
|
|
|
# ML/AI
|
|
openai >= 1.0.0
|
|
transformers >= 4.20.0
|
|
torch >= 1.12.0
|
|
|
|
# Data processing
|
|
pandas >= 1.5.0
|
|
numpy >= 1.21.0
|
|
spacy >= 3.4.0
|
|
```
|
|
|
|
---
|
|
|
|
## 📊 Module Statistics
|
|
|
|
- **Total Main Modules**: 20
|
|
- **Total Submodules**: 120+
|
|
- **Total Classes**: 200+
|
|
- **Total Functions**: 1000+
|
|
- **Supported Formats**: 50+
|
|
- **Supported Languages**: 100+
|
|
- **Integration Points**: 30+
|
|
|
|
---
|
|
|
|
## 🎯 Solving Real-World Knowledge Graph Challenges
|
|
|
|
> **Complete solution for the fundamental problems in building production-ready Knowledge Graphs**
|
|
|
|
### **Problem 1: Stick to a Fixed Template**
|
|
```python
|
|
from semantica.templates import SchemaTemplate, SchemaEnforcer
|
|
|
|
# Define your business-specific schema
|
|
business_schema = SchemaTemplate(
|
|
name="company_knowledge_graph",
|
|
entities=["Company", "Person", "Product", "Department", "Project", "Quarterly_Report"],
|
|
relationships=["founded_by", "works_for", "manages", "belongs_to", "reports_to", "produces"],
|
|
constraints={
|
|
"Company": {"required_props": ["name", "industry", "founded_year", "headquarters"]},
|
|
"Person": {"required_props": ["name", "title", "department", "employee_id"]},
|
|
"Quarterly_Report": {"required_props": ["quarter", "year", "revenue", "company"]},
|
|
"founded_by": {"domain": "Company", "range": "Person"},
|
|
"produces": {"domain": "Company", "range": "Product"}
|
|
}
|
|
)
|
|
|
|
# Enforce consistent structure
|
|
schema_enforcer = SchemaEnforcer()
|
|
enforced_entities = schema_enforcer.enforce_entity_schema(extracted_entities, business_schema)
|
|
enforced_relationships = schema_enforcer.enforce_relationship_schema(extracted_relations, business_schema)
|
|
```
|
|
|
|
### **Problem 2: Start with What We Already Know**
|
|
```python
|
|
from semantica.seed import SeedDataManager, KnowledgeSeeder
|
|
|
|
# Load your existing verified data
|
|
seed_manager = SeedDataManager()
|
|
seed_manager.load_products("verified_products.csv")
|
|
seed_manager.load_departments("org_chart.json")
|
|
seed_manager.load_employees("hr_database")
|
|
|
|
# Seed the knowledge graph with verified foundation
|
|
knowledge_seeder = KnowledgeSeeder()
|
|
seeded_graph = knowledge_seeder.seed_entities(seed_data, verification_rules=True)
|
|
seeded_graph = knowledge_seeder.seed_relationships(org_chart, hierarchy_rules=True)
|
|
|
|
# Build on this foundation
|
|
integrator = DataIntegrator()
|
|
final_graph = integrator.integrate_seed_with_extracted(seeded_graph, extracted_data)
|
|
```
|
|
|
|
### **Problem 3: Clean Up and Merge Duplicates**
|
|
```python
|
|
from semantica.deduplication import DuplicateDetector, EntityMerger, SimilarityEngine
|
|
|
|
# Detect semantic duplicates
|
|
duplicate_detector = DuplicateDetector()
|
|
duplicates = duplicate_detector.find_semantic_duplicates(entities)
|
|
# This will find "First Quarter Sales" and "Q1 Sales Report" as duplicates
|
|
|
|
# Advanced similarity detection
|
|
similarity_engine = SimilarityEngine()
|
|
similarity = similarity_engine.calculate_semantic_similarity("First Quarter Sales", "Q1 Sales Report")
|
|
# Returns high similarity score (e.g., 0.92)
|
|
|
|
# Merge duplicates intelligently
|
|
entity_merger = EntityMerger()
|
|
merged = entity_merger.merge_duplicates(duplicates, strategy="highest_confidence")
|
|
merged = entity_merger.merge_with_validation(duplicates, validation_rules)
|
|
```
|
|
|
|
### **Problem 4: Flag When Sources Disagree**
|
|
```python
|
|
from semantica.conflicts import ConflictDetector, SourceTracker, DisagreementResolver
|
|
|
|
# Detect conflicting information
|
|
conflict_detector = ConflictDetector()
|
|
conflicts = conflict_detector.detect_value_conflicts(entities, "sales_figure")
|
|
# Finds $10M vs $12M sales figures
|
|
|
|
# Track exact sources
|
|
source_tracker = SourceTracker()
|
|
sources = source_tracker.track_property_sources(property, "sales_figure", "$10M")
|
|
# Returns: [{"document": "Q1_Report.pdf", "page": 5, "section": "Financial Summary"}]
|
|
|
|
# Flag for investigation
|
|
disagreement_resolver = DisagreementResolver()
|
|
flagged = disagreement_resolver.flag_for_investigation(conflicts)
|
|
# Generates investigation report with exact document locations
|
|
```
|
|
|
|
### **Complete Production-Ready Solution**
|
|
```python
|
|
from semantica import Semantica
|
|
from semantica.templates import SchemaTemplate
|
|
from semantica.seed import SeedDataManager
|
|
from semantica.deduplication import DuplicateDetector, EntityMerger
|
|
from semantica.conflicts import ConflictDetector, SourceTracker
|
|
from semantica.kg_qa import KGQualityAssessor
|
|
|
|
# Initialize Semantica with quality assurance
|
|
core = Semantica(
|
|
llm_provider="openai",
|
|
embedding_model="text-embedding-3-large",
|
|
vector_store="pinecone",
|
|
graph_db="neo4j",
|
|
quality_assurance=True, # Enable all QA features
|
|
conflict_detection=True, # Enable conflict detection
|
|
deduplication=True # Enable advanced deduplication
|
|
)
|
|
|
|
# 1. Define your business schema
|
|
business_schema = SchemaTemplate.load("business_schema.yaml")
|
|
|
|
# 2. Seed with verified data
|
|
seed_manager = SeedDataManager()
|
|
seed_manager.load_verified_data("verified_entities.json")
|
|
seeded_graph = seed_manager.create_foundation_graph(business_schema)
|
|
|
|
# 3. Process documents with quality controls
|
|
knowledge_base = core.build_knowledge_base(
|
|
sources=["documents/"],
|
|
schema_template=business_schema,
|
|
seed_data=seeded_graph,
|
|
enable_deduplication=True,
|
|
enable_conflict_detection=True,
|
|
enable_quality_assurance=True
|
|
)
|
|
|
|
# 4. Get quality report
|
|
kg_qa = KGQualityAssessor()
|
|
quality_report = kg_qa.generate_quality_report(knowledge_base)
|
|
print(f"Knowledge Graph Quality Score: {quality_report.overall_score}")
|
|
print(f"Duplicates Found: {quality_report.duplicates_count}")
|
|
print(f"Conflicts Detected: {quality_report.conflicts_count}")
|
|
print(f"Source Disagreements: {quality_report.disagreements_count}")
|
|
|
|
# 5. Get investigation guide for conflicts
|
|
if quality_report.conflicts_count > 0:
|
|
investigation_guide = quality_report.get_investigation_guide()
|
|
print("Conflicts requiring investigation:")
|
|
for conflict in investigation_guide.conflicts:
|
|
print(f"- {conflict.property}: {conflict.values}")
|
|
print(f" Sources: {conflict.source_documents}")
|
|
```
|
|
|
|
---
|
|
|
|
*This comprehensive module reference covers all major components of the Semantica toolkit. Each module is designed to be modular, extensible, and production-ready for enterprise use cases. The toolkit specifically addresses the fundamental challenges in building Knowledge Graphs that are consistent, reliable, and production-ready.*
|