From b716b17bdee1e6f8b99fb733d4c65db3089c7fa0 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sat, 22 Nov 2025 14:49:08 +0530 Subject: [PATCH] Upgrade documentation: mkdocstrings, jupyter notebooks, and theme fix --- docs/MODULES_DOCUMENTATION.md | 4361 ----------------- docs/api.md | 295 -- docs/cookbook.md | 182 - .../advanced/Advanced_Extraction.ipynb | 211 + .../advanced/Advanced_Graph_Analytics.ipynb | 179 + .../Complete_Visualization_Suite.ipynb | 250 + .../Conflict_Resolution_Strategies.ipynb | 313 ++ .../advanced/Multi_Format_Export.ipynb | 221 + .../Multi_Source_Data_Integration.ipynb | 194 + .../advanced/Pipeline_Orchestration.ipynb | 195 + .../advanced/Reasoning_and_Inference.ipynb | 272 + .../Semantic_Layer_Construction.ipynb | 194 + .../advanced/Temporal_Knowledge_Graphs.ipynb | 171 + .../advanced/Text_Chunking_Strategies.ipynb | 260 + .../advanced/Unstructured_to_Ontology.ipynb | 166 + .../Building_Knowledge_Graphs.ipynb | 168 + .../introduction/Configuration_Basics.ipynb | 352 ++ .../introduction/Conflict_Detection.ipynb | 125 + .../introduction/Data_Ingestion.ipynb | 208 + .../introduction/Data_Normalization.ipynb | 228 + .../cookbook/introduction/Deduplication.ipynb | 123 + .../introduction/Document_Parsing.ipynb | 247 + .../introduction/Embedding_Generation.ipynb | 100 + .../introduction/Entity_Extraction.ipynb | 106 + docs/cookbook/introduction/Export.ipynb | 177 + .../introduction/Graph_Analytics.ipynb | 162 + .../cookbook/introduction/Graph_Quality.ipynb | 156 + docs/cookbook/introduction/Ontology.ipynb | 155 + .../introduction/Relation_Extraction.ipynb | 105 + docs/cookbook/introduction/Vector_Store.ipynb | 132 + .../cookbook/introduction/Visualization.ipynb | 136 + .../introduction/Welcome_to_Semantica.ipynb | 903 ++++ .../Your_First_Knowledge_Graph.ipynb | 287 ++ .../advanced_rag/GraphRAG_Complete.ipynb | 18 + .../biomedical/Drug_Discovery_Pipeline.ipynb | 770 +++ .../biomedical/Genomic_Variant_Analysis.ipynb | 775 +++ .../DeFi_Protocol_Intelligence.ipynb | 703 +++ .../Transaction_Network_Analysis.ipynb | 642 +++ .../Anomaly_Detection_Real_Time.ipynb | 471 ++ .../cybersecurity/Incident_Analysis.ipynb | 455 ++ .../cybersecurity/Threat_Correlation.ipynb | 426 ++ .../Threat_Intelligence_Hybrid_RAG.ipynb | 585 +++ .../Threat_Intelligence_Integration.ipynb | 575 +++ .../Vulnerability_Tracking.ipynb | 142 + .../finance/Financial_Data_Integration.ipynb | 538 ++ .../finance/Financial_Reports_Analysis.ipynb | 420 ++ .../use_cases/finance/Fraud_Detection.ipynb | 402 ++ .../Investment_Analysis_Hybrid_RAG.ipynb | 398 ++ .../finance/Market_Intelligence.ipynb | 493 ++ .../finance/Regulatory_Compliance.ipynb | 370 ++ .../Clinical_Reports_Processing.ipynb | 423 ++ .../healthcare/Disease_Network_Analysis.ipynb | 405 ++ .../Drug_Interactions_Analysis.ipynb | 415 ++ .../Healthcare_GraphRAG_Hybrid.ipynb | 810 +++ .../Medical_Database_Integration.ipynb | 601 +++ .../Medical_Literature_GraphRAG.ipynb | 373 ++ .../healthcare/Patient_Records_Temporal.ipynb | 324 ++ ...etwork_Analysis_Intelligence_Reports.ipynb | 779 +++ .../Energy_Market_Analysis.ipynb | 433 ++ .../Environmental_Impact.ipynb | 472 ++ .../renewable_energy/Grid_Management.ipynb | 455 ++ .../Resource_Optimization.ipynb | 430 ++ .../Supply_Chain_Analysis.ipynb | 447 ++ .../Supply_Chain_Data_Integration.ipynb | 589 +++ .../Supply_Chain_Risk_Management.ipynb | 699 +++ .../trading/Market_Data_Analysis.ipynb | 355 ++ .../trading/News_Sentiment_Analysis.ipynb | 367 ++ .../trading/Real_Time_Market_Data.ipynb | 566 +++ .../trading/Real_Time_Monitoring.ipynb | 394 ++ .../use_cases/trading/Risk_Assessment.ipynb | 399 ++ .../trading/Strategy_Backtesting.ipynb | 412 ++ docs/css/custom.css | 4 + docs/reference/core.md | 3 + docs/reference/embeddings.md | 3 + docs/reference/export.md | 3 + docs/reference/ingest.md | 3 + docs/reference/kg.md | 3 + docs/reference/normalize.md | 3 + docs/reference/ontology.md | 3 + docs/reference/parse.md | 3 + docs/reference/pipeline.md | 3 + docs/reference/reasoning.md | 3 + docs/reference/semantic_extract.md | 3 + docs/reference/triple_store.md | 3 + docs/reference/utils.md | 3 + docs/reference/vector_store.md | 3 + docs/reference/visualization.md | 3 + mkdocs.yml | 58 +- requirements-docs.txt | 4 +- setup_docs.py | 46 + 90 files changed, 24975 insertions(+), 4847 deletions(-) delete mode 100644 docs/MODULES_DOCUMENTATION.md delete mode 100644 docs/api.md delete mode 100644 docs/cookbook.md create mode 100644 docs/cookbook/advanced/Advanced_Extraction.ipynb create mode 100644 docs/cookbook/advanced/Advanced_Graph_Analytics.ipynb create mode 100644 docs/cookbook/advanced/Complete_Visualization_Suite.ipynb create mode 100644 docs/cookbook/advanced/Conflict_Resolution_Strategies.ipynb create mode 100644 docs/cookbook/advanced/Multi_Format_Export.ipynb create mode 100644 docs/cookbook/advanced/Multi_Source_Data_Integration.ipynb create mode 100644 docs/cookbook/advanced/Pipeline_Orchestration.ipynb create mode 100644 docs/cookbook/advanced/Reasoning_and_Inference.ipynb create mode 100644 docs/cookbook/advanced/Semantic_Layer_Construction.ipynb create mode 100644 docs/cookbook/advanced/Temporal_Knowledge_Graphs.ipynb create mode 100644 docs/cookbook/advanced/Text_Chunking_Strategies.ipynb create mode 100644 docs/cookbook/advanced/Unstructured_to_Ontology.ipynb create mode 100644 docs/cookbook/introduction/Building_Knowledge_Graphs.ipynb create mode 100644 docs/cookbook/introduction/Configuration_Basics.ipynb create mode 100644 docs/cookbook/introduction/Conflict_Detection.ipynb create mode 100644 docs/cookbook/introduction/Data_Ingestion.ipynb create mode 100644 docs/cookbook/introduction/Data_Normalization.ipynb create mode 100644 docs/cookbook/introduction/Deduplication.ipynb create mode 100644 docs/cookbook/introduction/Document_Parsing.ipynb create mode 100644 docs/cookbook/introduction/Embedding_Generation.ipynb create mode 100644 docs/cookbook/introduction/Entity_Extraction.ipynb create mode 100644 docs/cookbook/introduction/Export.ipynb create mode 100644 docs/cookbook/introduction/Graph_Analytics.ipynb create mode 100644 docs/cookbook/introduction/Graph_Quality.ipynb create mode 100644 docs/cookbook/introduction/Ontology.ipynb create mode 100644 docs/cookbook/introduction/Relation_Extraction.ipynb create mode 100644 docs/cookbook/introduction/Vector_Store.ipynb create mode 100644 docs/cookbook/introduction/Visualization.ipynb create mode 100644 docs/cookbook/introduction/Welcome_to_Semantica.ipynb create mode 100644 docs/cookbook/introduction/Your_First_Knowledge_Graph.ipynb create mode 100644 docs/cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb create mode 100644 docs/cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb create mode 100644 docs/cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb create mode 100644 docs/cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/cybersecurity/Anomaly_Detection_Real_Time.ipynb create mode 100644 docs/cookbook/use_cases/cybersecurity/Incident_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/cybersecurity/Threat_Correlation.ipynb create mode 100644 docs/cookbook/use_cases/cybersecurity/Threat_Intelligence_Hybrid_RAG.ipynb create mode 100644 docs/cookbook/use_cases/cybersecurity/Threat_Intelligence_Integration.ipynb create mode 100644 docs/cookbook/use_cases/cybersecurity/Vulnerability_Tracking.ipynb create mode 100644 docs/cookbook/use_cases/finance/Financial_Data_Integration.ipynb create mode 100644 docs/cookbook/use_cases/finance/Financial_Reports_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/finance/Fraud_Detection.ipynb create mode 100644 docs/cookbook/use_cases/finance/Investment_Analysis_Hybrid_RAG.ipynb create mode 100644 docs/cookbook/use_cases/finance/Market_Intelligence.ipynb create mode 100644 docs/cookbook/use_cases/finance/Regulatory_Compliance.ipynb create mode 100644 docs/cookbook/use_cases/healthcare/Clinical_Reports_Processing.ipynb create mode 100644 docs/cookbook/use_cases/healthcare/Disease_Network_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/healthcare/Drug_Interactions_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/healthcare/Healthcare_GraphRAG_Hybrid.ipynb create mode 100644 docs/cookbook/use_cases/healthcare/Medical_Database_Integration.ipynb create mode 100644 docs/cookbook/use_cases/healthcare/Medical_Literature_GraphRAG.ipynb create mode 100644 docs/cookbook/use_cases/healthcare/Patient_Records_Temporal.ipynb create mode 100644 docs/cookbook/use_cases/intelligence/Network_Analysis_Intelligence_Reports.ipynb create mode 100644 docs/cookbook/use_cases/renewable_energy/Energy_Market_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/renewable_energy/Environmental_Impact.ipynb create mode 100644 docs/cookbook/use_cases/renewable_energy/Grid_Management.ipynb create mode 100644 docs/cookbook/use_cases/renewable_energy/Resource_Optimization.ipynb create mode 100644 docs/cookbook/use_cases/renewable_energy/Supply_Chain_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/supply_chain/Supply_Chain_Data_Integration.ipynb create mode 100644 docs/cookbook/use_cases/supply_chain/Supply_Chain_Risk_Management.ipynb create mode 100644 docs/cookbook/use_cases/trading/Market_Data_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/trading/News_Sentiment_Analysis.ipynb create mode 100644 docs/cookbook/use_cases/trading/Real_Time_Market_Data.ipynb create mode 100644 docs/cookbook/use_cases/trading/Real_Time_Monitoring.ipynb create mode 100644 docs/cookbook/use_cases/trading/Risk_Assessment.ipynb create mode 100644 docs/cookbook/use_cases/trading/Strategy_Backtesting.ipynb create mode 100644 docs/reference/core.md create mode 100644 docs/reference/embeddings.md create mode 100644 docs/reference/export.md create mode 100644 docs/reference/ingest.md create mode 100644 docs/reference/kg.md create mode 100644 docs/reference/normalize.md create mode 100644 docs/reference/ontology.md create mode 100644 docs/reference/parse.md create mode 100644 docs/reference/pipeline.md create mode 100644 docs/reference/reasoning.md create mode 100644 docs/reference/semantic_extract.md create mode 100644 docs/reference/triple_store.md create mode 100644 docs/reference/utils.md create mode 100644 docs/reference/vector_store.md create mode 100644 docs/reference/visualization.md create mode 100644 setup_docs.py diff --git a/docs/MODULES_DOCUMENTATION.md b/docs/MODULES_DOCUMENTATION.md deleted file mode 100644 index f9a7fadc..00000000 --- a/docs/MODULES_DOCUMENTATION.md +++ /dev/null @@ -1,4361 +0,0 @@ -# Semantica Framework - Complete Modules Documentation - -This document provides detailed documentation of all modules, submodules, classes, methods, and parameters in the Semantica framework. - ---- - -## Table of Contents - -1. [Core Modules](#core-modules) -2. [Ingestion Modules](#ingestion-modules) -3. [Parsing Modules](#parsing-modules) -4. [Normalization Modules](#normalization-modules) -5. [Semantic Extraction Modules](#semantic-extraction-modules) -6. [Knowledge Graph Modules](#knowledge-graph-modules) -7. [Embeddings Modules](#embeddings-modules) -8. [Pipeline Modules](#pipeline-modules) -9. [Reasoning Modules](#reasoning-modules) -10. [Vector Store Modules](#vector-store-modules) -11. [Triple Store Modules](#triple-store-modules) -12. [Export Modules](#export-modules) -13. [Visualization Modules](#visualization-modules) -14. [Quality Assurance Modules](#quality-assurance-modules) -15. [Context Modules](#context-modules) -16. [Deduplication Modules](#deduplication-modules) -17. [Conflict Modules](#conflict-modules) -18. [Split Modules](#split-modules) -19. [Ontology Modules](#ontology-modules) -20. [Seed Modules](#seed-modules) -21. [Utils Modules](#utils-modules) - ---- - -## Core Modules - -### `semantica.core.config_manager` - -**What it does:** -This module provides comprehensive configuration management for the Semantica framework. It handles loading configuration from files (YAML/JSON), environment variables, validation, and dynamic updates. The module supports nested configuration access via dot notation, configuration inheritance and merging, and automatic type conversion from environment variables. - -**Key Features:** -- Load configuration from YAML/JSON files -- Support for environment variables with `SEMANTICA_` prefix -- Configuration validation with detailed error messages -- Dynamic configuration updates at runtime -- Configuration inheritance and merging -- Nested configuration access via dot notation - -#### Class: `Config` - -Configuration data class that stores all framework configuration settings. - -**Methods:** - -##### `__init__(config_dict: Optional[Dict[str, Any]] = None, **kwargs)` -Initialize configuration. - -**Parameters:** -- `config_dict` (Optional[Dict[str, Any]]): Dictionary of configuration values -- `**kwargs`: Additional configuration parameters merged into config - -**Attributes:** -- `llm_provider`: LLM provider configuration -- `embedding_model`: Embedding model settings -- `vector_store`: Vector store configuration -- `graph_db`: Graph database settings -- `processing`: Processing pipeline settings -- `logging`: Logging configuration -- `quality`: Quality assurance settings -- `security`: Security settings -- `custom`: Custom configuration - -##### `validate() -> None` -Validate configuration settings. Checks types, ranges, and required fields. - -**Raises:** -- `ConfigurationError`: If configuration is invalid with detailed error messages - -##### `to_dict() -> Dict[str, Any]` -Convert configuration to dictionary. - -**Returns:** -- `dict`: Configuration as dictionary - -##### `get(key_path: str, default: Any = None) -> Any` -Get nested configuration value by key path. - -**Parameters:** -- `key_path` (str): Dot-separated key path (e.g., "processing.batch_size") -- `default` (Any): Default value if key not found - -**Returns:** -- Configuration value or default - -##### `set(key_path: str, value: Any) -> None` -Set nested configuration value by key path. - -**Parameters:** -- `key_path` (str): Dot-separated key path -- `value` (Any): Value to set - -##### `update(updates: Dict[str, Any], merge: bool = True) -> None` -Update configuration with new values. - -**Parameters:** -- `updates` (Dict[str, Any]): Dictionary of updates -- `merge` (bool): Whether to merge nested dictionaries (default: True) - -#### Class: `ConfigManager` - -Configuration management system for loading, validating, and managing configuration. - -**Methods:** - -##### `__init__()` -Initialize configuration manager. - -##### `load_from_file(file_path: Union[str, Path], validate: bool = True) -> Config` -Load configuration from file. - -**Parameters:** -- `file_path` (Union[str, Path]): Path to configuration file (YAML or JSON) -- `validate` (bool): Whether to validate configuration after loading (default: True) - -**Returns:** -- `Config`: Loaded configuration object - -**Raises:** -- `ConfigurationError`: If file cannot be loaded or is invalid - -##### `load_from_dict(config_dict: Dict[str, Any], validate: bool = True) -> Config` -Load configuration from dictionary. - -**Parameters:** -- `config_dict` (Dict[str, Any]): Dictionary of configuration values -- `validate` (bool): Whether to validate configuration after loading (default: True) - -**Returns:** -- `Config`: Configuration object - -##### `merge_configs(*configs: Config, validate: bool = True) -> Config` -Merge multiple configurations. Later configurations take priority. - -**Parameters:** -- `*configs` (Config): Configuration objects to merge -- `validate` (bool): Whether to validate merged configuration (default: True) - -**Returns:** -- `Config`: Merged configuration - -##### `get_config() -> Optional[Config]` -Get current configuration. - -**Returns:** -- Current Config object or None if not loaded - -##### `set_config(config: Config, validate: bool = True) -> None` -Set current configuration. - -**Parameters:** -- `config` (Config): Configuration object to set -- `validate` (bool): Whether to validate configuration (default: True) - -##### `reload(file_path: Optional[Union[str, Path]] = None) -> Config` -Reload configuration from file. - -**Parameters:** -- `file_path` (Optional[Union[str, Path]]): Path to configuration file. If None, uses last loaded file. - -**Returns:** -- `Config`: Reloaded configuration - -**Different Approaches and Strategies:** - -The `ConfigManager` supports multiple approaches for configuration management: - -1. **File-based Configuration** - Load from YAML/JSON files (recommended for production) -2. **Dictionary-based Configuration** - Load from Python dictionaries (useful for programmatic setup) -3. **Environment Variable Configuration** - Use environment variables with `SEMANTICA_` prefix -4. **Merged Configuration** - Combine multiple configuration sources with priority -5. **Dynamic Configuration** - Update configuration at runtime without reloading - -**When to Use Each Approach:** - -- **File-based**: Production environments, version-controlled configurations, team collaboration -- **Dictionary-based**: Testing, programmatic configuration, dynamic setups -- **Environment Variables**: Docker containers, CI/CD pipelines, sensitive data -- **Merged Configuration**: Override defaults, environment-specific settings, feature flags -- **Dynamic Configuration**: Runtime adjustments, A/B testing, hot-reloading - -**Code Examples for Different Approaches:** - -**Approach 1: File-based Configuration (Production)** -```python -from semantica.core import ConfigManager - -# Load from YAML file (recommended for production) -manager = ConfigManager() -config = manager.load_from_file("config.yaml") - -# Access nested values -batch_size = config.get("processing.batch_size", default=32) -model_name = config.get("embedding_model.name", default="default-model") - -# Example config.yaml structure: -# processing: -# batch_size: 32 -# max_workers: 4 -# embedding_model: -# name: "sentence-transformers/all-MiniLM-L6-v2" -# device: "cpu" -``` - -**Approach 2: Dictionary-based Configuration (Programmatic)** -```python -from semantica.core import ConfigManager - -# Load from dictionary (useful for testing or dynamic setup) -manager = ConfigManager() -config_dict = { - "processing": { - "batch_size": 32, - "max_workers": 4 - }, - "quality": { - "min_confidence": 0.8 - } -} -config = manager.load_from_dict(config_dict) - -# Update programmatically -config.set("processing.batch_size", 64) -config.update({"processing": {"max_workers": 8}}) -``` - -**Approach 3: Environment Variable Configuration (Docker/CI/CD)** -```python -import os -from semantica.core import ConfigManager, Config - -# Set environment variables (typically done in shell/Docker) -os.environ["SEMANTICA_PROCESSING_BATCH_SIZE"] = "64" -os.environ["SEMANTICA_EMBEDDING_MODEL_NAME"] = "custom-model" - -# Load configuration (environment variables automatically loaded) -manager = ConfigManager() -config = manager.load_from_dict({}) # Base config, env vars override - -# Environment variables with SEMANTICA_ prefix are automatically loaded -# Format: SEMANTICA_
_ (uppercase, underscores) -``` - -**Approach 4: Merged Configuration (Override Defaults)** -```python -from semantica.core import ConfigManager, Config - -# Create base configuration -base_config = Config(config_dict={ - "processing": {"batch_size": 32, "max_workers": 4}, - "embedding_model": {"name": "default-model"} -}) - -# Create override configuration -override_config = Config(config_dict={ - "processing": {"batch_size": 64}, # Override batch_size - "embedding_model": {"device": "cuda"} # Add new setting -}) - -# Merge configurations (later configs take priority) -manager = ConfigManager() -merged = manager.merge_configs(base_config, override_config) - -# Result: batch_size=64, max_workers=4, name="default-model", device="cuda" -print(merged.get("processing.batch_size")) # 64 (from override) -print(merged.get("processing.max_workers")) # 4 (from base) -``` - -**Approach 5: Dynamic Runtime Configuration** -```python -from semantica.core import ConfigManager - -# Load initial configuration -manager = ConfigManager() -config = manager.load_from_file("config.yaml") - -# Dynamically update configuration at runtime -config.set("processing.batch_size", 128) # Increase batch size -config.update({ - "embedding_model": { - "device": "cuda", # Switch to GPU - "normalize": True - } -}) - -# Changes take effect immediately without reloading -# Useful for A/B testing or performance tuning -``` - -**Approach 6: Configuration Validation and Error Handling** -```python -from semantica.core import ConfigManager, Config -from semantica.utils.exceptions import ConfigurationError - -manager = ConfigManager() - -try: - # Load and validate configuration - config = manager.load_from_file("config.yaml", validate=True) - - # Manual validation - config.validate() # Raises ConfigurationError if invalid - -except ConfigurationError as e: - print(f"Configuration error: {e}") - # Handle invalid configuration - # Option 1: Use defaults - config = manager.load_from_dict({"processing": {"batch_size": 32}}) - - # Option 2: Load fallback configuration - config = manager.load_from_file("config.default.yaml", validate=False) -``` - -**Best Practices:** - -1. **Use file-based configuration for production** - Easier to version control and manage -2. **Use environment variables for secrets** - API keys, passwords, tokens -3. **Merge configurations for flexibility** - Base config + environment-specific overrides -4. **Validate early** - Always validate configuration on load -5. **Use dot notation** - Access nested values with `config.get("section.key")` -6. **Document defaults** - Provide sensible defaults for all configuration options - ---- - -### `semantica.core.lifecycle` - -**What it does:** -This module manages the complete lifecycle of the Semantica framework, including startup and shutdown sequences, component health monitoring, and resource management. It provides a hook-based system for executing code at specific lifecycle stages with priority ordering, allowing components to initialize and cleanup in the correct order. - -**Key Features:** -- Priority-based startup/shutdown hooks -- Component registration and health monitoring -- State management and tracking (uninitialized, ready, running, stopped, error) -- Graceful error handling during lifecycle transitions -- Health check system for all registered components - -#### Class: `LifecycleManager` - -System lifecycle manager that coordinates startup, shutdown, and health monitoring. - -**Methods:** - -##### `__init__()` -Initialize lifecycle manager. Creates manager in UNINITIALIZED state. - -##### `startup() -> None` -Execute startup sequence. Runs all registered startup hooks in priority order. - -**Raises:** -- `SemanticaError`: If startup fails - -##### `shutdown(graceful: bool = True) -> None` -Execute shutdown sequence. - -**Parameters:** -- `graceful` (bool): Whether to shutdown gracefully (default: True) - - True: Continue shutdown even if hooks fail - - False: Stop shutdown on first hook failure - -**Raises:** -- `SemanticaError`: If shutdown fails and graceful=False - -##### `health_check() -> Dict[str, HealthStatus]` -Perform comprehensive system health check. - -**Returns:** -- Dictionary mapping component names to HealthStatus objects - -##### `register_component(name: str, component: Any) -> None` -Register a component for health monitoring. - -**Parameters:** -- `name` (str): Component name -- `component` (Any): Component instance - -##### `unregister_component(name: str) -> None` -Unregister a component. - -**Parameters:** -- `name` (str): Component name - -##### `register_startup_hook(hook_fn: Callable[[], None], priority: int = 50) -> None` -Register a startup hook. - -**Parameters:** -- `hook_fn` (Callable[[], None]): Function to call during startup (no arguments) -- `priority` (int): Hook priority (lower = earlier execution, default: 50) - -##### `register_shutdown_hook(hook_fn: Callable[[], None], priority: int = 50) -> None` -Register a shutdown hook. - -**Parameters:** -- `hook_fn` (Callable[[], None]): Function to call during shutdown -- `priority` (int): Hook priority (lower = earlier execution, default: 50) - -##### `get_state() -> SystemState` -Get current system state. - -**Returns:** -- Current system state - -##### `is_ready() -> bool` -Check if system is ready. - -**Returns:** -- True if system is ready, False otherwise - -##### `is_running() -> bool` -Check if system is running. - -**Returns:** -- True if system is running, False otherwise - -##### `get_health_summary() -> Dict[str, Any]` -Get summary of system health. - -**Returns:** -- Dictionary with health summary information - -**Different Approaches and Strategies:** - -The lifecycle manager provides 6 different approaches for managing system lifecycle: - -1. **Priority-Based Hook System** - Execute hooks in priority order (lower priority = earlier execution) -2. **Graceful vs Non-Graceful Shutdown** - Continue or stop on errors during shutdown -3. **Component Health Monitoring** - Automatic health checking with different health check methods -4. **State-Based Management** - Track system state transitions (uninitialized → initializing → ready → running → stopping → stopped) -5. **Error Handling Strategies** - Different error handling during lifecycle transitions -6. **Resource Cleanup Approaches** - Automatic cleanup using cleanup() or close() methods - -**When to Use Each Approach:** - -| Approach | Use Case | Example | -|----------|----------|---------| -| Priority-Based Hooks | Need ordered initialization | Config (priority=10) → Database (priority=20) → Cache (priority=30) | -| Graceful Shutdown | Production systems | Continue cleanup even if some components fail | -| Non-Graceful Shutdown | Development/Debugging | Stop immediately on first error for easier debugging | -| Component Health Monitoring | Production monitoring | Track health of database, cache, API connections | -| State-Based Management | Complex systems | Track system state for UI dashboards, monitoring | -| Resource Cleanup | Resource management | Automatically close connections, files, threads | - -**Detailed Examples for Each Approach:** - -**Approach 1: Priority-Based Hook System** -```python -from semantica.core import LifecycleManager - -manager = LifecycleManager() - -# Lower priority = earlier execution -manager.register_startup_hook(init_logging, priority=1) # First -manager.register_startup_hook(init_config, priority=10) # Second -manager.register_startup_hook(init_database, priority=20) # Third -manager.register_startup_hook(init_cache, priority=30) # Fourth - -manager.startup() # Executes in order: logging → config → database → cache -``` - -**Approach 2: Graceful vs Non-Graceful Shutdown** -```python -# Graceful shutdown (production) - continues even if hooks fail -manager.shutdown(graceful=True) # Logs warnings but continues - -# Non-graceful shutdown (debugging) - stops on first error -manager.shutdown(graceful=False) # Raises error on first failure -``` - -**Approach 3: Component Health Monitoring** -```python -# Register components with automatic health checking -manager.register_component("database", db_connection) -manager.register_component("cache", cache_client) - -# Components can implement health_check() method -class DatabaseConnection: - def health_check(self): - return {"healthy": self.is_connected(), "message": "Connected"} - -# Or use simple boolean -class CacheClient: - def health_check(self): - return self.is_alive() # Returns True/False - -# Automatic health checking -health = manager.health_check() -for name, status in health.items(): - print(f"{name}: {status.healthy} - {status.message}") -``` - -**Approach 4: State-Based Management** -```python -# Track system state transitions -state = manager.get_state() # Returns SystemState enum -print(f"Current state: {state}") # uninitialized, initializing, ready, running, etc. - -# Check if system is ready -if manager.is_ready(): - print("System ready for processing") - -# Check if system is running -if manager.is_running(): - print("System is actively running") -``` - -**Approach 5: Error Handling Strategies** -```python -# Startup hooks with error handling -def init_database(): - try: - db.connect() - except Exception as e: - # Error stops startup (raises SemanticaError) - raise - -# Shutdown hooks with graceful error handling -def cleanup_database(): - try: - db.close() - except Exception as e: - # In graceful mode, error is logged but doesn't stop shutdown - logger.warning(f"Cleanup failed: {e}") -``` - -**Approach 6: Resource Cleanup Approaches** -```python -# Components with cleanup() method -class Resource: - def cleanup(self): - self.close_connections() - self.release_resources() - -# Components with close() method -class Connection: - def close(self): - self.connection.close() - -# Automatic cleanup on shutdown -manager.register_component("resource", Resource()) -manager.register_component("connection", Connection()) -manager.shutdown() # Automatically calls cleanup() or close() -``` - -**Code Example:** -```python -from semantica.core import LifecycleManager - -# Initialize lifecycle manager -manager = LifecycleManager() - -# Register components for health monitoring -manager.register_component("database", db_connection) -manager.register_component("cache", cache_client) - -# Register startup hooks with priorities (lower = earlier execution) -def init_config(): - print("Initializing configuration...") - -def init_database(): - print("Connecting to database...") - -manager.register_startup_hook(init_config, priority=10) # Runs first -manager.register_startup_hook(init_database, priority=20) # Runs second - -# Register shutdown hooks -def cleanup_database(): - print("Closing database connections...") - -manager.register_shutdown_hook(cleanup_database, priority=10) - -# Execute startup sequence -manager.startup() # Hooks execute in priority order - -# Check system health -health = manager.health_check() -for component, status in health.items(): - print(f"{component}: {'Healthy' if status.healthy else 'Unhealthy'}") - -# Get health summary -summary = manager.get_health_summary() -print(f"System state: {summary['state']}") -print(f"Healthy components: {summary['healthy_components']}/{summary['total_components']}") - -# Check if system is ready -if manager.is_ready(): - print("System is ready for processing") - -# Shutdown gracefully -manager.shutdown(graceful=True) # Continues even if hooks fail -``` - ---- - -### `semantica.core.orchestrator` - -**What it does:** -This is the main orchestrator module that coordinates all framework components and manages the overall execution flow. It provides the primary entry point for the Semantica framework, handling framework initialization, knowledge base construction from various data sources, pipeline execution, resource management, plugin system coordination, and system health monitoring. - -**Key Features:** -- Framework initialization and lifecycle management -- Knowledge base construction from various data sources -- Pipeline execution and resource management -- Plugin system coordination -- System health monitoring -- Automatic component initialization - -#### Class: `Semantica` - -Main Semantica framework class - primary entry point. - -**Methods:** - -##### `__init__(config: Optional[Union[Config, Dict[str, Any]]] = None, **kwargs)` -Initialize Semantica framework. - -**Parameters:** -- `config` (Optional[Union[Config, Dict[str, Any]]): Configuration object or dict -- `**kwargs`: Additional configuration parameters - -##### `initialize() -> None` -Initialize all framework components. - -**Raises:** -- `ConfigurationError`: If configuration is invalid -- `SemanticaError`: If initialization fails - -##### `build_knowledge_base(sources: List[Union[str, Path]], **kwargs) -> Dict[str, Any]` -Build knowledge base from data sources. - -**Parameters:** -- `sources` (List[Union[str, Path]]): List of data sources (files, URLs, streams) -- `**kwargs`: Additional processing options: - - `pipeline`: Custom pipeline configuration - - `embeddings`: Whether to generate embeddings (default: True) - - `graph`: Whether to build knowledge graph (default: True) - - `normalize`: Whether to normalize data (default: True) - - `fail_fast`: Whether to fail on first error (default: False) - -**Returns:** -- Dictionary containing: - - `knowledge_graph`: Knowledge graph data - - `embeddings`: Embedding vectors - - `metadata`: Processing metadata - - `statistics`: Processing statistics - - `results`: Processing results - -**Raises:** -- `ProcessingError`: If processing fails - -##### `run_pipeline(pipeline: Union[Dict[str, Any], Any], data: Any) -> Dict[str, Any]` -Execute a processing pipeline. - -**Parameters:** -- `pipeline` (Union[Dict[str, Any], Any]): Pipeline object or configuration dictionary -- `data` (Any): Input data for pipeline - -**Returns:** -- Dictionary containing: - - `output`: Pipeline output data - - `metadata`: Processing metadata - - `metrics`: Performance metrics - -**Raises:** -- `ProcessingError`: If pipeline execution fails - -##### `get_status() -> Dict[str, Any]` -Get system health and status. - -**Returns:** -- Dictionary containing: - - `state`: System state - - `health`: Health summary - - `modules`: Module status - - `plugins`: Plugin status - - `metrics`: System metrics - -##### `shutdown(graceful: bool = True) -> None` -Shutdown the framework. - -**Parameters:** -- `graceful` (bool): Whether to shutdown gracefully (default: True) - -**Code Example:** -```python -from semantica import Semantica - -# Initialize framework with configuration -framework = Semantica(config={ - "processing": {"batch_size": 32}, - "embedding_model": {"provider": "openai"} -}) - -# Initialize all components (auto-initializes if not done) -framework.initialize() - -# Build knowledge base from multiple sources -result = framework.build_knowledge_base( - sources=["doc1.pdf", "doc2.docx", "https://example.com/article"], - embeddings=True, # Generate embeddings - graph=True, # Build knowledge graph - normalize=True, # Normalize data - fail_fast=False # Continue on errors -) - -# Access results -knowledge_graph = result["knowledge_graph"] -embeddings = result["embeddings"] -statistics = result["statistics"] - -print(f"Processed {statistics['sources_processed']} sources") -print(f"Success rate: {statistics['success_rate']:.2%}") - -# Get system status -status = framework.get_status() -print(f"System state: {status['state']}") -print(f"Healthy components: {status['health']['healthy_components']}") - -# Shutdown gracefully -framework.shutdown(graceful=True) -``` - ---- - -### `semantica.core.plugin_registry` - -**What it does:** -This module provides comprehensive plugin management for the Semantica framework, including dynamic plugin discovery from file system, loading, dependency resolution, and lifecycle management. It supports automatic plugin discovery from directories, version management, and plugin isolation with error handling. - -**Key Features:** -- Dynamic plugin discovery from file system -- Plugin version management and compatibility checking -- Automatic dependency resolution and loading -- Plugin lifecycle management (load, unload, cleanup) -- Plugin isolation and error handling -- Plugin metadata and capability tracking - -#### Class: `PluginRegistry` - -Plugin registry and management system. - -**Methods:** - -##### `__init__(plugin_paths: Optional[List[Union[str, Path]]] = None)` -Initialize plugin registry. - -**Parameters:** -- `plugin_paths` (Optional[List[Union[str, Path]]]): List of directory paths to search for plugins - -##### `register_plugin(plugin_name: str, plugin_class: Type, version: str = "1.0.0", **metadata: Any) -> None` -Register a plugin. - -**Parameters:** -- `plugin_name` (str): Name of the plugin -- `plugin_class` (Type): Plugin class to register -- `version` (str): Plugin version (default: "1.0.0") -- `**metadata`: Additional plugin metadata: - - `description`: Plugin description - - `author`: Plugin author - - `dependencies`: List of dependency plugin names - - `capabilities`: List of plugin capabilities - -**Raises:** -- `ValidationError`: If plugin is invalid - -##### `load_plugin(plugin_name: str, **config: Any) -> Any` -Load and initialize a plugin. - -**Parameters:** -- `plugin_name` (str): Name of the plugin to load -- `**config`: Plugin configuration passed to plugin constructor - -**Returns:** -- Loaded and initialized plugin instance - -**Raises:** -- `ConfigurationError`: If plugin not found, dependencies missing, or initialization fails - -##### `unload_plugin(plugin_name: str) -> None` -Unload a plugin. - -**Parameters:** -- `plugin_name` (str): Name of plugin to unload - -**Raises:** -- `ConfigurationError`: If plugin not loaded - -##### `list_plugins() -> List[Dict[str, Any]]` -List all available plugins. - -**Returns:** -- List of plugin information dictionaries - -##### `get_plugin_info(plugin_name: str) -> Dict[str, Any]` -Get information about a plugin. - -**Parameters:** -- `plugin_name` (str): Name of plugin - -**Returns:** -- Dictionary with plugin information - -**Raises:** -- `ConfigurationError`: If plugin not found - -##### `is_plugin_loaded(plugin_name: str) -> bool` -Check if a plugin is loaded. - -**Parameters:** -- `plugin_name` (str): Name of plugin - -**Returns:** -- True if plugin is loaded, False otherwise - -##### `get_loaded_plugin(plugin_name: str) -> Optional[Any]` -Get loaded plugin instance. - -**Parameters:** -- `plugin_name` (str): Name of plugin - -**Returns:** -- Plugin instance or None if not loaded - -**Code Example:** -```python -from semantica.core import PluginRegistry - -# Initialize registry with plugin paths -registry = PluginRegistry(plugin_paths=["./plugins", "./custom_plugins"]) - -# Register a plugin manually -class MyPlugin: - def initialize(self): - print("Plugin initialized") - - def execute(self, data): - return f"Processed: {data}" - -registry.register_plugin( - "my_plugin", - MyPlugin, - version="1.0.0", - description="My custom plugin", - author="John Doe", - dependencies=["base_plugin"], - capabilities=["processing", "analysis"] -) - -# Load a plugin (dependencies are automatically loaded first) -plugin = registry.load_plugin("my_plugin", config={"key": "value"}) - -# Use the plugin -result = plugin.execute("test data") - -# List all available plugins -plugins = registry.list_plugins() -for plugin_info in plugins: - print(f"{plugin_info['name']} v{plugin_info['version']}: {plugin_info['description']}") - -# Get plugin information -info = registry.get_plugin_info("my_plugin") -print(f"Plugin loaded: {info['loaded']}") - -# Check if plugin is loaded -if registry.is_plugin_loaded("my_plugin"): - plugin_instance = registry.get_loaded_plugin("my_plugin") - -# Unload plugin -registry.unload_plugin("my_plugin") -``` - ---- - -## Ingestion Modules - -### `semantica.ingest.file_ingestor` - -**What it does:** -This module provides comprehensive file ingestion capabilities from local filesystems and cloud storage providers (AWS S3, Google Cloud Storage, Azure Blob). It automatically detects file types using multiple methods (extension, MIME type, magic numbers), validates file sizes, and supports batch processing with progress tracking. - -**Key Features:** -- Local file system scanning (recursive and filtered) -- Cloud storage integration (AWS S3, Google Cloud Storage, Azure Blob) -- Automatic file type detection (extension, MIME type, magic numbers) -- Batch processing with progress tracking -- File size validation and limits -- Support for all common document, image, audio, and video formats - -#### Class: `FileIngestor` - -File system and cloud storage ingestion handler. - -**Methods:** - -##### `__init__(config: Optional[Dict[str, Any]] = None, **kwargs)` -Initialize file ingestor. - -**Parameters:** -- `config` (Optional[Dict[str, Any]]): Ingestion configuration dictionary -- `**kwargs`: Additional configuration parameters - -##### `ingest_directory(directory_path: Union[str, Path], recursive: bool = True, **filters) -> List[FileObject]` -Ingest all files from a directory. - -**Parameters:** -- `directory_path` (Union[str, Path]): Path to directory -- `recursive` (bool): Whether to scan subdirectories (default: True) -- `**filters`: File filtering criteria - -**Returns:** -- List of ingested file objects - -##### `ingest_file(file_path: Union[str, Path], **options) -> FileObject` -Ingest a single file from the filesystem. - -**Parameters:** -- `file_path` (Union[str, Path]): Path to the file to ingest -- `**options`: Processing options: - - `read_content` (bool): Whether to read file content (default: True) - - Additional metadata to include in FileObject - -**Returns:** -- `FileObject`: Ingested file object with metadata and optional content - -**Raises:** -- `ValidationError`: If file doesn't exist, isn't a file, or exceeds size limits -- `ProcessingError`: If file cannot be read - -##### `ingest_cloud(provider: str, bucket: str, prefix: str = "", **config) -> List[FileObject]` -Ingest files from cloud storage. - -**Parameters:** -- `provider` (str): Cloud provider (s3, gcs, azure) -- `bucket` (str): Storage bucket name -- `prefix` (str): Object prefix filter (default: "") -- `**config`: Cloud provider configuration - -**Returns:** -- List of ingested file objects - -##### `scan_directory(directory_path: Union[str, Path], **filters) -> List[Dict[str, Any]]` -Scan directory and return file information without processing. - -**Parameters:** -- `directory_path` (Union[str, Path]): Path to directory -- `**filters`: File filtering criteria: - - `recursive` (bool): Whether to scan subdirectories (default: True) - - `extensions` (List[str]): List of allowed extensions - - `min_size` (int): Minimum file size - - `max_size` (int): Maximum file size - - `pattern` (str): Filename pattern (glob) - -**Returns:** -- List of file metadata - -##### `set_progress_callback(callback) -> None` -Set progress tracking callback. - -**Parameters:** -- `callback`: Callback function for progress tracking - -#### Class: `FileTypeDetector` - -File type detection and validation. - -**Methods:** - -##### `__init__()` -Initialize file type detector. - -##### `detect_type(file_path: Union[str, Path], content: Optional[bytes] = None) -> str` -Detect file type using multiple detection methods. - -**Parameters:** -- `file_path` (Union[str, Path]): Path to file -- `content` (Optional[bytes]): Optional file content bytes for magic number detection - -**Returns:** -- Detected file type (extension without dot, e.g., "pdf", "jpg") - Returns "unknown" if type cannot be determined - -##### `is_supported(file_type: str) -> bool` -Check if file type is supported. - -**Parameters:** -- `file_type` (str): File type to check - -**Returns:** -- Whether type is supported - -**Code Example:** -```python -from semantica.ingest import FileIngestor -from pathlib import Path - -# Initialize file ingestor -ingestor = FileIngestor() - -# Ingest a single file -file_obj = ingestor.ingest_file( - "document.pdf", - read_content=True # Read file content into memory -) - -print(f"File: {file_obj.name}") -print(f"Type: {file_obj.file_type}") -print(f"Size: {file_obj.size:,} bytes") -print(f"MIME: {file_obj.mime_type}") - -# Ingest entire directory (recursive) -file_objects = ingestor.ingest_directory( - "./documents", - recursive=True, # Scan subdirectories - extensions=[".pdf", ".docx", ".txt"], # Filter by extension - min_size=1024, # Minimum file size (1KB) - max_size=10485760 # Maximum file size (10MB) -) - -print(f"Ingested {len(file_objects)} files") - -# Scan directory without processing (faster) -file_info = ingestor.scan_directory( - "./documents", - recursive=True, - extensions=[".pdf", ".docx"] -) - -# Ingest from cloud storage (AWS S3) -cloud_files = ingestor.ingest_cloud( - provider="s3", - bucket="my-bucket", - prefix="documents/", - access_key_id="YOUR_KEY", - secret_access_key="YOUR_SECRET", - region="us-east-1" -) - -# Set progress callback -def progress_callback(current, total, file_obj): - print(f"Progress: {current}/{total} - {file_obj.name}") - -ingestor.set_progress_callback(progress_callback) - -# Detect file type -detector = FileTypeDetector() -file_type = detector.detect_type("document.pdf") -print(f"Detected type: {file_type}") -print(f"Supported: {detector.is_supported(file_type)}") -``` - ---- - -## Parsing Modules - -### `semantica.parse.document_parser` - -**What it does:** -This module handles parsing of various document formats including PDF, DOCX, HTML, and plain text files. It extracts text content, metadata, and document structure, handles embedded images and tables, supports batch document processing, and can handle password-protected documents. - -**Key Features:** -- PDF text and metadata extraction -- DOCX content parsing -- HTML content cleaning -- Plain text processing -- Document structure analysis -- Batch document processing -- Password-protected document handling -- Embedded image and table extraction - -#### Class: `DocumentParser` - -Document format parsing handler. - -**Methods:** - -##### `__init__(config=None, **kwargs)` -Initialize document parser. - -**Parameters:** -- `config`: Configuration dictionary -- `**kwargs`: Additional configuration options - -##### `parse_document(file_path: Union[str, Path], file_type: Optional[str] = None, **options) -> Dict[str, Any]` -Parse document of any supported format. - -**Parameters:** -- `file_path` (Union[str, Path]): Path to document file -- `file_type` (Optional[str]): Document type (auto-detected if None) -- `**options`: Parsing options - -**Returns:** -- Parsed document data dictionary - -##### `extract_text(file_path: Union[str, Path], **options) -> str` -Extract text content from document. - -**Parameters:** -- `file_path` (Union[str, Path]): Path to document file -- `**options`: Parsing options - -**Returns:** -- Extracted text content - -##### `extract_metadata(file_path: Union[str, Path]) -> Dict[str, Any]` -Extract document metadata and properties. - -**Parameters:** -- `file_path` (Union[str, Path]): Path to document file - -**Returns:** -- Document metadata dictionary - -##### `parse_batch(file_paths: List[Union[str, Path]], **options) -> Dict[str, Any]` -Parse multiple documents in batch. - -**Parameters:** -- `file_paths` (List[Union[str, Path]]): List of document file paths -- `**options`: Parsing options: - - `max_workers` (int): Maximum parallel workers - - `continue_on_error` (bool): Continue on errors (default: True) - -**Returns:** -- Batch processing results dictionary - -**Code Example:** -```python -from semantica.parse import DocumentParser - -# Initialize document parser -parser = DocumentParser() - -# Parse a document (auto-detects format) -result = parser.parse_document("document.pdf") -print(f"Text length: {len(result['text'])} characters") -print(f"Metadata: {result['metadata']}") - -# Extract just the text content -text = parser.extract_text("document.docx") -print(f"Extracted text: {text[:100]}...") - -# Extract metadata only -metadata = parser.extract_metadata("document.pdf") -print(f"Title: {metadata.get('title')}") -print(f"Author: {metadata.get('author')}") -print(f"Pages: {metadata.get('page_count')}") - -# Parse multiple documents in batch -results = parser.parse_batch( - ["doc1.pdf", "doc2.docx", "doc3.html"], - continue_on_error=True, # Continue if one fails - max_workers=4 # Parallel processing -) - -print(f"Successful: {results['success_count']}") -print(f"Failed: {results['failure_count']}") - -# Access parsed documents -for item in results["successful"]: - print(f"File: {item['file_path']}") - print(f"Text length: {len(item['result']['text'])}") -``` - ---- - -## Knowledge Graph Modules - -### `semantica.kg.graph_builder` - -**What it does:** -This module provides comprehensive knowledge graph construction capabilities from extracted entities and relationships. It supports temporal knowledge graphs with time-aware edges, entity resolution and deduplication, conflict detection and resolution, temporal snapshots and versioning, and Neo4j integration for graph storage. - -**Key Features:** -- Build knowledge graphs from entities and relationships -- Temporal knowledge graph support with time-aware edges -- Entity resolution and deduplication -- Conflict detection and resolution -- Temporal snapshots and versioning -- Neo4j integration for graph storage - -#### Class: `GraphBuilder` - -Knowledge graph builder with temporal support. - -**Methods:** - -##### `__init__(merge_entities=True, entity_resolution_strategy="fuzzy", resolve_conflicts=True, enable_temporal=False, temporal_granularity="day", track_history=False, version_snapshots=False, **kwargs)` -Initialize graph builder. - -**Parameters:** -- `merge_entities` (bool): Whether to merge duplicate entities (default: True) -- `entity_resolution_strategy` (str): Strategy for entity resolution ("fuzzy", "exact", "ml-based") (default: "fuzzy") -- `resolve_conflicts` (bool): Whether to resolve conflicts (default: True) -- `enable_temporal` (bool): Enable temporal knowledge graph features (default: False) -- `temporal_granularity` (str): Time granularity ("second", "minute", "hour", "day", "week", "month", "year") (default: "day") -- `track_history` (bool): Track historical changes (default: False) -- `version_snapshots` (bool): Create version snapshots at intervals (default: False) -- `**kwargs`: Additional configuration options - -##### `build(sources: Union[List[Any], Any], entity_resolver: Optional[Any] = None, **options) -> Dict[str, Any]` -Build knowledge graph from sources. - -**Parameters:** -- `sources` (Union[List[Any], Any]): List of sources in various formats: - - Dict with "entities" and/or "relationships" keys - - Dict with entity-like structure (has "id" or "entity_id") - - Dict with relationship structure (has "source" and "target") - - List of entity/relationship dicts -- `entity_resolver` (Optional[Any]): Optional custom entity resolver (overrides default) -- `**options`: Additional build options - -**Returns:** -- Dictionary containing: - - `entities`: List of resolved entities - - `relationships`: List of relationships - - `metadata`: Graph metadata including counts and timestamps - -##### `add_temporal_edge(graph, source, target, relationship, valid_from=None, valid_until=None, temporal_metadata=None, **kwargs)` -Add edge with temporal validity information. - -**Parameters:** -- `graph`: Knowledge graph to add edge to -- `source`: Source entity/node -- `target`: Target entity/node -- `relationship`: Relationship type -- `valid_from`: Start time for relationship validity (datetime, timestamp, or ISO string) -- `valid_until`: End time for relationship validity (None for ongoing) -- `temporal_metadata`: Additional temporal metadata (timezone, precision, etc.) -- `**kwargs`: Additional edge properties - -**Returns:** -- Edge object with temporal annotations - -##### `create_temporal_snapshot(graph, timestamp=None, snapshot_name=None, **options)` -Create temporal snapshot of graph at specific time point. - -**Parameters:** -- `graph`: Knowledge graph to snapshot -- `timestamp`: Time point for snapshot (None for current time) -- `snapshot_name`: Optional name for snapshot -- `**options`: Additional snapshot options - -**Returns:** -- Temporal snapshot object - -##### `query_temporal(graph, query, at_time=None, time_range=None, temporal_window=None, **options)` -Query graph at specific time point or time range. - -**Parameters:** -- `graph`: Knowledge graph to query -- `query`: Query (Cypher, SPARQL, or natural language) -- `at_time`: Query at specific time point -- `time_range`: Query within time range (start, end) -- `temporal_window`: Temporal window size -- `**options`: Additional query options - -**Returns:** -- Query results with temporal context - -##### `load_from_neo4j(uri="bolt://localhost:7687", username="neo4j", password="password", database="neo4j", enable_temporal=False, temporal_property="valid_time", **kwargs)` -Load graph from Neo4j database. - -**Parameters:** -- `uri` (str): Neo4j connection URI (default: "bolt://localhost:7687") -- `username` (str): Neo4j username (default: "neo4j") -- `password` (str): Neo4j password (default: "password") -- `database` (str): Neo4j database name (default: "neo4j") -- `enable_temporal` (bool): Enable temporal features for loaded graph (default: False) -- `temporal_property` (str): Property name for temporal data (default: "valid_time") -- `**kwargs`: Additional connection options - -**Returns:** -- Knowledge graph loaded from Neo4j - -**Different Approaches and Strategies:** - -The knowledge graph builder provides 8 different approaches for building knowledge graphs: - -1. **Entity Resolution Strategies** - Three methods: fuzzy matching, exact matching, semantic similarity -2. **Temporal Graph Approaches** - Time-aware graphs with different granularities (second, minute, hour, day, week, month, year) -3. **Conflict Resolution Methods** - Automatic conflict detection and resolution (voting, credibility-weighted, recency-based) -4. **Graph Building Modes** - Incremental vs batch building, with or without entity merging -5. **Temporal Snapshot Strategies** - Version snapshots, history tracking, time-point queries -6. **Source Format Handling** - Multiple input formats (entities/relationships dicts, entity lists, relationship lists) -7. **Neo4j Integration Approaches** - Load from Neo4j, enable temporal features, custom temporal properties -8. **Graph Query Methods** - Temporal queries, time-range queries, Cypher/SPARQL queries - -**When to Use Each Approach:** - -| Approach | Use Case | Example | -|----------|----------|---------| -| Fuzzy Entity Resolution | Handling name variations | "Apple Inc." vs "Apple" vs "Apple Corporation" | -| Exact Entity Resolution | High precision requirements | Exact ID matching, no variations allowed | -| Semantic Entity Resolution | Context-aware matching | Using embeddings for similarity | -| Temporal Graphs | Time-sensitive data | Employee relationships, contract validity periods | -| Conflict Resolution | Multiple data sources | Resolving conflicting entity properties | -| Incremental Building | Large datasets | Add entities/relationships over time | -| Batch Building | Small datasets | Build entire graph at once | -| Temporal Snapshots | Version control | Track graph state at different times | - -**Detailed Examples for Each Approach:** - -**Approach 1: Entity Resolution Strategies** -```python -from semantica.kg import GraphBuilder - -# Fuzzy matching (default) - handles name variations -builder_fuzzy = GraphBuilder( - merge_entities=True, - entity_resolution_strategy="fuzzy", # Levenshtein, Jaro-Winkler - similarity_threshold=0.8 -) - -# Exact matching - only exact string matches -builder_exact = GraphBuilder( - merge_entities=True, - entity_resolution_strategy="exact" # Exact string comparison -) - -# Semantic matching - uses embeddings -builder_semantic = GraphBuilder( - merge_entities=True, - entity_resolution_strategy="semantic", # Embedding-based similarity - similarity_threshold=0.85 -) -``` - -**Approach 2: Temporal Graph Approaches** -```python -# Different temporal granularities -builder_second = GraphBuilder( - enable_temporal=True, - temporal_granularity="second" # Second-level precision -) - -builder_day = GraphBuilder( - enable_temporal=True, - temporal_granularity="day" # Day-level precision (default) -) - -builder_month = GraphBuilder( - enable_temporal=True, - temporal_granularity="month" # Month-level precision -) - -# Add temporal edge with validity period -builder.add_temporal_edge( - graph, - source="e1", - target="e3", - relationship="works_at", - valid_from="2020-01-01", - valid_until="2023-12-31" -) -``` - -**Approach 3: Conflict Resolution Methods** -```python -# Automatic conflict resolution -builder = GraphBuilder( - resolve_conflicts=True, # Automatically resolve conflicts - conflict_resolution_strategy="voting" # or "credibility", "recency", "confidence" -) - -# Manual conflict resolution -builder = GraphBuilder( - resolve_conflicts=False # Detect but don't auto-resolve -) -``` - -**Approach 4: Graph Building Modes** -```python -# Incremental building (add entities over time) -builder = GraphBuilder(merge_entities=True) -graph1 = builder.build([{"entities": [e1, e2]}]) -graph2 = builder.build([{"entities": [e3, e4]}], existing_graph=graph1) - -# Batch building (build entire graph at once) -sources = [ - {"entities": [e1, e2, e3], "relationships": [r1, r2]}, - {"entities": [e4, e5], "relationships": [r3]} -] -graph = builder.build(sources) -``` - -**Approach 5: Temporal Snapshot Strategies** -```python -# Create snapshots at specific times -snapshot_2022 = builder.create_temporal_snapshot( - graph, - timestamp="2022-06-15", - snapshot_name="mid_2022" -) - -# Track history -builder = GraphBuilder( - track_history=True, # Track all changes - version_snapshots=True # Create version snapshots -) - -# Query at specific time point -results = builder.query_temporal( - graph, - query="MATCH (p:Person)-[:works_at]->(o:Organization) RETURN p, o", - at_time="2022-06-15" -) -``` - -**Approach 6: Source Format Handling** -```python -# Format 1: Entities and relationships dict -source1 = { - "entities": [{"id": "e1", "name": "Alice"}], - "relationships": [{"source": "e1", "target": "e2", "type": "knows"}] -} - -# Format 2: Entity list -source2 = [{"id": "e1", "name": "Alice"}, {"id": "e2", "name": "Bob"}] - -# Format 3: Relationship list -source3 = [{"source": "e1", "target": "e2", "type": "knows"}] - -# All formats work -graph = builder.build([source1, source2, source3]) -``` - -**Approach 7: Neo4j Integration** -```python -# Load from Neo4j -graph = builder.load_from_neo4j( - uri="bolt://localhost:7687", - username="neo4j", - password="password", - enable_temporal=True, # Enable temporal features - temporal_property="valid_time" # Custom temporal property name -) -``` - -**Approach 8: Graph Query Methods** -```python -# Temporal query at specific time -results = builder.query_temporal( - graph, - query="MATCH (p:Person) RETURN p", - at_time="2022-06-15" -) - -# Time range query -results = builder.query_temporal( - graph, - query="MATCH (p:Person)-[:works_at]->(o:Organization) RETURN p, o", - time_range=("2020-01-01", "2023-12-31") -) - -# Cypher query -results = builder.query_temporal( - graph, - query="MATCH (p:Person)-[:knows*2]->(f:Person) RETURN p, f", - at_time="2022-06-15" -) -``` - -**Code Example:** -```python -from semantica.kg import GraphBuilder - -# Initialize graph builder with entity resolution and conflict detection -builder = GraphBuilder( - merge_entities=True, # Merge duplicate entities - entity_resolution_strategy="fuzzy", # Use fuzzy matching - resolve_conflicts=True, # Automatically resolve conflicts - enable_temporal=True, # Enable temporal features - temporal_granularity="day", # Time granularity - track_history=True # Track changes over time -) - -# Build knowledge graph from sources -sources = [ - { - "entities": [ - {"id": "e1", "name": "Alice", "type": "Person"}, - {"id": "e2", "name": "Bob", "type": "Person"}, - {"id": "e3", "name": "Company X", "type": "Organization"} - ], - "relationships": [ - {"source": "e1", "target": "e2", "type": "knows"}, - {"source": "e1", "target": "e3", "type": "works_at"} - ] - } -] - -graph = builder.build(sources) -print(f"Entities: {len(graph['entities'])}") -print(f"Relationships: {len(graph['relationships'])}") - -# Add temporal edge (relationship valid for specific time period) -builder.add_temporal_edge( - graph, - source="e1", - target="e3", - relationship="works_at", - valid_from="2020-01-01", - valid_until="2023-12-31", - temporal_metadata={"timezone": "UTC"} -) - -# Create temporal snapshot (graph state at specific time) -snapshot = builder.create_temporal_snapshot( - graph, - timestamp="2022-06-15", - snapshot_name="mid_2022" -) - -# Query graph at specific time point -results = builder.query_temporal( - graph, - query="MATCH (p:Person)-[:works_at]->(o:Organization) RETURN p, o", - at_time="2022-06-15" -) - -# Load graph from Neo4j -neo4j_graph = builder.load_from_neo4j( - uri="bolt://localhost:7687", - username="neo4j", - password="password", - database="neo4j", - enable_temporal=True -) -``` - ---- - -## Embeddings Modules - -### `semantica.embeddings.embedding_generator` - -**What it does:** -This module provides comprehensive embedding generation capabilities for text, images, audio, and multi-modal content. It supports multiple embedding models (sentence-transformers, OpenAI, BGE, CLIP), batch processing for efficiency, embedding optimization and compression, and similarity comparison utilities. - -**Key Features:** -- Text embedding generation (multiple models: sentence-transformers, OpenAI, BGE, etc.) -- Image embedding generation -- Audio embedding generation -- Multi-modal embedding support -- Batch processing for efficiency -- Embedding optimization and compression -- Similarity comparison utilities - -#### Class: `EmbeddingGenerator` - -Main embedding generation handler. - -**Methods:** - -##### `__init__(config: Optional[Dict[str, Any]] = None, **kwargs)` -Initialize embedding generator. - -**Parameters:** -- `config` (Optional[Dict[str, Any]]): Configuration dictionary with keys: - - `text`: Text embedder configuration - - `image`: Image embedder configuration - - `audio`: Audio embedder configuration - - `multimodal`: Multi-modal embedder configuration - - `optimizer`: Embedding optimizer configuration -- `**kwargs`: Additional configuration (merged into config) - -##### `generate_embeddings(data: Union[str, Path, List[Union[str, Path]]], data_type: Optional[str] = None, **options) -> np.ndarray` -Generate embeddings for input data. - -**Parameters:** -- `data` (Union[str, Path, List[Union[str, Path]]]): Input data to embed: - - str: Text string or file path - - Path: File path object - - List: Batch of texts or file paths -- `data_type` (Optional[str]): Explicit data type ("text", "image", "audio"). If None, auto-detects from input -- `**options`: Additional generation options passed to embedder - -**Returns:** -- `np.ndarray`: Generated embeddings - - For single input: 1D array - - For batch input: 2D array (batch_size, embedding_dim) - -**Raises:** -- `ProcessingError`: If data type is unsupported or embedding fails - -##### `optimize_embeddings(embeddings: np.ndarray, **options) -> np.ndarray` -Optimize embedding quality and performance. - -**Parameters:** -- `embeddings` (np.ndarray): Input embeddings -- `**options`: Optimization options - -**Returns:** -- `np.ndarray`: Optimized embeddings - -##### `compare_embeddings(embedding1: np.ndarray, embedding2: np.ndarray, **options) -> float` -Compare embeddings for similarity. - -**Parameters:** -- `embedding1` (np.ndarray): First embedding -- `embedding2` (np.ndarray): Second embedding -- `**options`: Comparison options: - - `method` (str): Similarity method ("cosine", "euclidean") (default: "cosine") - -**Returns:** -- `float`: Similarity score (0-1) - -##### `process_batch(data_items: List[Union[str, Path]], **options) -> Dict[str, Any]` -Process multiple data items for embedding generation. - -**Parameters:** -- `data_items` (List[Union[str, Path]]): List of data items -- `**options`: Processing options - -**Returns:** -- Dictionary with batch processing results: - - `embeddings`: List of generated embeddings - - `successful`: List of successfully processed items - - `failed`: List of failed items with error information - - `total`: Total number of items - - `success_count`: Number of successful items - - `failure_count`: Number of failed items - -**Different Approaches and Strategies:** - -The embeddings module supports multiple embedding providers and models, each optimized for different use cases: - -1. **Sentence-Transformers** - Open-source, high-quality sentence embeddings (recommended for most use cases) -2. **OpenAI Embeddings** - Cloud-based, high-quality embeddings via API -3. **BGE (BAAI General Embedding)** - State-of-the-art multilingual embeddings -4. **CLIP** - Multi-modal embeddings for images and text -5. **Custom Models** - Support for custom embedding models - -**When to Use Each Provider:** - -- **Sentence-Transformers**: Local processing, no API costs, good quality, open-source -- **OpenAI**: Highest quality, cloud-based, requires API key, paid service -- **BGE**: Multilingual support, best for non-English text, open-source -- **CLIP**: Image-text similarity, multi-modal applications -- **Custom Models**: Domain-specific embeddings, fine-tuned models - -**Comparison of Embedding Providers:** - -| Provider | Quality | Speed | Cost | Multilingual | Best For | -|----------|---------|-------|------|--------------|----------| -| Sentence-Transformers | High | Fast | Free | Limited | General purpose | -| OpenAI | Very High | Medium | Paid | Yes | Production, quality-critical | -| BGE | Very High | Fast | Free | Yes | Multilingual applications | -| CLIP | High | Medium | Free | Limited | Image-text tasks | - -**Code Examples for Different Approaches:** - -**Approach 1: Sentence-Transformers (Open-source, Recommended)** -```python -from semantica.embeddings import EmbeddingGenerator - -# Initialize with sentence-transformers (default, no API key needed) -generator = EmbeddingGenerator( - config={ - "text": { - "model": "sentence-transformers/all-MiniLM-L6-v2", # Fast, 384-dim - # Alternative models: - # "all-mpnet-base-v2" - Higher quality, 768-dim - # "paraphrase-multilingual-MiniLM-L12-v2" - Multilingual - "device": "cpu", # or "cuda" for GPU - "normalize": True - } - } -) - -# Generate single embedding -text = "The quick brown fox jumps over the lazy dog" -embedding = generator.generate_embeddings(text, data_type="text") -print(f"Embedding shape: {embedding.shape}") # (384,) - -# Batch processing (more efficient) -texts = ["Document 1", "Document 2", "Document 3"] -embeddings = generator.generate_embeddings(texts, data_type="text") -print(f"Batch embeddings shape: {embeddings.shape}") # (3, 384) -``` - -**Approach 2: OpenAI Embeddings (Cloud-based, High Quality)** -```python -from semantica.embeddings import EmbeddingGenerator -import os - -# Set OpenAI API key (or use environment variable) -os.environ["OPENAI_API_KEY"] = "your-api-key-here" - -# Initialize with OpenAI adapter -generator = EmbeddingGenerator( - config={ - "text": { - "provider": "openai", - "model": "text-embedding-3-small", # or "text-embedding-3-large" - # text-embedding-3-small: 1536 dimensions, fast, cost-effective - # text-embedding-3-large: 3072 dimensions, highest quality - } - } -) - -# Generate embeddings (same API as sentence-transformers) -embedding = generator.generate_embeddings( - "Your text here", - data_type="text" -) -print(f"OpenAI embedding shape: {embedding.shape}") # (1536,) or (3072,) - -# Batch processing with OpenAI (handles rate limits automatically) -texts = ["Text 1", "Text 2", "Text 3"] -embeddings = generator.generate_embeddings(texts, data_type="text") -``` - -**Approach 3: BGE Embeddings (Multilingual, High Quality)** -```python -from semantica.embeddings import EmbeddingGenerator - -# Initialize with BGE model (excellent for multilingual) -generator = EmbeddingGenerator( - config={ - "text": { - "provider": "bge", - "model_name": "BAAI/bge-small-en-v1.5", # English - # Alternative: "BAAI/bge-m3" - Multilingual, 1024-dim - # Alternative: "BAAI/bge-large-en-v1.5" - Higher quality, 1024-dim - } - } -) - -# Generate embeddings for English text -english_text = "Hello, world!" -embedding = generator.generate_embeddings(english_text, data_type="text") - -# BGE models work well with multiple languages -multilingual_texts = [ - "Hello, world!", # English - "Bonjour le monde!", # French - "Hola, mundo!", # Spanish - "你好，世界！" # Chinese -] -embeddings = generator.generate_embeddings(multilingual_texts, data_type="text") -``` - -**Approach 4: Using Provider Adapters Directly** -```python -from semantica.embeddings import ProviderAdapterFactory - -# Create provider adapter directly (more control) -openai_adapter = ProviderAdapterFactory.create( - "openai", - api_key="your-key", - model="text-embedding-3-small" -) - -# Use adapter directly -embedding = openai_adapter.embed("Your text") -batch_embeddings = openai_adapter.embed_batch(["Text 1", "Text 2"]) - -# Switch providers easily -bge_adapter = ProviderAdapterFactory.create( - "bge", - model_name="BAAI/bge-small-en-v1.5" -) -bge_embedding = bge_adapter.embed("Your text") -``` - -**Approach 5: Multi-modal Embeddings (Text + Images)** -```python -from semantica.embeddings import EmbeddingGenerator - -# Initialize with CLIP for multi-modal embeddings -generator = EmbeddingGenerator( - config={ - "text": {"model": "sentence-transformers/all-MiniLM-L6-v2"}, - "image": {"model": "clip-vit-base-patch32"}, # CLIP model - "multimodal": { - "model": "clip-vit-base-patch32" # For image-text similarity - } - } -) - -# Generate text embedding -text_embedding = generator.generate_embeddings( - "A photo of a cat", - data_type="text" -) - -# Generate image embedding -image_embedding = generator.generate_embeddings( - "cat_photo.jpg", - data_type="image" -) - -# Compare text and image embeddings (CLIP enables cross-modal similarity) -similarity = generator.compare_embeddings( - text_embedding, - image_embedding, - method="cosine" -) -print(f"Text-Image similarity: {similarity:.4f}") -``` - -**Approach 6: Embedding Optimization and Compression** -```python -from semantica.embeddings import EmbeddingGenerator -import numpy as np - -generator = EmbeddingGenerator() - -# Generate embeddings -texts = ["Document 1", "Document 2", "Document 3"] -embeddings = generator.generate_embeddings(texts, data_type="text") -print(f"Original shape: {embeddings.shape}") # (3, 384) - -# Optimize embeddings (reduce dimensionality, improve quality) -optimized = generator.optimize_embeddings( - embeddings, - method="pca", # Principal Component Analysis - target_dim=256 # Reduce from 384 to 256 dimensions -) -print(f"Optimized shape: {optimized.shape}") # (3, 256) - -# Compare original vs optimized (should maintain similarity structure) -original_sim = generator.compare_embeddings(embeddings[0], embeddings[1]) -optimized_sim = generator.compare_embeddings(optimized[0], optimized[1]) -print(f"Original similarity: {original_sim:.4f}") -print(f"Optimized similarity: {optimized_sim:.4f}") # Should be similar -``` - -**Approach 7: Batch Processing with Error Handling** -```python -from semantica.embeddings import EmbeddingGenerator - -generator = EmbeddingGenerator() - -# Process large batch with error handling -file_paths = ["doc1.txt", "doc2.txt", "doc3.txt", "invalid_file.txt"] - -results = generator.process_batch( - file_paths, - data_type="text", - continue_on_error=True, # Continue even if some fail - batch_size=32 # Process in batches of 32 -) - -print(f"Total: {results['total']}") -print(f"Successful: {results['success_count']}") -print(f"Failed: {results['failure_count']}") - -# Access successful embeddings -for i, embedding in enumerate(results['embeddings']): - if embedding is not None: - print(f"Document {i}: {embedding.shape}") - -# Check failed items -for failed_item in results['failed']: - print(f"Failed: {failed_item['item']} - {failed_item['error']}") -``` - -**Approach 8: Similarity Search and Comparison** -```python -from semantica.embeddings import EmbeddingGenerator -import numpy as np - -generator = EmbeddingGenerator() - -# Generate embeddings for query and documents -query = "machine learning algorithms" -documents = [ - "Deep learning neural networks", - "Statistical analysis methods", - "Computer vision applications", - "Natural language processing" -] - -query_embedding = generator.generate_embeddings(query, data_type="text") -doc_embeddings = generator.generate_embeddings(documents, data_type="text") - -# Find most similar document -similarities = [] -for doc_emb in doc_embeddings: - sim = generator.compare_embeddings( - query_embedding, - doc_emb, - method="cosine" # or "euclidean" - ) - similarities.append(sim) - -# Get top-k most similar -top_k = 2 -top_indices = np.argsort(similarities)[-top_k:][::-1] - -print(f"Query: {query}") -for i, idx in enumerate(top_indices): - print(f"{i+1}. {documents[idx]} (similarity: {similarities[idx]:.4f})") -``` - -**Best Practices:** - -1. **Choose provider based on requirements**: - - Local/offline → Sentence-Transformers or BGE - - Highest quality → OpenAI - - Multilingual → BGE or OpenAI - - Image-text → CLIP - -2. **Use batch processing**: - - Always use batch processing for multiple items - - Set appropriate batch_size based on memory - - Process in batches for large datasets - -3. **Normalize embeddings**: - - Always normalize embeddings for cosine similarity - - Use unit vectors for better similarity calculations - -4. **Handle errors gracefully**: - - Use `process_batch` with `continue_on_error=True` - - Log failed items for debugging - - Retry failed items with exponential backoff - -5. **Optimize for your use case**: - - Use smaller models for speed (all-MiniLM-L6-v2) - - Use larger models for quality (all-mpnet-base-v2) - - Consider dimensionality reduction for storage - ---- - -## Pipeline Modules - -### `semantica.pipeline.pipeline_builder` - -**What it does:** -This module handles construction and configuration of processing pipelines, providing a fluent DSL for building workflows, step chaining, validation, and serialization. It supports complex pipeline topologies, dependency management, step status tracking, error handling and recovery, and pipeline versioning. - -**Key Features:** -- Pipeline construction DSL -- Step configuration and chaining -- Pipeline validation and optimization -- Error handling and recovery -- Pipeline serialization and deserialization -- Dependency management -- Step status tracking - -#### Class: `PipelineBuilder` - -Pipeline construction and configuration handler. - -**Methods:** - -##### `__init__(config=None, **kwargs)` -Initialize pipeline builder. - -**Parameters:** -- `config`: Configuration dictionary -- `**kwargs`: Additional configuration options - -##### `add_step(step_name: str, step_type: str, **config) -> "PipelineBuilder"` -Add step to pipeline. - -**Parameters:** -- `step_name` (str): Step name/identifier -- `step_type` (str): Step type/category -- `**config`: Step configuration - -**Returns:** -- Self for method chaining - -##### `connect_steps(from_step: str, to_step: str, **options) -> "PipelineBuilder"` -Connect pipeline steps. - -**Parameters:** -- `from_step` (str): Source step name -- `to_step` (str): Target step name -- `**options`: Connection options - -**Returns:** -- Self for method chaining - -##### `set_parallelism(level: int) -> "PipelineBuilder"` -Set parallelism level. - -**Parameters:** -- `level` (int): Parallelism level (number of parallel workers) - -**Returns:** -- Self for method chaining - -##### `build(name: str = "default_pipeline") -> Pipeline` -Build pipeline from configuration. - -**Parameters:** -- `name` (str): Pipeline name (default: "default_pipeline") - -**Returns:** -- Built pipeline - -##### `build_pipeline(pipeline_config: Dict[str, Any], **options) -> Pipeline` -Build pipeline from configuration dictionary. - -**Parameters:** -- `pipeline_config` (Dict[str, Any]): Pipeline configuration -- `**options`: Additional options - -**Returns:** -- Built pipeline - -##### `register_step_handler(step_type: str, handler: Callable) -> None` -Register step handler function. - -**Parameters:** -- `step_type` (str): Step type -- `handler` (Callable): Handler function - -##### `get_step(step_name: str) -> Optional[PipelineStep]` -Get step by name. - -**Parameters:** -- `step_name` (str): Step name - -**Returns:** -- PipelineStep or None if not found - -##### `serialize(format: str = "json") -> Union[str, Dict[str, Any]]` -Serialize pipeline configuration. - -**Parameters:** -- `format` (str): Serialization format (default: "json") - -**Returns:** -- Serialized pipeline (string or dictionary) - -##### `validate_pipeline() -> Dict[str, Any]` -Validate pipeline structure and configuration. - -**Returns:** -- Validation results dictionary - -#### Class: `PipelineSerializer` - -Pipeline serialization handler. - -**Methods:** - -##### `__init__(**config)` -Initialize pipeline serializer. - -**Parameters:** -- `**config`: Configuration options - -##### `serialize_pipeline(pipeline: Pipeline, format: str = "json", **options) -> Union[str, Dict[str, Any]]` -Serialize pipeline to specified format. - -**Parameters:** -- `pipeline` (Pipeline): Pipeline object -- `format` (str): Serialization format (default: "json") -- `**options`: Additional options - -**Returns:** -- Serialized pipeline (string or dictionary) - -##### `deserialize_pipeline(serialized_pipeline: Union[str, Dict[str, Any]], **options) -> Pipeline` -Deserialize pipeline from serialized format. - -**Parameters:** -- `serialized_pipeline` (Union[str, Dict[str, Any]]): Serialized pipeline data -- `**options`: Additional options - -**Returns:** -- Reconstructed pipeline - -##### `version_pipeline(pipeline: Pipeline, version_info: Dict[str, Any]) -> Pipeline` -Add versioning information to pipeline. - -**Parameters:** -- `pipeline` (Pipeline): Pipeline object -- `version_info` (Dict[str, Any]): Version information - -**Returns:** -- Versioned pipeline - -**Code Example:** -```python -from semantica.pipeline import PipelineSerializer, Pipeline - -# Initialize serializer -serializer = PipelineSerializer() - -# Serialize pipeline to JSON -json_str = serializer.serialize_pipeline( - pipeline, - format="json" -) - -# Deserialize pipeline from JSON -restored_pipeline = serializer.deserialize_pipeline(json_str) - -# Add versioning information -versioned = serializer.version_pipeline( - pipeline, - version_info={ - "version": "1.2.0", - "author": "John Doe", - "date": "2024-01-15" - } -) -``` - ---- - -## Semantic Extraction Modules - -### `semantica.semantic_extract.ner_extractor` - -**What it does:** -This module provides core Named Entity Recognition (NER) capabilities using spaCy and transformers for entity identification and classification, with fallback pattern-based extraction. It supports multiple entity types, confidence scoring, batch processing, and entity filtering by confidence. - -**Key Features:** -- spaCy-based entity extraction -- Pattern-based fallback extraction -- Multiple entity type support -- Confidence scoring -- Batch processing -- Entity filtering by confidence - -#### Class: `NERExtractor` - -Named Entity Recognition extractor using spaCy and pattern-based fallback. - -**Methods:** - -##### `__init__(**config)` -Initialize NER extractor. - -**Parameters:** -- `**config`: Configuration options: - - `model` (str): Model name (default: "en_core_web_sm") - - `language` (str): Language code (default: "en") - - `min_confidence` (float): Minimum confidence threshold (default: 0.5) - -##### `extract_entities(text: str, **options) -> List[Entity]` -Extract named entities from text. - -**Parameters:** -- `text` (str): Input text -- `**options`: Extraction options: - - `entity_types` (List[str]): Filter by entity types - - `min_confidence` (float): Minimum confidence threshold - -**Returns:** -- List of extracted Entity objects - -##### `extract_entities_batch(texts: List[str], **options) -> List[List[Entity]]` -Extract entities from multiple texts. - -**Parameters:** -- `texts` (List[str]): List of input texts -- `**options`: Extraction options - -**Returns:** -- List of entity lists for each text - -##### `classify_entities(entities: List[Entity]) -> Dict[str, List[Entity]]` -Classify entities by type. - -**Parameters:** -- `entities` (List[Entity]): List of entities - -**Returns:** -- Dictionary with entities grouped by type - -##### `filter_by_confidence(entities: List[Entity], min_confidence: float) -> List[Entity]` -Filter entities by confidence score. - -**Parameters:** -- `entities` (List[Entity]): List of entities -- `min_confidence` (float): Minimum confidence threshold - -**Returns:** -- Filtered entities - -#### Class: `Entity` - -Entity representation dataclass. - -**Attributes:** -- `text` (str): Entity text -- `label` (str): Entity label/type -- `start_char` (int): Start character position -- `end_char` (int): End character position -- `confidence` (float): Confidence score (default: 1.0) -- `metadata` (Dict[str, Any]): Additional metadata - -**Different Entity Recognition Methods and Strategies:** - -The NER module supports multiple extraction methods, from exact keyword matching to advanced similarity-based approaches: - -1. **spaCy-based NER** - Machine learning model for entity recognition (default, highest quality) -2. **Pattern-based Fallback** - Regex patterns for exact matching when spaCy unavailable -3. **Keyword Exact Matching** - Simple exact string matching -4. **Confidence-based Filtering** - Filter entities by confidence scores -5. **Type-based Filtering** - Filter by entity types (PERSON, ORG, GPE, etc.) -6. **Batch Processing** - Process multiple texts efficiently - -**When to Use Each Method:** - -- **spaCy NER**: Production use, high accuracy needed, multiple entity types -- **Pattern-based**: Fallback when spaCy unavailable, specific entity patterns known -- **Keyword Matching**: Simple use cases, known entity lists, fast processing -- **Confidence Filtering**: Quality control, reduce false positives -- **Type Filtering**: Focus on specific entity categories - -**Comparison of NER Methods:** - -| Method | Accuracy | Speed | Requires Model | Best For | -|--------|----------|-------|----------------|----------| -| spaCy NER | Very High | Medium | Yes | Production, general purpose | -| Pattern-based | Medium | Fast | No | Fallback, known patterns | -| Keyword Exact | Low | Very Fast | No | Simple lists, fast lookup | - -**Code Examples for All NER Methods:** - -**Method 1: spaCy-based NER (Machine Learning, Recommended)** -```python -from semantica.semantic_extract import NERExtractor - -# Initialize with spaCy model (highest quality) -extractor = NERExtractor( - model="en_core_web_sm", # spaCy English model - # Alternative models: - # "en_core_web_md" - Medium model, better accuracy - # "en_core_web_lg" - Large model, best accuracy - # "en_core_web_trf" - Transformer model, highest accuracy - language="en", - min_confidence=0.7 -) - -# Extract entities (uses spaCy's ML model) -text = "Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976." -entities = extractor.extract_entities(text) - -# spaCy automatically detects: -# - PERSON: "Steve Jobs" -# - ORG: "Apple Inc." -# - GPE: "Cupertino", "California" -# - DATE: "1976" - -for entity in entities: - print(f"{entity.text} ({entity.label}): {entity.confidence:.2f}") -``` - -**Method 2: Pattern-based Fallback (Regex Patterns, No Model Required)** -```python -from semantica.semantic_extract import NERExtractor - -# Initialize without spaCy (uses pattern-based fallback) -extractor = NERExtractor( - model=None, # No spaCy model - language="en" -) - -# Pattern-based extraction uses regex patterns: -# - PERSON: Capitalized names (e.g., "Steve Jobs") -# - ORG: Company names with suffixes (e.g., "Apple Inc.") -# - GPE: Location names (e.g., "New York City") -# - DATE: Date patterns (e.g., "1976", "01/01/2024") - -text = "Apple Inc. was founded by Steve Jobs in 1976." -entities = extractor.extract_entities(text) # Uses _extract_fallback() - -# Pattern-based has lower confidence (0.7 default) -for entity in entities: - print(f"{entity.text} ({entity.label}): {entity.confidence:.2f}") - print(f" Method: {entity.metadata.get('extraction_method', 'unknown')}") -``` - -**Method 3: Keyword Exact Matching (Custom Entity Lists)** -```python -from semantica.semantic_extract import NERExtractor - -# Create custom keyword list for exact matching -known_entities = { - "PERSON": ["Steve Jobs", "Tim Cook", "Bill Gates"], - "ORG": ["Apple Inc.", "Microsoft", "Google"], - "GPE": ["Cupertino", "Redmond", "Mountain View"] -} - -# Extract using exact matching -text = "Steve Jobs founded Apple Inc. in Cupertino." -extracted = [] - -for entity_type, keywords in known_entities.items(): - for keyword in keywords: - if keyword.lower() in text.lower(): - # Find position - start = text.lower().find(keyword.lower()) - if start >= 0: - extracted.append({ - "text": keyword, - "label": entity_type, - "start_char": start, - "end_char": start + len(keyword), - "confidence": 1.0, # Exact match = 100% confidence - "method": "exact_keyword_match" - }) - -print(f"Found {len(extracted)} entities via exact matching") -``` - -**Method 4: Confidence-based Filtering (Quality Control)** -```python -from semantica.semantic_extract import NERExtractor - -extractor = NERExtractor(min_confidence=0.5) # Low threshold initially - -# Extract all entities -text = "Apple Inc. was founded by Steve Jobs in 1976." -all_entities = extractor.extract_entities(text, min_confidence=0.0) # Get all - -print(f"Total entities found: {len(all_entities)}") - -# Filter by different confidence thresholds -high_confidence = extractor.filter_by_confidence(all_entities, min_confidence=0.9) -medium_confidence = extractor.filter_by_confidence(all_entities, min_confidence=0.7) -low_confidence = extractor.filter_by_confidence(all_entities, min_confidence=0.5) - -print(f"High confidence (>=0.9): {len(high_confidence)}") -print(f"Medium confidence (>=0.7): {len(medium_confidence)}") -print(f"Low confidence (>=0.5): {len(low_confidence)}") - -# Use case: Progressive filtering -# Start with high confidence, lower threshold if not enough entities -if len(high_confidence) < 3: - entities_to_use = medium_confidence -else: - entities_to_use = high_confidence -``` - -**Method 5: Type-based Filtering (Entity Category Selection)** -```python -from semantica.semantic_extract import NERExtractor - -extractor = NERExtractor() - -text = "Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976." - -# Extract only specific entity types -persons_only = extractor.extract_entities( - text, - entity_types=["PERSON"], # Only persons - min_confidence=0.8 -) -print(f"Persons: {[e.text for e in persons_only]}") - -# Extract organizations and locations -orgs_and_locations = extractor.extract_entities( - text, - entity_types=["ORG", "GPE"], # Organizations and locations - min_confidence=0.8 -) -print(f"Orgs & Locations: {[e.text for e in orgs_and_locations]}") - -# Extract all types, then classify -all_entities = extractor.extract_entities(text) -classified = extractor.classify_entities(all_entities) - -for entity_type, entity_list in classified.items(): - print(f"{entity_type}: {[e.text for e in entity_list]}") -``` - -**Method 6: Batch Processing (Efficient Multi-text Processing)** -```python -from semantica.semantic_extract import NERExtractor - -extractor = NERExtractor() - -# Process multiple texts efficiently -texts = [ - "Microsoft is located in Redmond, Washington.", - "Tim Cook is the CEO of Apple.", - "Google was founded in Mountain View, California." -] - -# Batch extraction (more efficient than individual calls) -batch_entities = extractor.extract_entities_batch( - texts, - entity_types=["PERSON", "ORG", "GPE"], - min_confidence=0.8 -) - -# Returns list of entity lists (one per text) -for i, entities in enumerate(batch_entities): - print(f"Text {i+1}: {len(entities)} entities") - for entity in entities: - print(f" - {entity.text} ({entity.label})") -``` - -**Method 7: Hybrid Approach (Combine Multiple Methods)** -```python -from semantica.semantic_extract import NERExtractor - -extractor = NERExtractor() - -text = "Apple Inc. was founded by Steve Jobs in 1976." - -# Step 1: Extract with spaCy (if available) -spacy_entities = extractor.extract_entities(text, min_confidence=0.8) - -# Step 2: Add custom keyword matches -custom_keywords = { - "PERSON": ["Steve Jobs", "Tim Cook"], - "ORG": ["Apple Inc.", "Apple"] -} - -all_entities = list(spacy_entities) - -# Add exact keyword matches not found by spaCy -for entity_type, keywords in custom_keywords.items(): - for keyword in keywords: - if keyword.lower() in text.lower(): - # Check if already extracted - if not any(e.text == keyword for e in all_entities): - start = text.lower().find(keyword.lower()) - all_entities.append({ - "text": keyword, - "label": entity_type, - "start_char": start, - "end_char": start + len(keyword), - "confidence": 1.0, - "method": "custom_keyword" - }) - -print(f"Total entities (spaCy + custom): {len(all_entities)}") -``` - -**Best Practices:** - -1. **Use spaCy for production** - Highest accuracy, supports many entity types -2. **Set appropriate confidence thresholds** - Balance between recall and precision -3. **Filter by entity types** - Focus on relevant entity categories -4. **Use batch processing** - More efficient for multiple texts -5. **Combine methods** - Use spaCy + custom keywords for best coverage -6. **Validate extracted entities** - Check confidence scores and positions - ---- - -## Normalization Modules - -### `semantica.normalize.text_normalizer` - -**What it does:** -This module provides comprehensive text normalization capabilities for the Semantica framework, enabling standardization of text content across various formats and encodings. It handles Unicode normalization, whitespace handling, special character processing, case normalization, and format standardization. - -**Key Features:** -- Text cleaning and sanitization -- Unicode normalization (NFC, NFD, NFKC, NFKD) -- Case normalization (lower, upper, title, preserve) -- Whitespace handling (normalization, line breaks, indentation) -- Special character processing (punctuation, diacritics) -- Format standardization - -#### Class: `TextNormalizer` - -Text normalization and cleaning coordinator. - -**Methods:** - -##### `__init__(config: Optional[Dict[str, Any]] = None, **kwargs)` -Initialize text normalizer. - -**Parameters:** -- `config` (Optional[Dict[str, Any]]): Configuration dictionary -- `**kwargs`: Additional configuration options - -##### `normalize_text(text: str, unicode_form: str = "NFC", case: str = "preserve", normalize_diacritics: bool = False, line_break_type: str = "unix", **options) -> str` -Normalize text content. - -**Parameters:** -- `text` (str): Input text to normalize -- `unicode_form` (str): Unicode normalization form (default: "NFC"): - - "NFC": Canonical composition - - "NFD": Canonical decomposition - - "NFKC": Compatibility composition - - "NFKD": Compatibility decomposition -- `case` (str): Case normalization type (default: "preserve"): - - "preserve": Keep original case - - "lower": Convert to lowercase - - "upper": Convert to uppercase - - "title": Convert to title case -- `normalize_diacritics` (bool): Whether to normalize diacritics (default: False) -- `line_break_type` (str): Line break type (default: "unix") -- `**options`: Additional normalization options - -**Returns:** -- Normalized text string - -##### `clean_text(text: str, **options) -> str` -Clean and sanitize text content. - -**Parameters:** -- `text` (str): Input text to clean -- `**options`: Cleaning options (passed to TextCleaner.clean) - -**Returns:** -- Cleaned text string - -##### `standardize_format(text: str, format_type: str = "standard") -> str` -Standardize text format. - -**Parameters:** -- `text` (str): Input text to standardize -- `format_type` (str): Format type (default: "standard"): - - "standard": Apply standard formatting - - "compact": Remove extra whitespace - - "preserve": Preserve original formatting - -**Returns:** -- Formatted text string - -##### `process_batch(texts: List[str], **options) -> List[str]` -Process multiple texts in batch. - -**Parameters:** -- `texts` (List[str]): List of texts to process -- `**options`: Processing options - -**Returns:** -- List of normalized texts - -#### Class: `UnicodeNormalizer` - -Unicode normalization engine. - -**Methods:** - -##### `normalize_unicode(text: str, form: str = "NFC") -> str` -Normalize Unicode text. - -**Parameters:** -- `text` (str): Input text to normalize -- `form` (str): Unicode normalization form (default: "NFC") - -**Returns:** -- Unicode-normalized text - -##### `handle_encoding(text: str, source_encoding: str, target_encoding: str = "utf-8") -> str` -Handle text encoding conversion. - -**Parameters:** -- `text` (str): Input text (string or bytes) -- `source_encoding` (str): Source encoding name -- `target_encoding` (str): Target encoding name (default: "utf-8") - -**Returns:** -- Converted text in target encoding - -##### `process_special_chars(text: str) -> str` -Process special Unicode characters. - -**Parameters:** -- `text` (str): Input text containing special Unicode characters - -**Returns:** -- Text with special Unicode characters replaced with ASCII equivalents - -#### Class: `WhitespaceNormalizer` - -Whitespace normalization engine. - -**Methods:** - -##### `normalize_whitespace(text: str, line_break_type: str = "unix", **options) -> str` -Normalize whitespace in text. - -**Parameters:** -- `text` (str): Input text with potentially irregular whitespace -- `line_break_type` (str): Line break type (default: "unix") -- `**options`: Additional normalization options - -**Returns:** -- Text with normalized whitespace - -##### `handle_line_breaks(text: str, line_break_type: str = "unix") -> str` -Normalize line breaks. - -**Parameters:** -- `text` (str): Input text with potentially mixed line breaks -- `line_break_type` (str): Line break type (default: "unix") - -**Returns:** -- Text with normalized line breaks - -##### `process_indentation(text: str, indent_type: str = "spaces") -> str` -Normalize text indentation. - -**Parameters:** -- `text` (str): Input text with potentially mixed indentation -- `indent_type` (str): Indentation type (default: "spaces") - -**Returns:** -- Text with normalized indentation - -#### Class: `SpecialCharacterProcessor` - -Special character processing engine. - -**Methods:** - -##### `process_special_chars(text: str, normalize_diacritics: bool = False, **options) -> str` -Process special characters in text. - -**Parameters:** -- `text` (str): Input text to process -- `normalize_diacritics` (bool): Whether to normalize diacritics (default: False) -- `**options`: Additional processing options - -**Returns:** -- Text with special characters processed - -##### `normalize_punctuation(text: str) -> str` -Normalize punctuation marks. - -**Parameters:** -- `text` (str): Input text with potentially mixed punctuation - -**Returns:** -- Text with normalized punctuation marks - -##### `process_diacritics(text: str, remove_diacritics: bool = False, **options) -> str` -Process diacritical marks. - -**Parameters:** -- `text` (str): Input text with diacritical marks -- `remove_diacritics` (bool): Whether to remove diacritics (default: False) -- `**options`: Additional processing options - -**Returns:** -- Text with diacritics processed - -**Code Example:** -```python -from semantica.normalize import TextNormalizer - -# Initialize text normalizer -normalizer = TextNormalizer() - -# Normalize text with various options -text = "Hello World!!! This is a test." -normalized = normalizer.normalize_text( - text, - unicode_form="NFC", # Unicode normalization - case="lower", # Convert to lowercase - normalize_diacritics=True, # Normalize diacritics - line_break_type="unix" # Unix line breaks -) -print(f"Normalized: {normalized}") - -# Clean text (remove HTML, special chars, etc.) -dirty_text = "

Hello & World

" -cleaned = normalizer.clean_text( - dirty_text, - remove_html=True, - remove_special_chars=True -) -print(f"Cleaned: {cleaned}") - -# Standardize format -text = "This has extra spaces" -standardized = normalizer.standardize_format( - text, - format_type="compact" # Remove extra whitespace -) -print(f"Standardized: {standardized}") - -# Process batch of texts -texts = ["Text 1", "Text 2", "Text 3"] -normalized_batch = normalizer.process_batch( - texts, - case="lower", - unicode_form="NFC" -) - -# Use Unicode normalizer directly -from semantica.normalize.text_normalizer import UnicodeNormalizer -unicode_norm = UnicodeNormalizer() -normalized_unicode = unicode_norm.normalize_unicode("café", form="NFC") - -# Use whitespace normalizer -from semantica.normalize.text_normalizer import WhitespaceNormalizer -ws_norm = WhitespaceNormalizer() -normalized_ws = ws_norm.normalize_whitespace( - "Text with\t\t\t tabs", - line_break_type="unix" -) -``` - ---- - -## Export Modules - -### `semantica.export.json_exporter` - -**What it does:** -This module provides comprehensive JSON and JSON-LD export capabilities for the Semantica framework, enabling structured data export for knowledge graphs and semantic information. It supports both standard JSON and JSON-LD formats with configurable indentation, encoding, metadata, and provenance tracking. - -**Key Features:** -- JSON and JSON-LD format export -- Knowledge graph serialization -- Entity and relationship export -- Metadata and provenance tracking -- Configurable indentation and encoding -- JSON-LD context management - -#### Class: `JSONExporter` - -JSON exporter for knowledge graphs and semantic data. - -**Methods:** - -##### `__init__(indent: int = 2, ensure_ascii: bool = False, format: str = "json", config: Optional[Dict[str, Any]] = None, **kwargs)` -Initialize JSON exporter. - -**Parameters:** -- `indent` (int): JSON indentation level (default: 2) -- `ensure_ascii` (bool): Whether to escape non-ASCII characters (default: False) -- `format` (str): Export format - 'json' or 'json-ld' (default: 'json') -- `config` (Optional[Dict[str, Any]]): Optional configuration dictionary -- `**kwargs`: Additional configuration options - -##### `export(data: Any, file_path: Union[str, Path], format: Optional[str] = None, include_metadata: bool = True, include_provenance: bool = True, **options) -> None` -Export data to JSON file. - -**Parameters:** -- `data` (Any): Data to export (dict, list, or any JSON-serializable value) -- `file_path` (Union[str, Path]): Output JSON file path -- `format` (Optional[str]): Export format - 'json' or 'json-ld' (default: self.format) -- `include_metadata` (bool): Whether to include metadata (default: True) -- `include_provenance` (bool): Whether to include provenance information (default: True) -- `**options`: Additional options passed to conversion methods - -##### `export_knowledge_graph(knowledge_graph: Dict[str, Any], file_path: Union[str, Path], format: Optional[str] = None, **options) -> None` -Export knowledge graph to JSON or JSON-LD format. - -**Parameters:** -- `knowledge_graph` (Dict[str, Any]): Knowledge graph dictionary containing entities, relationships, nodes, edges, metadata, statistics -- `file_path` (Union[str, Path]): Output JSON file path -- `format` (Optional[str]): Export format (default: self.format) -- `**options`: Additional options - -##### `export_entities(entities: List[Dict[str, Any]], file_path: Union[str, Path], **options) -> None` -Export entities to JSON file. - -**Parameters:** -- `entities` (List[Dict[str, Any]]): List of entity dictionaries to export -- `file_path` (Union[str, Path]): Output JSON file path -- `**options`: Additional options: - - `metadata`: Additional metadata to include in export - -##### `export_relationships(relationships: List[Dict[str, Any]], file_path: Union[str, Path], **options) -> None` -Export relationships to JSON. - -**Parameters:** -- `relationships` (List[Dict[str, Any]]): List of relationship dictionaries -- `file_path` (Union[str, Path]): Output file path -- `**options`: Additional options - -**Code Example:** -```python -from semantica.export import JSONExporter - -# Initialize JSON exporter -exporter = JSONExporter( - indent=2, # Pretty print with 2 spaces - ensure_ascii=False, # Allow Unicode characters - format="json-ld" # Use JSON-LD format -) - -# Export knowledge graph -knowledge_graph = { - "entities": [ - {"id": "e1", "name": "Alice", "type": "Person"}, - {"id": "e2", "name": "Bob", "type": "Person"} - ], - "relationships": [ - {"source": "e1", "target": "e2", "type": "knows"} - ], - "metadata": {"created": "2024-01-15"} -} - -exporter.export_knowledge_graph( - knowledge_graph, - "output.json", - format="json-ld", - include_metadata=True, - include_provenance=True -) - -# Export entities only -entities = [ - {"id": "e1", "name": "Alice", "type": "Person"}, - {"id": "e2", "name": "Bob", "type": "Person"} -] - -exporter.export_entities( - entities, - "entities.json", - metadata={"source": "manual_entry"} -) - -# Export relationships -relationships = [ - {"source": "e1", "target": "e2", "type": "knows", "confidence": 0.9} -] - -exporter.export_relationships( - relationships, - "relationships.json" -) - -# Export any data structure -data = {"key": "value", "nested": {"a": 1, "b": 2}} -exporter.export( - data, - "data.json", - format="json", - include_metadata=True -) -``` - ---- - -## Vector Store Modules - -### `semantica.vector_store.vector_store` - -**What it does:** -This module provides the core vector storage, indexing, and retrieval operations for the Semantica framework. It includes vector storage, similarity search, indexing management, metadata association with vectors, and vector store maintenance capabilities. It supports multiple backends through adapters (FAISS, Pinecone, Weaviate, Qdrant, Milvus). - -**Key Features:** -- Vector storage and management -- Similarity search and retrieval -- Vector indexing and optimization -- Metadata association with vectors -- Vector update and deletion operations -- Multi-backend support through adapters - -#### Class: `VectorStore` - -Vector store interface and management. - -**Methods:** - -##### `__init__(backend="faiss", config=None, **kwargs)` -Initialize vector store. - -**Parameters:** -- `backend` (str): Backend name (default: "faiss") -- `config`: Configuration dictionary -- `**kwargs`: Additional configuration options - -##### `store_vectors(vectors: List[np.ndarray], metadata: Optional[List[Dict[str, Any]]] = None, **options) -> List[str]` -Store vectors in vector store. - -**Parameters:** -- `vectors` (List[np.ndarray]): List of vector arrays -- `metadata` (Optional[List[Dict[str, Any]]]): List of metadata dictionaries -- `**options`: Storage options - -**Returns:** -- List of vector IDs - -##### `search_vectors(query_vector: np.ndarray, k: int = 10, **options) -> List[Dict[str, Any]]` -Search for similar vectors. - -**Parameters:** -- `query_vector` (np.ndarray): Query vector -- `k` (int): Number of results to return (default: 10) -- `**options`: Search options - -**Returns:** -- List of search results with scores - -##### `update_vectors(vector_ids: List[str], new_vectors: List[np.ndarray], **options) -> bool` -Update existing vectors. - -**Parameters:** -- `vector_ids` (List[str]): List of vector IDs to update -- `new_vectors` (List[np.ndarray]): List of new vector arrays -- `**options`: Update options - -**Returns:** -- True if successful - -##### `delete_vectors(vector_ids: List[str], **options) -> bool` -Delete vectors from store. - -**Parameters:** -- `vector_ids` (List[str]): List of vector IDs to delete -- `**options`: Delete options - -**Returns:** -- True if successful - -##### `get_vector(vector_id: str) -> Optional[np.ndarray]` -Get vector by ID. - -**Parameters:** -- `vector_id` (str): Vector ID - -**Returns:** -- Vector array or None if not found - -##### `get_metadata(vector_id: str) -> Optional[Dict[str, Any]]` -Get metadata for vector. - -**Parameters:** -- `vector_id` (str): Vector ID - -**Returns:** -- Metadata dictionary or None if not found - -#### Class: `VectorIndexer` - -Vector indexing engine. - -**Methods:** - -##### `__init__(backend: str = "faiss", dimension: int = 768, **config)` -Initialize vector indexer. - -**Parameters:** -- `backend` (str): Backend name (default: "faiss") -- `dimension` (int): Vector dimension (default: 768) -- `**config`: Configuration options - -##### `create_index(vectors: List[np.ndarray], ids: Optional[List[str]] = None, **options) -> Any` -Create vector index. - -**Parameters:** -- `vectors` (List[np.ndarray]): List of vectors -- `ids` (Optional[List[str]]): Vector IDs -- `**options`: Indexing options - -**Returns:** -- Index object - -##### `update_index(index: Any, new_vectors: List[np.ndarray], **options) -> Any` -Update existing index. - -**Parameters:** -- `index` (Any): Existing index -- `new_vectors` (List[np.ndarray]): New vectors to add -- `**options`: Update options - -**Returns:** -- Updated index object - -##### `optimize_index(index: Any, **options) -> Any` -Optimize index for better performance. - -**Parameters:** -- `index` (Any): Index to optimize -- `**options`: Optimization options - -**Returns:** -- Optimized index object - -#### Class: `VectorRetriever` - -Vector retrieval engine. - -**Methods:** - -##### `__init__(backend: str = "faiss", **config)` -Initialize vector retriever. - -**Parameters:** -- `backend` (str): Backend name (default: "faiss") -- `**config`: Configuration options - -##### `search_similar(query_vector: np.ndarray, vectors: List[np.ndarray], ids: List[str], k: int = 10, **options) -> List[Dict[str, Any]]` -Search for similar vectors. - -**Parameters:** -- `query_vector` (np.ndarray): Query vector -- `vectors` (List[np.ndarray]): List of vectors to search -- `ids` (List[str]): Vector IDs -- `k` (int): Number of results (default: 10) -- `**options`: Search options - -**Returns:** -- List of results with scores - -##### `search_by_metadata(metadata_filters: Dict[str, Any], vectors: List[np.ndarray], metadata: List[Dict[str, Any]], **options) -> List[Dict[str, Any]]` -Search vectors by metadata. - -**Parameters:** -- `metadata_filters` (Dict[str, Any]): Metadata filter criteria -- `vectors` (List[np.ndarray]): List of vectors -- `metadata` (List[Dict[str, Any]]): List of metadata dictionaries -- `**options`: Search options - -**Returns:** -- List of matching vectors with metadata - -##### `search_hybrid(query_vector: np.ndarray, metadata_filters: Dict[str, Any], vectors: List[np.ndarray], metadata: List[Dict[str, Any]], **options) -> List[Dict[str, Any]]` -Perform hybrid search (metadata + similarity). - -**Parameters:** -- `query_vector` (np.ndarray): Query vector -- `metadata_filters` (Dict[str, Any]): Metadata filter criteria -- `vectors` (List[np.ndarray]): List of vectors -- `metadata` (List[Dict[str, Any]]): List of metadata dictionaries -- `**options`: Search options - -**Returns:** -- List of hybrid search results - -#### Class: `VectorManager` - -Vector store management engine. - -**Methods:** - -##### `__init__(**config)` -Initialize vector manager. - -**Parameters:** -- `**config`: Configuration options - -##### `manage_store(store: VectorStore, **operations: Dict[str, Any]) -> Dict[str, Any]` -Manage vector store operations. - -**Parameters:** -- `store` (VectorStore): Vector store instance -- `**operations`: Dictionary of operations to perform - -**Returns:** -- Dictionary of operation results - -##### `maintain_store(store: VectorStore, **options: Dict[str, Any]) -> Dict[str, Any]` -Maintain vector store health. - -**Parameters:** -- `store` (VectorStore): Vector store instance -- `**options`: Maintenance options - -**Returns:** -- Dictionary with health status - -##### `collect_statistics(store: VectorStore) -> Dict[str, Any]` -Collect vector store statistics. - -**Parameters:** -- `store` (VectorStore): Vector store instance - -**Returns:** -- Dictionary with statistics - -**Code Example:** -```python -from semantica.vector_store import VectorStore -import numpy as np - -# Initialize vector store -store = VectorStore( - backend="faiss", - config={"dimension": 768} -) - -# Store vectors with metadata -vectors = [ - np.random.rand(768), - np.random.rand(768), - np.random.rand(768) -] - -metadata = [ - {"text": "Document 1", "category": "tech"}, - {"text": "Document 2", "category": "science"}, - {"text": "Document 3", "category": "tech"} -] - -vector_ids = store.store_vectors(vectors, metadata=metadata) -print(f"Stored {len(vector_ids)} vectors") - -# Search for similar vectors -query_vector = np.random.rand(768) -results = store.search_vectors( - query_vector, - k=5, # Top 5 results - method="cosine" -) - -for result in results: - print(f"ID: {result['id']}, Score: {result['score']:.4f}") - print(f"Metadata: {store.get_metadata(result['id'])}") - -# Update vectors -new_vectors = [np.random.rand(768) for _ in range(2)] -store.update_vectors(vector_ids[:2], new_vectors) - -# Get specific vector and metadata -vector = store.get_vector(vector_ids[0]) -meta = store.get_metadata(vector_ids[0]) - -# Delete vectors -store.delete_vectors(vector_ids[2:]) - -# Use vector indexer directly -from semantica.vector_store.vector_store import VectorIndexer -indexer = VectorIndexer(backend="faiss", dimension=768) -index = indexer.create_index(vectors, vector_ids) - -# Use vector retriever for advanced search -from semantica.vector_store.vector_store import VectorRetriever -retriever = VectorRetriever(backend="faiss") - -# Hybrid search (metadata + similarity) -results = retriever.search_hybrid( - query_vector, - metadata_filters={"category": "tech"}, - vectors=vectors, - metadata=metadata, - k=10 -) -``` - ---- - -## Reasoning Modules - -### `semantica.reasoning.inference_engine` - -**What it does:** -This module provides rule-based inference capabilities for knowledge graph reasoning and analysis. It supports forward chaining (data-driven), backward chaining (goal-driven), and bidirectional inference strategies. The module includes rule management, performance optimization, error handling, and custom rule support. - -**Key Features:** -- Rule-based inference and reasoning -- Forward and backward chaining -- Bidirectional inference -- Rule management and execution -- Performance optimization -- Error handling and recovery -- Custom rule support - -#### Class: `InferenceEngine` - -Rule-based inference engine. - -**Methods:** - -##### `__init__(config: Optional[Dict[str, Any]] = None, **kwargs)` -Initialize inference engine. - -**Parameters:** -- `config` (Optional[Dict[str, Any]]): Configuration dictionary -- `**kwargs`: Additional configuration options: - - `strategy` (str): Inference strategy (forward, backward, bidirectional) - - `max_iterations` (int): Maximum inference iterations (default: 100) - -##### `add_rule(rule_definition: str, **options) -> Rule` -Add inference rule to engine. - -**Parameters:** -- `rule_definition` (str): Rule definition string or Rule object -- `**options`: Additional options - -**Returns:** -- Created rule - -##### `add_fact(fact: Any) -> None` -Add fact to knowledge base. - -**Parameters:** -- `fact` (Any): Fact to add - -##### `add_facts(facts: List[Any]) -> None` -Add multiple facts. - -**Parameters:** -- `facts` (List[Any]): List of facts - -##### `forward_chain(facts: Optional[List[Any]] = None, **options) -> List[InferenceResult]` -Perform forward chaining inference. - -**Parameters:** -- `facts` (Optional[List[Any]]): Optional initial facts -- `**options`: Additional options - -**Returns:** -- List of inference results - -##### `backward_chain(goal: Any, **options) -> Optional[InferenceResult]` -Perform backward chaining inference. - -**Parameters:** -- `goal` (Any): Goal to prove -- `**options`: Additional options - -**Returns:** -- Inference result or None - -##### `infer(query: Any, **options) -> List[InferenceResult]` -Perform inference based on strategy. - -**Parameters:** -- `query` (Any): Query or goal -- `**options`: Additional options - -**Returns:** -- List of inference results - -##### `get_facts() -> Set[Any]` -Get all facts. - -**Returns:** -- Set of all facts - -##### `get_inferred_facts() -> List[InferenceResult]` -Get all inferred facts. - -**Returns:** -- List of inference results - -##### `clear_facts() -> None` -Clear all facts. - -##### `reset() -> None` -Reset inference engine. - -#### Class: `InferenceResult` - -Inference result dataclass. - -**Attributes:** -- `conclusion` (Any): Inferred conclusion -- `premises` (List[Any]): Premises used for inference -- `rule_used` (Optional[Rule]): Rule used for inference -- `confidence` (float): Confidence score (default: 1.0) -- `metadata` (Dict[str, Any]): Additional metadata - -#### Enum: `InferenceStrategy` - -Inference strategies. - -**Values:** -- `FORWARD`: Forward chaining -- `BACKWARD`: Backward chaining -- `BIDIRECTIONAL`: Bidirectional inference - -**Code Example:** -```python -from semantica.reasoning import InferenceEngine, InferenceStrategy - -# Initialize inference engine -engine = InferenceEngine( - strategy="forward", # or "backward", "bidirectional" - max_iterations=100 -) - -# Add facts to knowledge base -engine.add_fact("Alice is a Person") -engine.add_fact("Bob is a Person") -engine.add_fact("Alice knows Bob") - -# Define and add rules -rule1 = engine.add_rule( - "IF Person(X) AND Person(Y) AND knows(X, Y) THEN friends(X, Y)", - name="friendship_rule", - confidence=0.9 -) - -rule2 = engine.add_rule( - "IF friends(X, Y) THEN can_trust(X, Y)", - name="trust_rule" -) - -# Perform forward chaining (derive new facts from existing facts) -results = engine.forward_chain() -for result in results: - print(f"Inferred: {result.conclusion}") - print(f" Using rule: {result.rule_used.name}") - print(f" Confidence: {result.confidence}") - -# Perform backward chaining (prove a specific goal) -goal_result = engine.backward_chain("can_trust(Alice, Bob)") -if goal_result: - print(f"Goal proven: {goal_result.conclusion}") - print(f"Premises: {goal_result.premises}") - -# Perform inference based on strategy -query_results = engine.infer( - query="can_trust(Alice, Bob)", - strategy=InferenceStrategy.BIDIRECTIONAL -) - -# Get all facts (original + inferred) -all_facts = engine.get_facts() -print(f"Total facts: {len(all_facts)}") - -# Get only inferred facts -inferred = engine.get_inferred_facts() -print(f"Inferred facts: {len(inferred)}") - -# Clear facts and reset -engine.clear_facts() -engine.reset() -``` - ---- - -## Split Modules - -### `semantica.split` - -**What it does:** -This module provides comprehensive document chunking and splitting capabilities for optimal processing and semantic analysis. It enables efficient handling of large documents through various chunking strategies, each optimized for different use cases and document types. - -**Key Features:** -- Semantic-based chunking using NLP (spaCy) -- Structure-aware chunking (headings, paragraphs, lists, code blocks) -- Sliding window chunking with configurable overlap -- Table-specific chunking -- Chunk validation and quality assessment -- Provenance tracking for data lineage - -**Different Approaches and Strategies:** - -The split module provides four main chunking strategies, each optimized for different scenarios: - -1. **Semantic Chunking** - Uses NLP to split at semantic boundaries (sentences, paragraphs) -2. **Structural Chunking** - Respects document structure (headings, sections, lists) -3. **Sliding Window Chunking** - Fixed-size chunks with overlap for context preservation -4. **Table Chunking** - Specialized chunking for tabular data - -**When to Use Each Approach:** - -- **Semantic Chunking**: Natural language documents, preserving meaning and context -- **Structural Chunking**: Markdown, HTML, technical documentation with clear structure -- **Sliding Window**: Fixed-size requirements, embedding generation, vector stores -- **Table Chunking**: Spreadsheets, CSV data, structured tabular content - -**Comparison of Chunking Strategies:** - -| Strategy | Best For | Preserves | Speed | Quality | -|----------|----------|-----------|-------|---------| -| Semantic | Natural language | Sentence/paragraph boundaries | Medium | High | -| Structural | Markdown/HTML docs | Document hierarchy | Fast | High | -| Sliding Window | Fixed-size needs | Context via overlap | Fast | Medium | -| Table | Tabular data | Table structure | Fast | High | - -#### Class: `SemanticChunker` - -Semantic chunker for meaning-based splitting using NLP. - -**Methods:** - -##### `__init__(**config)` -Initialize semantic chunker. - -**Parameters:** -- `model` (str): spaCy model name (default: "en_core_web_sm") -- `chunk_size` (int): Target chunk size in characters (default: 1000) -- `chunk_overlap` (int): Overlap between chunks in characters (default: 200) -- `language` (str): Language code (default: "en") - -##### `chunk(text: str, **options) -> List[Chunk]` -Split text into semantic chunks. - -**Parameters:** -- `text` (str): Input text to chunk -- `preserve_sentences` (bool): Preserve sentence boundaries (default: True) -- `preserve_paragraphs` (bool): Preserve paragraph boundaries (default: True) - -**Returns:** -- List of Chunk objects with text, start_index, end_index, and metadata - -##### `chunk_by_sentences(text: str, max_sentences: int = 5) -> List[Chunk]` -Chunk text by sentence boundaries. - -**Parameters:** -- `text` (str): Input text -- `max_sentences` (int): Maximum sentences per chunk (default: 5) - -**Returns:** -- List of chunks, each containing up to max_sentences - -**Code Examples for Different Approaches:** - -**Approach 1: Semantic Chunking (NLP-based, Recommended for Natural Language)** -```python -from semantica.split import SemanticChunker - -# Initialize semantic chunker with spaCy model -chunker = SemanticChunker( - model="en_core_web_sm", # spaCy model for English - chunk_size=1000, # Target 1000 characters per chunk - chunk_overlap=200, # 200 character overlap between chunks - language="en" -) - -# Chunk long document (preserves sentence and paragraph boundaries) -long_text = """ -This is a long document with multiple paragraphs. -Each paragraph contains several sentences. - -The semantic chunker uses NLP to identify natural boundaries. -It won't split sentences in the middle, preserving meaning. - -This ensures that each chunk is semantically coherent. -""" - -chunks = chunker.chunk(long_text, preserve_sentences=True, preserve_paragraphs=True) - -for i, chunk in enumerate(chunks): - print(f"Chunk {i+1}: {len(chunk.text)} chars") - print(f" Sentences: {chunk.metadata.get('sentence_count', 'N/A')}") - print(f" Text preview: {chunk.text[:100]}...") - print() - -# Chunk by sentences (fixed number of sentences per chunk) -sentence_chunks = chunker.chunk_by_sentences(long_text, max_sentences=3) -print(f"Created {len(sentence_chunks)} sentence-based chunks") -``` - -**Approach 2: Structural Chunking (Document Structure-aware)** -```python -from semantica.split import StructuralChunker - -# Initialize structural chunker -struct_chunker = StructuralChunker( - respect_headers=True, # Respect heading hierarchy - respect_sections=True, # Respect section boundaries - max_chunk_size=2000 # Maximum chunk size -) - -# Chunk structured document (Markdown, HTML, etc.) -markdown_doc = """ -# Main Title - -## Section 1 -This is the first section with multiple paragraphs. - -### Subsection 1.1 -Details about subsection 1.1. - -## Section 2 -Another section with content. - -- List item 1 -- List item 2 -- List item 3 -""" - -chunks = struct_chunker.chunk(markdown_doc) - -for i, chunk in enumerate(chunks): - print(f"Chunk {i+1}:") - print(f" Elements: {chunk.metadata.get('element_count', 0)}") - print(f" Types: {chunk.metadata.get('element_types', [])}") - print(f" Structure preserved: {chunk.metadata.get('structure_preserved', False)}") - print(f" Preview: {chunk.text[:100]}...") - print() -``` - -**Approach 3: Sliding Window Chunking (Fixed-size with Overlap)** -```python -from semantica.split import SlidingWindowChunker - -# Initialize sliding window chunker -window_chunker = SlidingWindowChunker( - chunk_size=512, # Fixed chunk size (characters) - overlap=100, # 100 character overlap - stride=412 # Stride = chunk_size - overlap -) - -# Chunk text with fixed-size windows -text = "Your long document text here..." * 100 - -chunks = window_chunker.chunk( - text, - preserve_boundaries=True # Try to preserve word/sentence boundaries -) - -print(f"Created {len(chunks)} fixed-size chunks") -for i, chunk in enumerate(chunks): - print(f"Chunk {i+1}: {len(chunk.text)} chars (start: {chunk.start_index}, end: {chunk.end_index})") - -# Use case: For embedding generation where fixed-size chunks are required -# The overlap ensures context is preserved across chunk boundaries -``` - -**Approach 4: Table Chunking (Tabular Data)** -```python -from semantica.split import TableChunker - -# Initialize table chunker -table_chunker = TableChunker() - -# Chunk table data (CSV, Excel, HTML tables) -table_data = """ -Name,Age,City -Alice,30,New York -Bob,25,San Francisco -Charlie,35,Chicago -""" - -table_chunks = table_chunker.chunk(table_data) - -for chunk in table_chunks: - print(f"Table chunk: {chunk.metadata.get('row_count', 0)} rows") - print(f"Columns: {chunk.metadata.get('column_count', 0)}") -``` - -**Approach 5: Hybrid Chunking (Combine Strategies)** -```python -from semantica.split import SemanticChunker, StructuralChunker - -# Use semantic chunking for natural language sections -semantic_chunker = SemanticChunker(chunk_size=1000, chunk_overlap=200) - -# Use structural chunking for code/documentation sections -struct_chunker = StructuralChunker(max_chunk_size=2000) - -# Process different sections with appropriate chunkers -document = { - "introduction": "Natural language introduction text...", - "code_section": "```python\ndef function():\n pass\n```", - "conclusion": "Natural language conclusion..." -} - -# Chunk each section with appropriate strategy -all_chunks = [] -all_chunks.extend(semantic_chunker.chunk(document["introduction"])) -all_chunks.extend(struct_chunker.chunk(document["code_section"])) -all_chunks.extend(semantic_chunker.chunk(document["conclusion"])) - -print(f"Total chunks: {len(all_chunks)}") -``` - -**Approach 6: Chunk Validation and Quality Assessment** -```python -from semantica.split import SemanticChunker, ChunkValidator - -# Create chunks -chunker = SemanticChunker(chunk_size=1000, chunk_overlap=200) -chunks = chunker.chunk(long_text) - -# Validate chunk quality -validator = ChunkValidator( - min_chunk_size=100, # Minimum chunk size - max_chunk_size=2000, # Maximum chunk size - min_sentence_count=1, # Minimum sentences per chunk - require_completeness=True # Require complete sentences -) - -validation_results = [] -for chunk in chunks: - result = validator.validate(chunk) - validation_results.append(result) - - if not result.is_valid: - print(f"Invalid chunk: {result.issues}") - else: - print(f"Valid chunk: Quality score = {result.metrics.get('quality_score', 0):.2f}") - -# Filter valid chunks -valid_chunks = [c for c, r in zip(chunks, validation_results) if r.is_valid] -print(f"Valid chunks: {len(valid_chunks)}/{len(chunks)}") -``` - -**Approach 7: Chunking with Provenance Tracking** -```python -from semantica.split import SemanticChunker, ProvenanceTracker - -# Initialize chunker and provenance tracker -chunker = SemanticChunker(chunk_size=1000, chunk_overlap=200) -tracker = ProvenanceTracker() - -# Chunk document with provenance tracking -document_id = "doc_123" -source_file = "document.pdf" -chunks = chunker.chunk(text) - -# Track provenance for each chunk -for i, chunk in enumerate(chunks): - tracker.track_chunk( - chunk_id=f"{document_id}_chunk_{i}", - chunk=chunk, - source={ - "document_id": document_id, - "file_path": source_file, - "page_number": 1, - "section": "main" - } - ) - -# Retrieve provenance information -provenance = tracker.get_provenance(f"{document_id}_chunk_0") -print(f"Chunk source: {provenance['source']}") -print(f"Document lineage: {provenance['lineage']}") -``` - -**Best Practices:** - -1. **Choose chunking strategy based on document type**: - - Natural language → Semantic chunking - - Structured documents → Structural chunking - - Fixed-size requirements → Sliding window - - Tables → Table chunking - -2. **Set appropriate chunk sizes**: - - Too small: Loses context, poor embeddings - - Too large: Exceeds model limits, inefficient - - Recommended: 500-2000 characters for most use cases - -3. **Use overlap for context preservation**: - - 10-20% overlap recommended for sliding window - - Semantic chunking automatically preserves context - -4. **Validate chunk quality**: - - Check chunk sizes are within acceptable range - - Ensure chunks contain complete sentences/paragraphs - - Verify semantic coherence - -5. **Track provenance**: - - Maintain source information for each chunk - - Enable traceability and debugging - - Support data lineage requirements - ---- - -## Conflict Resolution Modules - -### `semantica.conflicts.conflict_resolver` - -**What it does:** -This module provides comprehensive conflict resolution capabilities for resolving detected conflicts in knowledge graphs. It offers multiple resolution strategies including voting mechanisms, credibility-based resolution, recency-based resolution, and expert review workflows. - -**Key Features:** -- Multiple resolution strategies (voting, credibility-weighted, recency, confidence) -- Automatic conflict resolution -- Manual and expert review workflows -- Resolution rule configuration -- Conflict resolution history tracking -- Source credibility weighting - -**Different Conflict Resolution Strategies:** - -The conflict resolver supports 7 different resolution strategies: - -1. **Voting** - Most common value wins (democratic approach) -2. **Credibility Weighted** - Weight values by source credibility -3. **Most Recent** - Use the most recent value (temporal priority) -4. **First Seen** - Use the first encountered value (original priority) -5. **Highest Confidence** - Use value with highest confidence score -6. **Manual Review** - Flag for human review -7. **Expert Review** - Flag for domain expert review - -**When to Use Each Strategy:** - -- **Voting**: Multiple sources, democratic resolution, equal source credibility -- **Credibility Weighted**: Sources have different reliability, quality matters -- **Most Recent**: Temporal data, recent information preferred -- **First Seen**: Original data preferred, historical accuracy -- **Highest Confidence**: Confidence scores available, quality-based -- **Manual Review**: Complex conflicts, human judgment needed -- **Expert Review**: Domain-specific conflicts, expert knowledge required - -**Comparison of Resolution Strategies:** - -| Strategy | Automation | Quality | Speed | Best For | -|----------|------------|---------|-------|----------| -| Voting | Full | Medium | Fast | Multiple equal sources | -| Credibility Weighted | Full | High | Fast | Varying source quality | -| Most Recent | Full | Medium | Fast | Temporal data | -| First Seen | Full | Medium | Fast | Historical data | -| Highest Confidence | Full | High | Fast | Confidence scores available | -| Manual Review | None | Very High | Slow | Complex conflicts | -| Expert Review | None | Very High | Very Slow | Domain-specific | - -**Code Examples for All Resolution Strategies:** - -**Strategy 1: Voting (Most Common Value Wins)** -```python -from semantica.conflicts import ConflictResolver, Conflict - -resolver = ConflictResolver(default_strategy="voting") - -# Conflict with multiple conflicting values -conflict = Conflict( - conflict_id="conflict_1", - entity_id="entity_1", - property_name="name", - conflicting_values=["Apple Inc.", "Apple Inc.", "Apple", "Apple Corp"], - sources=[...], - conflict_type=ConflictType.VALUE_CONFLICT -) - -# Resolve by voting (most common value wins) -result = resolver.resolve_conflict(conflict, strategy="voting") - -print(f"Resolved value: {result.resolved_value}") # "Apple Inc." (2 votes) -print(f"Confidence: {result.confidence:.2f}") # 0.5 (2/4 votes) -print(f"Resolution: {result.resolution_notes}") -``` - -**Strategy 2: Credibility Weighted (Source Quality Matters)** -```python -from semantica.conflicts import ConflictResolver - -# Initialize with credibility tracking -resolver = ConflictResolver( - default_strategy="credibility_weighted" -) - -# Set source credibility scores -resolver.source_tracker.set_source_credibility("official_database", 0.9) -resolver.source_tracker.set_source_credibility("user_input", 0.5) -resolver.source_tracker.set_source_credibility("web_scraping", 0.3) - -# Conflict with values from different sources -conflict = Conflict( - conflict_id="conflict_2", - entity_id="entity_1", - property_name="founded_year", - conflicting_values=[1976, 1977, 1976], - sources=[ - {"document": "official_database", "confidence": 0.9}, - {"document": "user_input", "confidence": 0.6}, - {"document": "web_scraping", "confidence": 0.4} - ] -) - -# Resolve by credibility-weighted voting -result = resolver.resolve_conflict(conflict, strategy="credibility_weighted") - -# Official database value wins due to high credibility -print(f"Resolved value: {result.resolved_value}") # 1976 -print(f"Confidence: {result.confidence:.2f}") -``` - -**Strategy 3: Most Recent (Temporal Priority)** -```python -from semantica.conflicts import ConflictResolver -from datetime import datetime - -resolver = ConflictResolver() - -# Conflict with timestamps -conflict = Conflict( - conflict_id="conflict_3", - entity_id="entity_1", - property_name="ceo", - conflicting_values=["Tim Cook", "Steve Jobs"], - sources=[ - { - "document": "source_1", - "metadata": {"timestamp": datetime(2020, 1, 1)} - }, - { - "document": "source_2", - "metadata": {"timestamp": datetime(2023, 1, 1)} - } - ] -) - -# Resolve by most recent value -result = resolver.resolve_conflict(conflict, strategy="most_recent") - -print(f"Resolved value: {result.resolved_value}") # "Tim Cook" (most recent) -print(f"Confidence: {result.confidence:.2f}") # 0.8 -``` - -**Strategy 4: First Seen (Original Priority)** -```python -from semantica.conflicts import ConflictResolver - -resolver = ConflictResolver() - -# Conflict where first value is preferred -conflict = Conflict( - conflict_id="conflict_4", - entity_id="entity_1", - property_name="original_name", - conflicting_values=["Apple Computer", "Apple Inc.", "Apple"], - sources=[...] -) - -# Resolve by first seen (original value) -result = resolver.resolve_conflict(conflict, strategy="first_seen") - -print(f"Resolved value: {result.resolved_value}") # "Apple Computer" (first) -print(f"Confidence: {result.confidence:.2f}") # 0.7 -``` - -**Strategy 5: Highest Confidence (Quality-based)** -```python -from semantica.conflicts import ConflictResolver - -resolver = ConflictResolver() - -# Conflict with confidence scores -conflict = Conflict( - conflict_id="conflict_5", - entity_id="entity_1", - property_name="revenue", - conflicting_values=[1000000, 1200000, 1100000], - sources=[ - {"document": "source_1", "confidence": 0.6}, - {"document": "source_2", "confidence": 0.9}, # Highest - {"document": "source_3", "confidence": 0.7} - ] -) - -# Resolve by highest confidence -result = resolver.resolve_conflict(conflict, strategy="highest_confidence") - -print(f"Resolved value: {result.resolved_value}") # 1200000 (highest confidence) -print(f"Confidence: {result.confidence:.2f}") # 0.9 -``` - -**Strategy 6: Manual Review (Human Judgment)** -```python -from semantica.conflicts import ConflictResolver - -resolver = ConflictResolver() - -# Complex conflict requiring human judgment -conflict = Conflict( - conflict_id="conflict_6", - entity_id="entity_1", - property_name="industry", - conflicting_values=["Technology", "Consumer Electronics", "Software"], - sources=[...], - severity="high" -) - -# Flag for manual review -result = resolver.resolve_conflict(conflict, strategy="manual_review") - -print(f"Resolved: {result.resolved}") # False -print(f"Requires review: {result.metadata.get('requires_manual_review')}") # True -print(f"Notes: {result.resolution_notes}") # "Flagged for manual review" - -# Later, manually resolve -result.resolved = True -result.resolved_value = "Technology" -result.resolution_notes = "Manually resolved by domain expert" -``` - -**Strategy 7: Expert Review (Domain Expertise)** -```python -from semantica.conflicts import ConflictResolver - -resolver = ConflictResolver() - -# Domain-specific conflict -conflict = Conflict( - conflict_id="conflict_7", - entity_id="entity_1", - property_name="legal_classification", - conflicting_values=["Corporation", "LLC", "Partnership"], - sources=[...], - severity="critical" -) - -# Flag for expert review -result = resolver.resolve_conflict(conflict, strategy="expert_review") - -print(f"Requires expert review: {result.metadata.get('requires_expert_review')}") # True -``` - -**Strategy 8: Custom Resolution Rules (Property-specific)** -```python -from semantica.conflicts import ConflictResolver - -resolver = ConflictResolver() - -# Set custom resolution rule for specific property -resolver.set_resolution_rule( - entity_id="entity_1", - property_name="name", - strategy="voting" -) - -resolver.set_resolution_rule( - entity_id="entity_1", - property_name="founded_year", - strategy="most_recent" # Years should use most recent -) - -# Conflicts will automatically use appropriate strategy -conflict = Conflict( - conflict_id="conflict_8", - entity_id="entity_1", - property_name="name", # Will use VOTING - conflicting_values=["Apple Inc.", "Apple"], - sources=[...] -) - -result = resolver.resolve_conflict(conflict) # Uses VOTING automatically -``` - -**Strategy 9: Batch Conflict Resolution** -```python -from semantica.conflicts import ConflictResolver - -resolver = ConflictResolver(default_strategy="voting") - -# Resolve multiple conflicts at once -conflicts = [ - Conflict(conflict_id="c1", entity_id="e1", property_name="name", ...), - Conflict(conflict_id="c2", entity_id="e2", property_name="type", ...), - Conflict(conflict_id="c3", entity_id="e3", property_name="location", ...) -] - -# Resolve all conflicts -results = resolver.resolve_conflicts(conflicts) - -# Get statistics -stats = resolver.get_resolution_statistics() -print(f"Total resolved: {stats['resolved_count']}/{stats['total_resolutions']}") -print(f"Resolution rate: {stats['resolution_rate']:.2%}") -print(f"By strategy: {stats['by_strategy']}") -``` - -**Best Practices:** - -1. **Choose strategy based on conflict type**: - - Value conflicts → Voting or Credibility Weighted - - Temporal conflicts → Most Recent - - Quality conflicts → Highest Confidence - - Complex conflicts → Manual/Expert Review - -2. **Set property-specific rules** - Different properties may need different strategies -3. **Track resolution history** - Monitor resolution patterns and success rates -4. **Use credibility weighting** - When source quality varies significantly -5. **Flag complex conflicts** - Don't auto-resolve everything, use human judgment when needed - ---- - -## Deduplication Modules - -### `semantica.deduplication` - -**What it does:** -This module provides comprehensive duplicate detection and entity merging capabilities for knowledge graphs. It uses multiple similarity calculation methods from exact matching to advanced semantic similarity to identify and merge duplicate entities. - -**Key Features:** -- Multiple similarity calculation methods (exact, fuzzy, semantic) -- Duplicate detection with configurable thresholds -- Entity merging with multiple strategies -- Batch and incremental duplicate detection -- Confidence scoring for duplicate candidates -- Group-based duplicate clustering - -**Different Duplication Detection Methods:** - -The deduplication module supports multiple similarity calculation methods: - -1. **Exact String Matching** - Perfect string equality -2. **Levenshtein Distance** - Edit distance-based similarity -3. **Jaro-Winkler Similarity** - String similarity with prefix bonus -4. **Cosine Similarity** - Character n-gram based similarity -5. **Property Similarity** - Property value comparison -6. **Relationship Similarity** - Jaccard similarity of relationships -7. **Embedding Similarity** - Semantic similarity using vector embeddings -8. **Multi-factor Similarity** - Weighted combination of all methods - -**When to Use Each Method:** - -- **Exact Matching**: Identical strings, fast lookup, high precision -- **Levenshtein**: Typos, spelling variations, edit distance -- **Jaro-Winkler**: Names, addresses, prefix importance -- **Cosine (n-grams)**: General text similarity, character-level -- **Property Similarity**: Entity properties comparison -- **Relationship Similarity**: Graph structure comparison -- **Embedding Similarity**: Semantic meaning, best quality -- **Multi-factor**: Production use, comprehensive comparison - -**Comparison of Similarity Methods:** - -| Method | Accuracy | Speed | Use Case | Best For | -|--------|----------|-------|----------|----------| -| Exact Match | Perfect | Very Fast | Identical strings | Fast lookup | -| Levenshtein | High | Fast | Typos, variations | Spelling errors | -| Jaro-Winkler | High | Fast | Names, addresses | Prefix matching | -| Cosine (n-gram) | Medium | Fast | General text | Character similarity | -| Property Similarity | Medium | Medium | Entity properties | Structured data | -| Relationship Similarity | Medium | Medium | Graph structure | Network analysis | -| Embedding Similarity | Very High | Slow | Semantic meaning | Best quality | -| Multi-factor | Very High | Medium | Production | Comprehensive | - -**Code Examples for All Similarity Methods:** - -**Method 1: Exact String Matching** -```python -from semantica.deduplication import SimilarityCalculator - -calculator = SimilarityCalculator() - -# Exact matching (built into string similarity) -entity1 = {"name": "Apple Inc."} -entity2 = {"name": "Apple Inc."} - -# Exact match returns 1.0 -similarity = calculator.calculate_string_similarity( - entity1["name"], - entity2["name"], - method="levenshtein" # Will detect exact match first -) -print(f"Similarity: {similarity}") # 1.0 (exact match) -``` - -**Method 2: Levenshtein Distance (Edit Distance)** -```python -from semantica.deduplication import SimilarityCalculator - -calculator = SimilarityCalculator() - -# Levenshtein handles typos and variations -entity1 = {"name": "Apple Inc."} -entity2 = {"name": "Apple Inc"} # Missing period - -similarity = calculator.calculate_string_similarity( - entity1["name"], - entity2["name"], - method="levenshtein" -) -print(f"Levenshtein similarity: {similarity:.4f}") # ~0.91 - -# Works well for typos -entity3 = {"name": "Aple Inc."} # Typo -similarity2 = calculator.calculate_string_similarity( - entity1["name"], - entity3["name"], - method="levenshtein" -) -print(f"With typo: {similarity2:.4f}") # ~0.82 -``` - -**Method 3: Jaro-Winkler Similarity (Prefix Bonus)** -```python -from semantica.deduplication import SimilarityCalculator - -calculator = SimilarityCalculator() - -# Jaro-Winkler gives bonus for matching prefixes -entity1 = {"name": "Apple Inc."} -entity2 = {"name": "Apple Corporation"} - -similarity = calculator.calculate_string_similarity( - entity1["name"], - entity2["name"], - method="jaro_winkler" -) -print(f"Jaro-Winkler similarity: {similarity:.4f}") # Higher than Levenshtein - -# Great for names and addresses where prefix matters -name1 = "John Smith" -name2 = "John Smyth" # Different suffix -similarity2 = calculator.calculate_string_similarity( - name1, name2, method="jaro_winkler" -) -print(f"Name similarity: {similarity2:.4f}") # High due to prefix match -``` - -**Method 4: Cosine Similarity (Character N-grams)** -```python -from semantica.deduplication import SimilarityCalculator - -calculator = SimilarityCalculator() - -# Cosine similarity using character bigrams -entity1 = {"name": "Apple Inc."} -entity2 = {"name": "Apple Corporation"} - -similarity = calculator.calculate_string_similarity( - entity1["name"], - entity2["name"], - method="cosine" -) -print(f"Cosine similarity: {similarity:.4f}") # Based on character bigrams -``` - -**Method 5: Property Similarity (Entity Properties)** -```python -from semantica.deduplication import SimilarityCalculator - -calculator = SimilarityCalculator() - -# Compare entities based on properties -entity1 = { - "name": "Apple Inc.", - "properties": { - "founded": 1976, - "location": "Cupertino", - "industry": "Technology" - } -} - -entity2 = { - "name": "Apple", - "properties": { - "founded": 1976, - "location": "Cupertino", - "industry": "Tech" - } -} - -# Property similarity compares matching properties -property_sim = calculator.calculate_property_similarity(entity1, entity2) -print(f"Property similarity: {property_sim:.4f}") # High (founded, location match) -``` - -**Method 6: Relationship Similarity (Jaccard Similarity)** -```python -from semantica.deduplication import SimilarityCalculator - -calculator = SimilarityCalculator() - -# Compare entities based on relationships -entity1 = { - "name": "Apple Inc.", - "relationships": [ - {"type": "founded_by", "target": "Steve Jobs"}, - {"type": "located_in", "target": "Cupertino"} - ] -} - -entity2 = { - "name": "Apple", - "relationships": [ - {"type": "founded_by", "target": "Steve Jobs"}, - {"type": "located_in", "target": "California"} - ] -} - -# Relationship similarity (Jaccard) -rel_sim = calculator.calculate_relationship_similarity(entity1, entity2) -print(f"Relationship similarity: {rel_sim:.4f}") # 0.33 (1 common / 3 total) -``` - -**Method 7: Embedding Similarity (Semantic)** -```python -from semantica.deduplication import SimilarityCalculator -import numpy as np - -calculator = SimilarityCalculator() - -# Semantic similarity using embeddings -entity1 = { - "name": "Apple Inc.", - "embedding": np.random.rand(384) # Embedding vector -} - -entity2 = { - "name": "Apple Corporation", - "embedding": np.random.rand(384) # Similar embedding -} - -# Embedding similarity (cosine similarity of vectors) -embedding_sim = calculator.calculate_embedding_similarity( - entity1["embedding"].tolist(), - entity2["embedding"].tolist() -) -print(f"Embedding similarity: {embedding_sim:.4f}") # Semantic similarity -``` - -**Method 8: Multi-factor Similarity (Weighted Combination)** -```python -from semantica.deduplication import SimilarityCalculator - -# Configure weights for different factors -calculator = SimilarityCalculator( - string_weight=0.3, # 30% weight for string similarity - property_weight=0.2, # 20% weight for properties - relationship_weight=0.1, # 10% weight for relationships - embedding_weight=0.4 # 40% weight for embeddings (highest) -) - -entity1 = { - "name": "Apple Inc.", - "properties": {"founded": 1976}, - "relationships": [{"type": "founded_by", "target": "Steve Jobs"}], - "embedding": [...] # Embedding vector -} - -entity2 = { - "name": "Apple", - "properties": {"founded": 1976}, - "relationships": [{"type": "founded_by", "target": "Steve Jobs"}], - "embedding": [...] # Similar embedding -} - -# Multi-factor similarity (combines all methods) -result = calculator.calculate_similarity(entity1, entity2) - -print(f"Overall similarity: {result.score:.4f}") -print(f"Components: {result.components}") -# { -# "string": 0.85, -# "property": 1.0, -# "relationship": 1.0, -# "embedding": 0.92 -# } -print(f"Weights: {result.metadata['weights']}") -``` - -**Method 9: Duplicate Detection with Thresholds** -```python -from semantica.deduplication import DuplicateDetector - -# Initialize with similarity threshold -detector = DuplicateDetector( - similarity_threshold=0.8, # Entities with >= 0.8 similarity are duplicates - confidence_threshold=0.7 # Minimum confidence for duplicate candidates -) - -entities = [ - {"id": "1", "name": "Apple Inc."}, - {"id": "2", "name": "Apple"}, - {"id": "3", "name": "Microsoft"}, - {"id": "4", "name": "Apple Corporation"} -] - -# Detect duplicates -candidates = detector.detect_duplicates(entities, threshold=0.8) - -for candidate in candidates: - print(f"Duplicate pair:") - print(f" {candidate.entity1['name']} <-> {candidate.entity2['name']}") - print(f" Similarity: {candidate.similarity_score:.4f}") - print(f" Confidence: {candidate.confidence:.4f}") - print(f" Reasons: {candidate.reasons}") -``` - -**Method 10: Duplicate Group Detection (Clustering)** -```python -from semantica.deduplication import DuplicateDetector - -detector = DuplicateDetector(similarity_threshold=0.7) - -entities = [ - {"id": "1", "name": "Apple Inc."}, - {"id": "2", "name": "Apple"}, - {"id": "3", "name": "Apple Corp"}, - {"id": "4", "name": "Microsoft"} -] - -# Detect duplicate groups (clusters) -groups = detector.detect_duplicate_groups(entities, threshold=0.7) - -for group in groups: - print(f"Duplicate group: {len(group.entities)} entities") - print(f" Confidence: {group.confidence:.4f}") - print(f" Representative: {group.representative['name']}") - for entity in group.entities: - print(f" - {entity['name']}") -``` - -**Method 11: Incremental Duplicate Detection** -```python -from semantica.deduplication import DuplicateDetector - -detector = DuplicateDetector(similarity_threshold=0.8) - -# Existing entities in knowledge graph -existing_entities = [ - {"id": "1", "name": "Apple Inc."}, - {"id": "2", "name": "Microsoft"} -] - -# New entities to add -new_entities = [ - {"id": "3", "name": "Apple"}, - {"id": "4", "name": "Google"} -] - -# Incremental detection (only compare new vs existing) -candidates = detector.incremental_detect( - new_entities, - existing_entities, - threshold=0.8 -) - -# More efficient than full duplicate detection -for candidate in candidates: - print(f"New entity '{candidate.entity1['name']}' " - f"duplicates existing '{candidate.entity2['name']}'") -``` - -**Best Practices:** - -1. **Choose similarity method based on data type**: - - Names/Addresses → Jaro-Winkler - - General text → Levenshtein or Cosine - - Semantic meaning → Embedding similarity - - Production → Multi-factor - -2. **Set appropriate thresholds**: - - Too low: False positives (non-duplicates marked as duplicates) - - Too high: False negatives (duplicates missed) - - Recommended: 0.7-0.8 for most use cases - -3. **Use multi-factor similarity** - Combines multiple signals for better accuracy -4. **Use incremental detection** - More efficient for streaming/updates -5. **Review duplicate groups** - Validate before merging -6. **Track confidence scores** - Use for quality control - ---- - -## Additional Modules - -Due to the extensive number of modules (234 Python files), the above covers the core and most commonly used modules. The remaining modules follow similar patterns: - -- **Normalization Modules**: Data cleaning, text normalization, entity normalization, date/number normalization -- **Semantic Extraction Modules**: NER, relation extraction, triple extraction, event detection -- **Reasoning Modules**: Inference engines, rule managers, deductive/abductive reasoning -- **Vector Store Modules**: Adapters for FAISS, Pinecone, Weaviate, Qdrant, Milvus -- **Triple Store Modules**: Adapters for Jena, Virtuoso, Blazegraph, RDF4J -- **Export Modules**: Exporters for JSON, CSV, RDF, OWL, YAML formats -- **Visualization Modules**: Graph visualizers, ontology visualizers, quality visualizers -- **Quality Assurance Modules**: KG quality assessment, validation engines, automated fixes -- **Context Modules**: Context retrieval, entity linking, agent memory -- **Deduplication Modules**: Duplicate detection, entity merging, similarity calculation -- **Conflict Modules**: Conflict detection, resolution, analysis -- **Split Modules**: Text chunking (semantic, structural, sliding window) -- **Ontology Modules**: Ontology generation, validation, versioning -- **Utils Modules**: Helper functions, validators, constants, exceptions - -Each module typically follows this structure: -- `__init__()`: Initialize with optional config -- Main processing methods: Process data with various options -- Utility methods: Helper functions for specific operations -- Configuration methods: Get/set configuration - -For detailed documentation of specific modules not covered above, please refer to the individual module files or request specific module documentation. - ---- - -## Summary - -This framework provides a comprehensive semantic processing pipeline with: - -- **26 main modules** covering all aspects of semantic data processing -- **234 Python files** with detailed implementations -- **Modular architecture** allowing flexible composition -- **Extensive configuration** options for customization -- **Multiple adapters** for various storage and processing backends -- **Quality assurance** and validation throughout -- **Temporal support** for time-aware knowledge graphs -- **Multi-modal processing** for text, images, and audio - -All modules are designed with consistent interfaces, comprehensive error handling, and extensive logging capabilities. - diff --git a/docs/api.md b/docs/api.md deleted file mode 100644 index 6bdc1ac4..00000000 --- a/docs/api.md +++ /dev/null @@ -1,295 +0,0 @@ -# API Reference - -Complete API documentation for Semantica. - -## Core Classes - -### Semantica - -Main framework class for building semantic layers and knowledge graphs. - -```python -from semantica import Semantica - -semantica = Semantica(config=None) -``` - -#### Methods - -##### `build_knowledge_base(sources, **kwargs)` - -Build a knowledge base from one or more data sources. - -**Parameters:** -- `sources` (List[str] | str): Data source(s) - file paths or URLs -- `embeddings` (bool): Generate embeddings (default: True) -- `graph` (bool): Build knowledge graph (default: True) -- `normalize` (bool): Normalize data (default: True) - -**Returns:** -- `Dict[str, Any]`: Dictionary containing: - - `knowledge_graph`: Knowledge graph data - - `embeddings`: Embedding vectors - - `metadata`: Processing metadata - - `statistics`: Processing statistics - -**Example:** -```python -result = semantica.build_knowledge_base( - sources=["document.pdf"], - embeddings=True, - graph=True -) -kg = result["knowledge_graph"] -``` - -##### `process_document(source)` - -Process a single document. - -**Parameters:** -- `source` (str): File path or URL - -**Returns:** -- `Dict[str, Any]`: Processed document data - -##### `extract_entities(text)` - -Extract entities from text. - -**Parameters:** -- `text` (str): Input text - -**Returns:** -- `Dict[str, List]`: Dictionary with `entities` list - -##### `extract_relationships(text)` - -Extract relationships from text. - -**Parameters:** -- `text` (str): Input text - -**Returns:** -- `Dict[str, List]`: Dictionary with `relationships` list - ---- - -## Knowledge Graph Module - -### `semantica.kg` - -Knowledge graph construction and analysis. - -#### Methods - -##### `build_graph(sources)` - -Build a knowledge graph from sources. - -```python -kg = semantica.kg.build_graph(["document.pdf"]) -``` - -##### `analyze(graph)` - -Analyze a knowledge graph. - -```python -analysis = semantica.kg.analyze(kg) -print(analysis["statistics"]) -``` - -##### `visualize(graph, output_path=None)` - -Visualize a knowledge graph. - -```python -semantica.kg.visualize(kg, output_path="graph.html") -``` - -##### `merge(graphs)` - -Merge multiple knowledge graphs. - -```python -merged = semantica.kg.merge([kg1, kg2, kg3]) -``` - ---- - -## Semantic Extraction Module - -### `semantica.semantic_extract` - -Entity and relationship extraction. - -#### Methods - -##### `extract_entities(text)` - -Extract named entities from text. - -```python -result = semantica.semantic_extract.extract_entities(text) -entities = result["entities"] -``` - -##### `extract_relationships(text)` - -Extract relationships from text. - -```python -result = semantica.semantic_extract.extract_relationships(text) -relationships = result["relationships"] -``` - -##### `extract_triples(text)` - -Extract subject-predicate-object triples. - -```python -result = semantica.semantic_extract.extract_triples(text) -triples = result["triples"] -``` - ---- - -## Embeddings Module - -### `semantica.embeddings` - -Embedding generation and management. - -#### Methods - -##### `generate(text)` - -Generate embedding for a single text. - -```python -embedding = semantica.embeddings.generate("Your text here") -``` - -##### `generate_batch(texts)` - -Generate embeddings for multiple texts. - -```python -texts = ["text1", "text2", "text3"] -embeddings = semantica.embeddings.generate_batch(texts) -``` - ---- - -## Export Module - -### `semantica.export` - -Export knowledge graphs to various formats. - -#### Methods - -##### `to_rdf(kg, path)` - -Export to RDF format. - -```python -semantica.export.to_rdf(kg, "output.rdf") -``` - -##### `to_json(kg, path)` - -Export to JSON format. - -```python -semantica.export.to_json(kg, "output.json") -``` - -##### `to_csv(kg, path)` - -Export to CSV format. - -```python -semantica.export.to_csv(kg, "output.csv") -``` - -##### `to_owl(kg, path)` - -Export to OWL ontology format. - -```python -semantica.export.to_owl(kg, "output.owl") -``` - -##### `to_yaml(kg, path)` - -Export to YAML format. - -```python -semantica.export.to_yaml(kg, "output.yaml") -``` - ---- - -## Conflict Resolution Module - -### `semantica.conflicts` - -Conflict detection and resolution. - -#### Classes - -##### `ConflictResolver` - -```python -from semantica.conflicts import ConflictResolver - -resolver = ConflictResolver(default_strategy="voting") -``` - -**Methods:** -- `resolve_conflicts(conflicts)`: Resolve multiple conflicts -- `resolve_conflict(conflict, strategy=None)`: Resolve a single conflict -- `set_resolution_rule(property, strategy)`: Set custom resolution rules - -**Example:** -```python -resolver = ConflictResolver(default_strategy="voting") -resolved = resolver.resolve_conflicts(conflicts) -``` - ---- - -## Configuration - -### `Config` - -Configuration class for Semantica. - -```python -from semantica import Config - -config = Config( - embeddings=True, - graph=True, - normalize=True, - conflict_resolution="voting" -) - -semantica = Semantica(config=config) -``` - -**Parameters:** -- `embeddings` (bool): Enable embedding generation -- `graph` (bool): Enable knowledge graph construction -- `normalize` (bool): Enable data normalization -- `conflict_resolution` (str): Default conflict resolution strategy - ---- - -## Full Documentation - -For complete module documentation, see: -- [MODULES_DOCUMENTATION.md](../MODULES_DOCUMENTATION.md) - Detailed module documentation -- [GitHub Repository](https://github.com/Hawksight-AI/semantica) - Source code diff --git a/docs/cookbook.md b/docs/cookbook.md deleted file mode 100644 index 1ea6754a..00000000 --- a/docs/cookbook.md +++ /dev/null @@ -1,182 +0,0 @@ -# Cookbook Recipes - -Interactive Jupyter notebooks with hands-on examples and tutorials. - -## Introduction - -Get started with Semantica through these beginner-friendly tutorials. - -### Getting Started - -- **[Welcome to Semantica](cookbook/introduction/Welcome_to_Semantica.ipynb)** - Introduction to the framework -- **[Your First Knowledge Graph](cookbook/introduction/Your_First_Knowledge_Graph.ipynb)** - Build your first KG -- **[Configuration Basics](cookbook/introduction/Configuration_Basics.ipynb)** - Learn configuration options - -### Core Concepts - -- **[Data Ingestion](cookbook/introduction/Data_Ingestion.ipynb)** - Ingest data from various sources -- **[Document Parsing](cookbook/introduction/Document_Parsing.ipynb)** - Parse different document formats -- **[Data Normalization](cookbook/introduction/Data_Normalization.ipynb)** - Normalize and clean data -- **[Entity Extraction](cookbook/introduction/Entity_Extraction.ipynb)** - Extract entities from text -- **[Relation Extraction](cookbook/introduction/Relation_Extraction.ipynb)** - Extract relationships -- **[Building Knowledge Graphs](cookbook/introduction/Building_Knowledge_Graphs.ipynb)** - Construct KGs - -### Quality and Analysis - -- **[Conflict Detection](cookbook/introduction/Conflict_Detection.ipynb)** - Detect data conflicts -- **[Deduplication](cookbook/introduction/Deduplication.ipynb)** - Remove duplicates -- **[Graph Quality](cookbook/introduction/Graph_Quality.ipynb)** - Assess KG quality -- **[Graph Analytics](cookbook/introduction/Graph_Analytics.ipynb)** - Analyze knowledge graphs - -### Advanced Features - -- **[Embedding Generation](cookbook/introduction/Embedding_Generation.ipynb)** - Generate embeddings -- **[Vector Store](cookbook/introduction/Vector_Store.ipynb)** - Store and query vectors -- **[Ontology](cookbook/introduction/Ontology.ipynb)** - Work with ontologies -- **[Visualization](cookbook/introduction/Visualization.ipynb)** - Visualize knowledge graphs -- **[Export](cookbook/introduction/Export.ipynb)** - Export in various formats - -## Advanced - -Advanced techniques and patterns for experienced users. - -### Advanced Extraction - -- **[Advanced Extraction](cookbook/advanced/Advanced_Extraction.ipynb)** - Advanced extraction techniques -- **[Text Chunking Strategies](cookbook/advanced/Text_Chunking_Strategies.ipynb)** - Optimize text chunking - -### Graph Operations - -- **[Advanced Graph Analytics](cookbook/advanced/Advanced_Graph_Analytics.ipynb)** - Advanced graph analysis -- **[Temporal Knowledge Graphs](cookbook/advanced/Temporal_Knowledge_Graphs.ipynb)** - Work with temporal data -- **[Semantic Layer Construction](cookbook/advanced/Semantic_Layer_Construction.ipynb)** - Build semantic layers - -### Integration and Processing - -- **[Multi-Source Data Integration](cookbook/advanced/Multi_Source_Data_Integration.ipynb)** - Integrate multiple sources -- **[Pipeline Orchestration](cookbook/advanced/Pipeline_Orchestration.ipynb)** - Orchestrate complex pipelines -- **[Unstructured to Ontology](cookbook/advanced/Unstructured_to_Ontology.ipynb)** - Convert unstructured to ontology - -### Quality and Resolution - -- **[Conflict Resolution Strategies](cookbook/advanced/Conflict_Resolution_Strategies.ipynb)** - Advanced conflict resolution -- **[Reasoning and Inference](cookbook/advanced/Reasoning_and_Inference.ipynb)** - Perform reasoning - -### Visualization and Export - -- **[Complete Visualization Suite](cookbook/advanced/Complete_Visualization_Suite.ipynb)** - Comprehensive visualization -- **[Multi-Format Export](cookbook/advanced/Multi_Format_Export.ipynb)** - Export to multiple formats - -## Use Cases - -Real-world applications across various domains. - -### Advanced RAG - -- **[GraphRAG Complete](cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb)** - Complete GraphRAG implementation - -### Biomedical - -- **[Drug Discovery Pipeline](cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb)** - Drug discovery workflows -- **[Genomic Variant Analysis](cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb)** - Genomic data analysis - -### Blockchain - -- **[DeFi Protocol Intelligence](cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb)** - DeFi analysis -- **[Transaction Network Analysis](cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb)** - Blockchain network analysis - -### Cybersecurity - -- **[Threat Intelligence Integration](cookbook/use_cases/cybersecurity/Threat_Intelligence_Integration.ipynb)** - Threat intelligence -- **[Threat Intelligence Hybrid RAG](cookbook/use_cases/cybersecurity/Threat_Intelligence_Hybrid_RAG.ipynb)** - Hybrid RAG for threats -- **[Threat Correlation](cookbook/use_cases/cybersecurity/Threat_Correlation.ipynb)** - Correlate threats -- **[Anomaly Detection Real-Time](cookbook/use_cases/cybersecurity/Anomaly_Detection_Real_Time.ipynb)** - Real-time anomaly detection -- **[Incident Analysis](cookbook/use_cases/cybersecurity/Incident_Analysis.ipynb)** - Analyze security incidents -- **[Vulnerability Tracking](cookbook/use_cases/cybersecurity/Vulnerability_Tracking.ipynb)** - Track vulnerabilities - -### Finance - -- **[Financial Data Integration](cookbook/use_cases/finance/Financial_Data_Integration.ipynb)** - Integrate financial data -- **[Financial Reports Analysis](cookbook/use_cases/finance/Financial_Reports_Analysis.ipynb)** - Analyze reports -- **[Fraud Detection](cookbook/use_cases/finance/Fraud_Detection.ipynb)** - Detect fraud -- **[Investment Analysis Hybrid RAG](cookbook/use_cases/finance/Investment_Analysis_Hybrid_RAG.ipynb)** - Investment analysis -- **[Market Intelligence](cookbook/use_cases/finance/Market_Intelligence.ipynb)** - Market analysis -- **[Regulatory Compliance](cookbook/use_cases/finance/Regulatory_Compliance.ipynb)** - Compliance workflows - -### Healthcare - -- **[Clinical Reports Processing](cookbook/use_cases/healthcare/Clinical_Reports_Processing.ipynb)** - Process clinical data -- **[Disease Network Analysis](cookbook/use_cases/healthcare/Disease_Network_Analysis.ipynb)** - Analyze disease networks -- **[Drug Interactions Analysis](cookbook/use_cases/healthcare/Drug_Interactions_Analysis.ipynb)** - Drug interaction analysis -- **[Healthcare GraphRAG Hybrid](cookbook/use_cases/healthcare/Healthcare_GraphRAG_Hybrid.ipynb)** - Healthcare GraphRAG -- **[Medical Database Integration](cookbook/use_cases/healthcare/Medical_Database_Integration.ipynb)** - Integrate medical databases -- **[Medical Literature GraphRAG](cookbook/use_cases/healthcare/Medical_Literature_GraphRAG.ipynb)** - Literature analysis -- **[Patient Records Temporal](cookbook/use_cases/healthcare/Patient_Records_Temporal.ipynb)** - Temporal patient data - -### Intelligence - -- **[Network Analysis Intelligence Reports](cookbook/use_cases/intelligence/Network_Analysis_Intelligence_Reports.ipynb)** - Intelligence network analysis - -### Renewable Energy - -- **[Energy Market Analysis](cookbook/use_cases/renewable_energy/Energy_Market_Analysis.ipynb)** - Energy market insights -- **[Environmental Impact](cookbook/use_cases/renewable_energy/Environmental_Impact.ipynb)** - Environmental analysis -- **[Grid Management](cookbook/use_cases/renewable_energy/Grid_Management.ipynb)** - Grid optimization -- **[Resource Optimization](cookbook/use_cases/renewable_energy/Resource_Optimization.ipynb)** - Optimize resources -- **[Supply Chain Analysis](cookbook/use_cases/renewable_energy/Supply_Chain_Analysis.ipynb)** - Energy supply chains - -### Supply Chain - -- **[Supply Chain Data Integration](cookbook/use_cases/supply_chain/Supply_Chain_Data_Integration.ipynb)** - Integrate supply chain data -- **[Supply Chain Risk Management](cookbook/use_cases/supply_chain/Supply_Chain_Risk_Management.ipynb)** - Risk management - -### Trading - -- **[Market Data Analysis](cookbook/use_cases/trading/Market_Data_Analysis.ipynb)** - Analyze market data -- **[News Sentiment Analysis](cookbook/use_cases/trading/News_Sentiment_Analysis.ipynb)** - Sentiment analysis -- **[Real-Time Market Data](cookbook/use_cases/trading/Real_Time_Market_Data.ipynb)** - Real-time processing -- **[Real-Time Monitoring](cookbook/use_cases/trading/Real_Time_Monitoring.ipynb)** - Monitor markets -- **[Risk Assessment](cookbook/use_cases/trading/Risk_Assessment.ipynb)** - Assess risks -- **[Strategy Backtesting](cookbook/use_cases/trading/Strategy_Backtesting.ipynb)** - Backtest strategies - -## Running the Notebooks - -### Prerequisites - -```bash -# Install Semantica -pip install semantica - -# Install Jupyter -pip install jupyter notebook - -# Optional: Install JupyterLab -pip install jupyterlab -``` - -### Launch Jupyter - -```bash -# Start Jupyter Notebook -jupyter notebook - -# Or start JupyterLab -jupyter lab -``` - -Navigate to the `cookbook/` directory and open any notebook. - -### Viewing on GitHub - -All notebooks can be viewed directly on GitHub. Click any notebook link above to view it online. - -## Contributing - -Have a use case or example to share? We welcome contributions! - -1. Create a new notebook in the appropriate category -2. Follow the existing notebook structure -3. Submit a pull request - -See our [Contributing Guide](https://github.com/Hawksight-AI/semantica/blob/main/CONTRIBUTING.md) for details. - diff --git a/docs/cookbook/advanced/Advanced_Extraction.ipynb b/docs/cookbook/advanced/Advanced_Extraction.ipynb new file mode 100644 index 00000000..05c8ef7f --- /dev/null +++ b/docs/cookbook/advanced/Advanced_Extraction.ipynb @@ -0,0 +1,211 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced Extraction\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates advanced semantic extraction using EventDetector, CoreferenceResolver, TripleExtractor, SemanticAnalyzer, SemanticNetworkExtractor, LLMEnhancer, and ExtractionValidator.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use EventDetector to detect events\n", + "- Use CoreferenceResolver to resolve coreferences\n", + "- Use TripleExtractor to extract RDF triples\n", + "- Use SemanticAnalyzer for semantic analysis\n", + "- Use SemanticNetworkExtractor to extract semantic networks\n", + "- Use LLMEnhancer for LLM-based enhancement\n", + "- Use ExtractionValidator to validate extractions\n", + "\n", + "---\n", + "\n", + "## Workflow: Event Detection → Coreference Resolution → Triple Extraction → Semantic Analysis → Network Extraction → LLM Enhancement → Validation\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import (\n", + " EventDetector, CoreferenceResolver, TripleExtractor,\n", + " SemanticAnalyzer, SemanticNetworkExtractor, LLMEnhancer, ExtractionValidator\n", + ")\n", + "\n", + "text = \"Apple Inc. was founded by Steve Jobs in 1976. The company is now led by Tim Cook.\"\n", + "\n", + "event_detector = EventDetector()\n", + "events = event_detector.detect_events(text)\n", + "\n", + "print(f\"Detected {len(events)} events\")\n", + "for event in events[:3]:\n", + " print(f\" Event: {event.get('type', 'Unknown')} - {event.get('text', '')[:50]}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Coreference Resolution\n", + "\n", + "Resolve coreferences in text.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coreference_resolver = CoreferenceResolver()\n", + "\n", + "coreferences = coreference_resolver.resolve(text)\n", + "\n", + "print(f\"Resolved {len(coreferences)} coreference chains\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Triple Extraction\n", + "\n", + "Extract RDF triples.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "triple_extractor = TripleExtractor()\n", + "\n", + "triples = triple_extractor.extract_triples(text)\n", + "\n", + "print(f\"Extracted {len(triples)} triples\")\n", + "for triple in triples[:3]:\n", + " print(f\" ({triple.get('subject', '')}, {triple.get('predicate', '')}, {triple.get('object', '')})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Semantic Analysis\n", + "\n", + "Perform semantic analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "semantic_roles = semantic_analyzer.analyze_semantic_roles(text)\n", + "\n", + "print(f\"Analyzed semantic roles: {len(semantic_roles)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Semantic Network Extraction\n", + "\n", + "Extract semantic networks.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "semantic_network_extractor = SemanticNetworkExtractor()\n", + "\n", + "semantic_network = semantic_network_extractor.extract_network(text)\n", + "\n", + "print(f\"Extracted semantic network with {len(semantic_network.get('nodes', []))} nodes\")\n", + "print(f\"Edges: {len(semantic_network.get('edges', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: LLM Enhancement\n", + "\n", + "Enhance extractions using LLM.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm_enhancer = LLMEnhancer()\n", + "\n", + "enhanced_extractions = llm_enhancer.enhance_extractions(events, text)\n", + "\n", + "print(f\"Enhanced {len(enhanced_extractions)} extractions\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Extraction Validation\n", + "\n", + "Validate extractions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "extraction_validator = ExtractionValidator()\n", + "\n", + "validation_result = extraction_validator.validate(events, text)\n", + "\n", + "print(f\"Extraction validation:\")\n", + "print(f\" Valid: {validation_result.valid}\")\n", + "print(f\" Confidence: {validation_result.confidence:.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned advanced extraction capabilities:\n", + "\n", + "- **EventDetector**: Event detection and classification\n", + "- **CoreferenceResolver**: Coreference resolution\n", + "- **TripleExtractor**: RDF triple extraction\n", + "- **SemanticAnalyzer**: Semantic analysis and role labeling\n", + "- **SemanticNetworkExtractor**: Semantic network extraction\n", + "- **LLMEnhancer**: LLM-based extraction enhancement\n", + "- **ExtractionValidator**: Extraction validation\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Advanced_Graph_Analytics.ipynb b/docs/cookbook/advanced/Advanced_Graph_Analytics.ipynb new file mode 100644 index 00000000..c113f5b0 --- /dev/null +++ b/docs/cookbook/advanced/Advanced_Graph_Analytics.ipynb @@ -0,0 +1,179 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced Graph Analytics\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates advanced graph analytics using GraphAnalyzer, CentralityCalculator, CommunityDetector, ConnectivityAnalyzer, GraphValidator, and Deduplicator.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use GraphAnalyzer for comprehensive graph analysis\n", + "- Use CentralityCalculator for advanced centrality measures\n", + "- Use CommunityDetector for community detection\n", + "- Use ConnectivityAnalyzer for connectivity analysis\n", + "- Use GraphValidator and Deduplicator for graph quality\n", + "\n", + "---\n", + "\n", + "## Workflow: Graph Analysis → Centrality → Communities → Connectivity → Validation → Deduplication\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector, ConnectivityAnalyzer, GraphValidator, Deduplicator\n", + "\n", + "builder = GraphBuilder()\n", + "analyzer = GraphAnalyzer()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\", \"properties\": {}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Tim Cook\", \"properties\": {}},\n", + " {\"id\": \"e3\", \"type\": \"Location\", \"name\": \"Cupertino\", \"properties\": {}}\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e2\", \"target\": \"e1\", \"type\": \"CEO_of\", \"properties\": {}},\n", + " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"located_in\", \"properties\": {}}\n", + "]\n", + "\n", + "kg = builder.build(entities, relationships)\n", + "\n", + "metrics = analyzer.compute_metrics(kg)\n", + "\n", + "print(f\"Graph metrics:\")\n", + "print(f\" Entities: {metrics.get('entity_count', 0)}\")\n", + "print(f\" Relationships: {metrics.get('relationship_count', 0)}\")\n", + "print(f\" Density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Advanced Centrality Measures\n", + "\n", + "Calculate multiple centrality measures.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "centrality_calculator = CentralityCalculator()\n", + "\n", + "degree_centrality = centrality_calculator.calculate_centrality(kg, measure=\"degree\")\n", + "betweenness_centrality = centrality_calculator.calculate_centrality(kg, measure=\"betweenness\")\n", + "\n", + "print(f\"Degree centrality: {len(degree_centrality)} entities\")\n", + "print(f\"Betweenness centrality: {len(betweenness_centrality)} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Community Detection\n", + "\n", + "Detect communities in the graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "community_detector = CommunityDetector()\n", + "\n", + "communities = community_detector.detect_communities(kg)\n", + "\n", + "print(f\"Detected {len(communities)} communities\")\n", + "for i, community in enumerate(communities[:3], 1):\n", + " print(f\" Community {i}: {len(community)} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Connectivity Analysis\n", + "\n", + "Analyze graph connectivity.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "connectivity = connectivity_analyzer.analyze_connectivity(kg)\n", + "\n", + "print(f\"Connectivity analysis:\")\n", + "print(f\" Is connected: {connectivity.get('is_connected', False)}\")\n", + "print(f\" Components: {len(connectivity.get('components', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Graph Validation and Deduplication\n", + "\n", + "Validate and deduplicate the graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_validator = GraphValidator()\n", + "deduplicator = Deduplicator()\n", + "\n", + "validation_result = graph_validator.validate(kg)\n", + "deduplicated_kg = deduplicator.deduplicate(kg)\n", + "\n", + "print(f\"Graph validation: {validation_result.get('valid', False)}\")\n", + "print(f\"Deduplicated entities: {len(deduplicated_kg.get('entities', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned advanced graph analytics:\n", + "\n", + "- **GraphAnalyzer**: Comprehensive graph analysis and metrics\n", + "- **CentralityCalculator**: Multiple centrality measures\n", + "- **CommunityDetector**: Community detection\n", + "- **ConnectivityAnalyzer**: Connectivity analysis\n", + "- **GraphValidator**: Graph validation\n", + "- **Deduplicator**: Graph deduplication\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Complete_Visualization_Suite.ipynb b/docs/cookbook/advanced/Complete_Visualization_Suite.ipynb new file mode 100644 index 00000000..1f6e5f15 --- /dev/null +++ b/docs/cookbook/advanced/Complete_Visualization_Suite.ipynb @@ -0,0 +1,250 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Complete Visualization Suite\n", + "\n", + "## Overview\n", + "\n", + "Comprehensive visualization capabilities: visualize knowledge graphs, embeddings, quality metrics, analytics, and temporal data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import (\n", + " KGVisualizer,\n", + " EmbeddingVisualizer,\n", + " QualityVisualizer,\n", + " AnalyticsVisualizer,\n", + " TemporalVisualizer\n", + ")\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "import numpy as np\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Create Sample Knowledge Graph\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35}},\n", + " {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n", + " {\"id\": \"e4\", \"type\": \"Location\", \"name\": \"San Francisco\", \"properties\": {\"country\": \"USA\"}},\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"knows\", \"properties\": {\"since\": 2020}},\n", + " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\", \"properties\": {\"role\": \"Engineer\"}},\n", + " {\"source\": \"e3\", \"target\": \"e4\", \"type\": \"located_in\", \"properties\": {}},\n", + "]\n", + "\n", + "knowledge_graph = builder.build(entities, relationships)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Knowledge Graph Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "kg_visualizer.visualize(knowledge_graph, layout=\"spring\", show_labels=True)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Generate Embeddings and Visualize\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_generator = EmbeddingGenerator()\n", + "texts = [entity.get(\"name\", \"\") for entity in entities]\n", + "embeddings = embedding_generator.generate(texts)\n", + "\n", + "labels = [entity.get(\"type\", \"Unknown\") for entity in entities]\n", + "\n", + "embedding_visualizer = EmbeddingVisualizer()\n", + "embedding_visualizer.visualize_tsne(embeddings, labels, title=\"Entity Embeddings Visualization\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Quality Metrics Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "quality_metrics = quality_assessor.assess(knowledge_graph)\n", + "\n", + "quality_visualizer = QualityVisualizer()\n", + "quality_visualizer.visualize_metrics(quality_metrics, title=\"Knowledge Graph Quality Metrics\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Graph Analytics Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "centrality_results = graph_analyzer.calculate_centrality(\n", + " knowledge_graph, \n", + " centrality_type=\"degree\"\n", + ")\n", + "\n", + "centrality_scores = {}\n", + "if centrality_results and \"centrality_measures\" in centrality_results:\n", + " degree_centrality = centrality_results[\"centrality_measures\"].get(\"degree\", {})\n", + " if isinstance(degree_centrality, dict) and \"centrality\" in degree_centrality:\n", + " centrality_scores = degree_centrality[\"centrality\"]\n", + " elif isinstance(degree_centrality, dict):\n", + " centrality_scores = degree_centrality\n", + "\n", + "communities_result = graph_analyzer.detect_communities(\n", + " knowledge_graph, \n", + " algorithm=\"louvain\"\n", + ")\n", + "\n", + "communities = []\n", + "community_dict = {}\n", + "if communities_result and \"communities\" in communities_result:\n", + " communities_data = communities_result[\"communities\"]\n", + " if isinstance(communities_data, list):\n", + " communities = communities_data\n", + " for idx, community in enumerate(communities):\n", + " if isinstance(community, list):\n", + " for node in community:\n", + " community_dict[node] = idx\n", + " elif isinstance(community, dict) and \"nodes\" in community:\n", + " for node in community[\"nodes\"]:\n", + " community_dict[node] = idx\n", + "\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "analytics_visualizer.visualize_centrality(centrality_scores, title=\"Node Centrality Scores\")\n", + "\n", + "if community_dict:\n", + " analytics_visualizer.visualize_communities(\n", + " knowledge_graph, \n", + " community_dict, \n", + " title=\"Community Detection\"\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Temporal Data Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temporal_kg = {\n", + " \"entities\": entities,\n", + " \"relationships\": relationships,\n", + " \"timestamps\": {\n", + " \"e1\": [2020, 2021, 2022],\n", + " \"e2\": [2020, 2021],\n", + " \"e3\": [2010, 2015, 2020, 2022],\n", + " }\n", + "}\n", + "\n", + "entity_history = {\n", + " \"e1\": [\n", + " {\"timestamp\": 2020, \"properties\": {\"age\": 28}},\n", + " {\"timestamp\": 2021, \"properties\": {\"age\": 29}},\n", + " {\"timestamp\": 2022, \"properties\": {\"age\": 30}},\n", + " ]\n", + "}\n", + "\n", + "temporal_visualizer = TemporalVisualizer()\n", + "temporal_visualizer.visualize_timeline(temporal_kg, title=\"Temporal Knowledge Graph Timeline\")\n", + "temporal_visualizer.visualize_evolution(entity_history, entity_id=\"e1\", title=\"Entity Evolution\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "All visualization types demonstrated:\n", + "- Knowledge Graph Visualization\n", + "- Embedding Visualization (t-SNE)\n", + "- Quality Metrics Visualization\n", + "- Graph Analytics Visualization (Centrality & Communities)\n", + "- Temporal Data Visualization (Timeline & Evolution)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Complete Visualization Suite\")\n", + "print(\"All visualizations generated successfully\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Conflict_Resolution_Strategies.ipynb b/docs/cookbook/advanced/Conflict_Resolution_Strategies.ipynb new file mode 100644 index 00000000..3cb560f4 --- /dev/null +++ b/docs/cookbook/advanced/Conflict_Resolution_Strategies.ipynb @@ -0,0 +1,313 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conflict Resolution Strategies\n", + "\n", + "## Overview\n", + "\n", + "Detect conflicts in knowledge graphs, apply multiple resolution strategies, track sources, and maintain audit trails.\n", + "\n", + "## Workflow: Detect Conflicts → Multiple Resolution Strategies → Track Sources → Audit\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "from semantica.kg_qa import ConsistencyChecker\n", + "from datetime import datetime\n", + "import json\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Create Knowledge Graph with Conflicting Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\n", + " \"id\": \"e1\",\n", + " \"type\": \"Person\",\n", + " \"name\": \"John Doe\",\n", + " \"properties\": {\"age\": 30, \"location\": \"New York\"},\n", + " \"source\": \"source1\",\n", + " \"timestamp\": datetime(2023, 1, 1)\n", + " },\n", + " {\n", + " \"id\": \"e1\",\n", + " \"type\": \"Person\",\n", + " \"name\": \"John Doe\",\n", + " \"properties\": {\"age\": 32, \"location\": \"Boston\"},\n", + " \"source\": \"source2\",\n", + " \"timestamp\": datetime(2023, 6, 1)\n", + " },\n", + " {\n", + " \"id\": \"e2\",\n", + " \"type\": \"Organization\",\n", + " \"name\": \"Tech Corp\",\n", + " \"properties\": {\"founded\": 2010, \"employees\": 100},\n", + " \"source\": \"source1\",\n", + " \"timestamp\": datetime(2023, 1, 1)\n", + " },\n", + " {\n", + " \"id\": \"e2\",\n", + " \"type\": \"Organization\",\n", + " \"name\": \"Tech Corp\",\n", + " \"properties\": {\"founded\": 2012, \"employees\": 150},\n", + " \"source\": \"source2\",\n", + " \"timestamp\": datetime(2023, 3, 1)\n", + " },\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"works_for\", \"source\": \"source1\"},\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"founder_of\", \"source\": \"source2\"},\n", + "]\n", + "\n", + "knowledge_graph = builder.build(entities, relationships)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Detect Conflicts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "consistency_checker = ConsistencyChecker()\n", + "conflicts = consistency_checker.check_conflicts(knowledge_graph)\n", + "\n", + "for i, conflict in enumerate(conflicts, 1):\n", + " print(f\"Conflict {i}:\")\n", + " print(f\" Entity/Relationship: {conflict.get('entity_id', conflict.get('relationship_id'))}\")\n", + " print(f\" Type: {conflict.get('type')}\")\n", + " print(f\" Conflicting values: {conflict.get('values')}\")\n", + " print(f\" Sources: {conflict.get('sources')}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Multiple Resolution Strategies\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class ConflictResolver:\n", + " def __init__(self):\n", + " self.audit_trail = []\n", + " \n", + " def resolve(self, conflicts, strategy=\"most_recent\"):\n", + " resolved = []\n", + " \n", + " for conflict in conflicts:\n", + " if strategy == \"most_recent\":\n", + " values = conflict.get('values', [])\n", + " timestamps = conflict.get('timestamps', [])\n", + " if timestamps:\n", + " most_recent_idx = timestamps.index(max(timestamps))\n", + " resolved_value = values[most_recent_idx]\n", + " else:\n", + " resolved_value = values[-1] if values else None\n", + " \n", + " elif strategy == \"authoritative\":\n", + " sources = conflict.get('sources', [])\n", + " authoritative_sources = [\"source1\", \"official_db\", \"verified\"]\n", + " resolved_value = None\n", + " for auth_source in authoritative_sources:\n", + " if auth_source in sources:\n", + " idx = sources.index(auth_source)\n", + " resolved_value = conflict.get('values', [])[idx]\n", + " break\n", + " if resolved_value is None:\n", + " resolved_value = conflict.get('values', [])[0] if conflict.get('values') else None\n", + " \n", + " elif strategy == \"merge\":\n", + " values = conflict.get('values', [])\n", + " if isinstance(values[0], dict):\n", + " merged = {}\n", + " for val in values:\n", + " merged.update(val)\n", + " resolved_value = merged\n", + " elif isinstance(values[0], (int, float)):\n", + " resolved_value = sum(values) / len(values)\n", + " else:\n", + " resolved_value = \", \".join(set(str(v) for v in values))\n", + " else:\n", + " resolved_value = conflict.get('values', [])[0] if conflict.get('values') else None\n", + " \n", + " resolved.append({\n", + " 'conflict_id': conflict.get('entity_id', conflict.get('relationship_id')),\n", + " 'resolved_value': resolved_value,\n", + " 'strategy': strategy,\n", + " 'timestamp': datetime.now()\n", + " })\n", + " \n", + " self.audit_trail.append({\n", + " 'conflict': conflict,\n", + " 'resolution': resolved[-1],\n", + " 'resolved_at': datetime.now()\n", + " })\n", + " \n", + " return resolved\n", + "\n", + "resolver = ConflictResolver()\n", + "\n", + "resolved_1 = resolver.resolve(conflicts, strategy=\"most_recent\")\n", + "print(\"Strategy 1: Most Recent Wins\")\n", + "for r in resolved_1:\n", + " print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n", + "\n", + "resolver2 = ConflictResolver()\n", + "resolved_2 = resolver2.resolve(conflicts, strategy=\"authoritative\")\n", + "print(\"\\nStrategy 2: Most Authoritative Source Wins\")\n", + "for r in resolved_2:\n", + " print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n", + "\n", + "resolver3 = ConflictResolver()\n", + "resolved_3 = resolver3.resolve(conflicts, strategy=\"merge\")\n", + "print(\"\\nStrategy 3: Merge Conflicting Information\")\n", + "for r in resolved_3:\n", + " print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Track Sources\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class SourceTracker:\n", + " def __init__(self):\n", + " self.source_map = {}\n", + " \n", + " def track_sources(self, conflicts):\n", + " for conflict in conflicts:\n", + " conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n", + " sources = conflict.get('sources', [])\n", + " timestamps = conflict.get('timestamps', [])\n", + " \n", + " self.source_map[conflict_id] = {\n", + " 'sources': sources,\n", + " 'timestamps': timestamps,\n", + " 'values': conflict.get('values', [])\n", + " }\n", + " \n", + " def get_sources(self, conflict):\n", + " conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n", + " return self.source_map.get(conflict_id, {})\n", + "\n", + "tracker = SourceTracker()\n", + "tracker.track_sources(conflicts)\n", + "\n", + "for conflict in conflicts:\n", + " sources = tracker.get_sources(conflict)\n", + " conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n", + " print(f\"Conflict: {conflict_id}\")\n", + " print(f\" Sources: {sources.get('sources', [])}\")\n", + " print(f\" Timestamps: {sources.get('timestamps', [])}\")\n", + " print(f\" Values: {sources.get('values', [])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Audit Trail\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "audit_log = resolver.get_audit_trail() if hasattr(resolver, 'get_audit_trail') else resolver.audit_trail\n", + "\n", + "for i, entry in enumerate(audit_log, 1):\n", + " print(f\"Entry {i}:\")\n", + " print(f\" Conflict ID: {entry['conflict'].get('entity_id', entry['conflict'].get('relationship_id'))}\")\n", + " print(f\" Resolution Strategy: {entry['resolution']['strategy']}\")\n", + " print(f\" Resolved Value: {entry['resolution']['resolved_value']}\")\n", + " print(f\" Resolved At: {entry['resolved_at']}\")\n", + "\n", + "audit_export = []\n", + "for entry in audit_log:\n", + " audit_export.append({\n", + " 'conflict_id': entry['conflict'].get('entity_id', entry['conflict'].get('relationship_id')),\n", + " 'conflict_type': entry['conflict'].get('type'),\n", + " 'original_values': entry['conflict'].get('values'),\n", + " 'sources': entry['conflict'].get('sources'),\n", + " 'resolution_strategy': entry['resolution']['strategy'],\n", + " 'resolved_value': str(entry['resolution']['resolved_value']),\n", + " 'resolved_at': entry['resolved_at'].isoformat()\n", + " })\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Conflict resolution workflow:\n", + "- Conflict Detection\n", + "- Multiple Resolution Strategies (Most Recent, Authoritative, Merge)\n", + "- Source Tracking\n", + "- Complete Audit Trail\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Detected {len(conflicts)} conflicts\")\n", + "print(f\"Applied 3 resolution strategies\")\n", + "print(f\"Maintained audit trail with {len(audit_log)} entries\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Multi_Format_Export.ipynb b/docs/cookbook/advanced/Multi_Format_Export.ipynb new file mode 100644 index 00000000..3042d714 --- /dev/null +++ b/docs/cookbook/advanced/Multi_Format_Export.ipynb @@ -0,0 +1,221 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Multi-Format Export\n", + "\n", + "## Overview\n", + "\n", + "Export knowledge graphs and data to multiple formats: JSON, RDF, CSV, Graph formats, OWL, and Vector formats.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import (\n", + " JSONExporter,\n", + " RDFExporter,\n", + " CSVExporter,\n", + " GraphExporter,\n", + " OWLExporter,\n", + " VectorExporter\n", + ")\n", + "from semantica.kg import GraphBuilder\n", + "from semantica.embeddings import EmbeddingGenerator\n", + "from semantica.ontology import OntologyGenerator\n", + "import os\n", + "\n", + "os.makedirs(\"exports\", exist_ok=True)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Create Sample Knowledge Graph and Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35}},\n", + " {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"knows\"},\n", + " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\"},\n", + "]\n", + "\n", + "knowledge_graph = builder.build(entities, relationships)\n", + "\n", + "embedding_generator = EmbeddingGenerator()\n", + "texts = [e[\"name\"] for e in entities]\n", + "embeddings = embedding_generator.generate(texts)\n", + "\n", + "ontology_generator = OntologyGenerator()\n", + "ontology = ontology_generator.generate_from_graph(knowledge_graph)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Export to JSON\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "json_exporter.export(knowledge_graph, \"exports/output.json\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Export to RDF\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rdf_exporter = RDFExporter()\n", + "rdf_exporter.export(knowledge_graph, \"exports/output.rdf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Export to CSV\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "csv_exporter = CSVExporter()\n", + "csv_exporter.export(knowledge_graph, \"exports/output.csv\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Export to Graph Formats (GraphML, GEXF)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_exporter = GraphExporter()\n", + "graph_exporter.export(knowledge_graph, \"exports/output.graphml\", format=\"graphml\")\n", + "graph_exporter.export(knowledge_graph, \"exports/output.gexf\", format=\"gexf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Export to OWL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "owl_exporter = OWLExporter()\n", + "owl_exporter.export(ontology, \"exports/output.owl\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Export to Vector Formats\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vector_exporter = VectorExporter()\n", + "vector_exporter.export(embeddings, \"exports/output.vectors\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Export formats:\n", + "- JSON\n", + "- RDF\n", + "- CSV\n", + "- GraphML\n", + "- GEXF\n", + "- OWL\n", + "- Vector format\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "export_files = [\n", + " \"exports/output.json\",\n", + " \"exports/output.rdf\",\n", + " \"exports/output.csv\",\n", + " \"exports/output.graphml\",\n", + " \"exports/output.gexf\",\n", + " \"exports/output.owl\",\n", + " \"exports/output.vectors\"\n", + "]\n", + "\n", + "for file in export_files:\n", + " if os.path.exists(file):\n", + " size = os.path.getsize(file)\n", + " print(f\"{file} ({size} bytes)\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Multi_Source_Data_Integration.ipynb b/docs/cookbook/advanced/Multi_Source_Data_Integration.ipynb new file mode 100644 index 00000000..c89b66b4 --- /dev/null +++ b/docs/cookbook/advanced/Multi_Source_Data_Integration.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Multi-Source Data Integration\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates advanced multi-source data integration using multiple ingestion types, entity resolution, conflict detection, and provenance tracking.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Ingest data from multiple sources (files, web, databases, streams, feeds)\n", + "- Resolve entities across sources using EntityResolver\n", + "- Detect conflicts using ConflictDetector\n", + "- Track provenance using ProvenanceTracker\n", + "- Integrate data into a unified knowledge graph\n", + "\n", + "---\n", + "\n", + "## Workflow: Multi-Source Ingestion → Entity Resolution → Conflict Detection → Provenance Tracking → Unified KG\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor, FeedIngestor\n", + "from semantica.parse import DocumentParser, StructuredDataParser\n", + "from semantica.kg import GraphBuilder, EntityResolver, ConflictDetector, ProvenanceTracker\n", + "import tempfile\n", + "import os\n", + "import json\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "stream_ingestor = StreamIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "file1 = os.path.join(temp_dir, \"source1.txt\")\n", + "with open(file1, 'w') as f:\n", + " f.write(\"Apple Inc. is a technology company. Tim Cook is the CEO.\")\n", + "\n", + "file_objects = file_ingestor.ingest_file(file1, read_content=True)\n", + "\n", + "print(f\"Ingested {len([file_objects]) if file_objects else 0} files\")\n", + "print(f\"Multi-source ingestion initialized\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Entity Resolution\n", + "\n", + "Resolve entities across multiple sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "entity_resolver = EntityResolver()\n", + "\n", + "entities_from_source1 = [\n", + " {\"id\": \"e1\", \"name\": \"Apple Inc.\", \"type\": \"Organization\", \"source\": \"file1\"},\n", + " {\"id\": \"e2\", \"name\": \"Tim Cook\", \"type\": \"Person\", \"source\": \"file1\"}\n", + "]\n", + "\n", + "entities_from_source2 = [\n", + " {\"id\": \"e3\", \"name\": \"Apple Incorporated\", \"type\": \"Organization\", \"source\": \"web\"},\n", + " {\"id\": \"e4\", \"name\": \"Timothy Cook\", \"type\": \"Person\", \"source\": \"web\"}\n", + "]\n", + "\n", + "all_entities = entities_from_source1 + entities_from_source2\n", + "\n", + "resolved_entities = entity_resolver.resolve(all_entities)\n", + "\n", + "print(f\"Original entities: {len(all_entities)}\")\n", + "print(f\"Resolved entities: {len(resolved_entities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Conflict Detection\n", + "\n", + "Detect conflicts between sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "conflict_detector = ConflictDetector()\n", + "\n", + "conflicts = conflict_detector.detect_value_conflicts(all_entities, \"name\")\n", + "\n", + "print(f\"Detected {len(conflicts)} conflicts\")\n", + "for conflict in conflicts[:3]:\n", + " print(f\" Conflict: {conflict.entity_id} - {conflict.conflict_type}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Provenance Tracking\n", + "\n", + "Track data provenance across sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "provenance_tracker = ProvenanceTracker()\n", + "\n", + "for entity in all_entities:\n", + " provenance_tracker.track_entity(entity.get(\"id\"), entity.get(\"source\"), entity)\n", + "\n", + "relationships = [\n", + " {\"source\": \"e2\", \"target\": \"e1\", \"type\": \"CEO_of\", \"source\": \"file1\"}\n", + "]\n", + "\n", + "for rel in relationships:\n", + " provenance_tracker.track_relationship(rel.get(\"source\"), rel.get(\"target\"), rel.get(\"source\"), rel)\n", + "\n", + "print(f\"Tracked provenance for {len(all_entities)} entities and {len(relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Unified Knowledge Graph\n", + "\n", + "Build a unified knowledge graph from integrated sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "unified_kg = builder.build(resolved_entities, relationships)\n", + "\n", + "print(f\"Built unified knowledge graph\")\n", + "print(f\" Entities: {len(unified_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(unified_kg.get('relationships', []))}\")\n", + "print(f\" Sources integrated: {len(set(e.get('source', '') for e in resolved_entities))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned advanced multi-source data integration:\n", + "\n", + "- **Multiple Ingestion Types**: FileIngestor, WebIngestor, DBIngestor, StreamIngestor, FeedIngestor\n", + "- **EntityResolver**: Resolve entities across sources\n", + "- **ConflictDetector**: Detect conflicts between sources\n", + "- **ProvenanceTracker**: Track data provenance\n", + "- **Unified Knowledge Graph**: Build integrated graph from multiple sources\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Pipeline_Orchestration.ipynb b/docs/cookbook/advanced/Pipeline_Orchestration.ipynb new file mode 100644 index 00000000..291da668 --- /dev/null +++ b/docs/cookbook/advanced/Pipeline_Orchestration.ipynb @@ -0,0 +1,195 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Pipeline Orchestration\n", + "\n", + "## Overview\n", + "\n", + "Build complex pipelines, execute them, handle failures, enable parallel processing, and monitor execution.\n", + "\n", + "## Workflow: Build Pipelines → Execute → Handle Failures → Parallel Processing → Monitor\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.pipeline import (\n", + " PipelineBuilder,\n", + " ExecutionEngine,\n", + " FailureHandler,\n", + " ParallelismManager\n", + ")\n", + "from semantica.ingest import FileIngestor\n", + "from semantica.parse import DocumentParser\n", + "from semantica.semantic_extract import NERExtractor\n", + "from semantica.kg import GraphBuilder\n", + "import time\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Build Complex Pipelines\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = PipelineBuilder()\n", + "\n", + "file_ingestor = FileIngestor()\n", + "document_parser = DocumentParser()\n", + "ner_extractor = NERExtractor()\n", + "graph_builder = GraphBuilder()\n", + "\n", + "pipeline = builder.add_step(\"ingest\", file_ingestor) \\\n", + " .add_step(\"parse\", document_parser) \\\n", + " .add_step(\"extract\", ner_extractor) \\\n", + " .add_step(\"build_graph\", graph_builder) \\\n", + " .build()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Execute Pipeline\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "engine = ExecutionEngine()\n", + "\n", + "input_data = {\n", + " \"text\": \"Alice works at Tech Corp. Bob is a friend of Alice.\",\n", + " \"files\": []\n", + "}\n", + "\n", + "start_time = time.time()\n", + "results = engine.execute(pipeline, input_data)\n", + "execution_time = time.time() - start_time\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Handle Failures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "failure_handler = FailureHandler()\n", + "\n", + "pipeline_with_retry = failure_handler.configure_retry(pipeline, max_retries=3)\n", + "\n", + "pipeline_with_error_handling = failure_handler.configure_error_handling(\n", + " pipeline_with_retry, \n", + " on_error=\"skip\"\n", + ")\n", + "\n", + "try:\n", + " results = engine.execute(pipeline_with_error_handling, input_data)\n", + "except Exception as e:\n", + " print(f\"Error handled gracefully: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Parallel Processing\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parallelism = ParallelismManager()\n", + "\n", + "parallel_pipeline = parallelism.enable_parallel(pipeline, max_workers=4)\n", + "\n", + "start_time = time.time()\n", + "results_parallel = engine.execute(parallel_pipeline, input_data)\n", + "parallel_time = time.time() - start_time\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Monitor Pipeline Execution\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = engine.get_metrics() if hasattr(engine, 'get_metrics') else {\n", + " 'duration': execution_time,\n", + " 'items_processed': 1,\n", + " 'steps_completed': 4,\n", + " 'errors': 0\n", + "}\n", + "\n", + "print(f\"Duration: {metrics.get('duration', 0):.2f} seconds\")\n", + "print(f\"Items Processed: {metrics.get('items_processed', 0)}\")\n", + "print(f\"Steps Completed: {metrics.get('steps_completed', 0)}\")\n", + "print(f\"Errors: {metrics.get('errors', 0)}\")\n", + "print(f\"Success Rate: {(1 - metrics.get('errors', 0) / max(metrics.get('items_processed', 1), 1)) * 100:.1f}%\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Pipeline orchestration workflow:\n", + "- Complex Pipeline Built\n", + "- Pipeline Executed\n", + "- Failure Handling Configured\n", + "- Parallel Processing Enabled\n", + "- Full Monitoring and Observability\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Pipeline Orchestration Complete\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Reasoning_and_Inference.ipynb b/docs/cookbook/advanced/Reasoning_and_Inference.ipynb new file mode 100644 index 00000000..964df91e --- /dev/null +++ b/docs/cookbook/advanced/Reasoning_and_Inference.ipynb @@ -0,0 +1,272 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Reasoning and Inference\n", + "\n", + "## Overview\n", + "\n", + "Build knowledge graphs, define rules, perform forward/backward chaining, and generate explanations for AI reasoning.\n", + "\n", + "## Workflow: Build KG → Define Rules → Forward/Backward Chaining → Generate Explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Build Knowledge Graph\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n", + " {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n", + " {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n", + " {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n", + " {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"},\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n", + " {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n", + " {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n", + " {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"},\n", + "]\n", + "\n", + "knowledge_graph = builder.build(entities, relationships)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Define Rules\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class RuleManager:\n", + " def __init__(self):\n", + " self.rules = []\n", + " \n", + " def add_rules(self, rules):\n", + " self.rules.extend(rules)\n", + "\n", + "rule_manager = RuleManager()\n", + "\n", + "rules = [\n", + " \"IF A is parent_of B AND B is parent_of C THEN A is grandparent_of C\",\n", + " \"IF X is located_in Y AND Y is part_of Z THEN X is located_in Z\",\n", + " \"IF X lives_in Y AND Y is located_in Z THEN X lives_in Z\"\n", + "]\n", + "\n", + "rule_manager.add_rules(rules)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Forward Chaining\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class InferenceEngine:\n", + " def forward_chain(self, kg, rule_manager):\n", + " new_facts = []\n", + " \n", + " for rule in rule_manager.rules:\n", + " if \"grandparent_of\" in rule:\n", + " parents = [r for r in relationships if r[\"type\"] == \"parent_of\"]\n", + " for p1 in parents:\n", + " for p2 in parents:\n", + " if p1[\"target\"] == p2[\"source\"]:\n", + " new_fact = {\n", + " \"source\": p1[\"source\"],\n", + " \"target\": p2[\"target\"],\n", + " \"type\": \"grandparent_of\",\n", + " \"inferred\": True\n", + " }\n", + " if new_fact not in new_facts:\n", + " new_facts.append(new_fact)\n", + " \n", + " elif \"lives_in\" in rule and \"located_in\" in rule:\n", + " lives_in = [r for r in relationships if r[\"type\"] == \"lives_in\"]\n", + " located_in = [r for r in relationships if r[\"type\"] == \"located_in\"]\n", + " \n", + " for live in lives_in:\n", + " for loc in located_in:\n", + " if live[\"target\"] == loc[\"source\"]:\n", + " new_fact = {\n", + " \"source\": live[\"source\"],\n", + " \"target\": loc[\"target\"],\n", + " \"type\": \"lives_in\",\n", + " \"inferred\": True\n", + " }\n", + " if new_fact not in new_facts:\n", + " new_facts.append(new_fact)\n", + " \n", + " return new_facts\n", + "\n", + "inference_engine = InferenceEngine()\n", + "new_facts = inference_engine.forward_chain(knowledge_graph, rule_manager)\n", + "\n", + "for fact in new_facts:\n", + " print(f\"{fact['source']} {fact['type']} {fact['target']} (inferred)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Backward Chaining\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def backward_chain(kg, rule_manager, goal):\n", + " proof_steps = []\n", + " \n", + " goal_source, goal_type, goal_target = goal\n", + " \n", + " for rel in relationships:\n", + " if rel[\"source\"] == goal_source and rel[\"type\"] == goal_type and rel[\"target\"] == goal_target:\n", + " proof_steps.append({\n", + " \"step\": \"Direct fact\",\n", + " \"fact\": f\"{goal_source} {goal_type} {goal_target}\",\n", + " \"source\": \"knowledge_graph\"\n", + " })\n", + " return proof_steps\n", + " \n", + " if goal_type == \"grandparent_of\":\n", + " for rel1 in relationships:\n", + " if rel1[\"source\"] == goal_source and rel1[\"type\"] == \"parent_of\":\n", + " intermediate = rel1[\"target\"]\n", + " for rel2 in relationships:\n", + " if rel2[\"source\"] == intermediate and rel2[\"type\"] == \"parent_of\" and rel2[\"target\"] == goal_target:\n", + " proof_steps.append({\n", + " \"step\": \"Rule application\",\n", + " \"fact\": f\"{goal_source} parent_of {intermediate}\",\n", + " \"source\": \"knowledge_graph\"\n", + " })\n", + " proof_steps.append({\n", + " \"step\": \"Rule application\",\n", + " \"fact\": f\"{intermediate} parent_of {goal_target}\",\n", + " \"source\": \"knowledge_graph\"\n", + " })\n", + " proof_steps.append({\n", + " \"step\": \"Inference\",\n", + " \"fact\": f\"{goal_source} grandparent_of {goal_target}\",\n", + " \"source\": \"inference_rule\"\n", + " })\n", + " return proof_steps\n", + " \n", + " return proof_steps\n", + "\n", + "goal = (\"alice\", \"grandparent_of\", \"charlie\")\n", + "proof = backward_chain(knowledge_graph, rule_manager, goal)\n", + "\n", + "for i, step in enumerate(proof, 1):\n", + " print(f\"Step {i}: {step['step']} - {step['fact']}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class ExplanationGenerator:\n", + " def generate(self, proof, kg):\n", + " if not proof:\n", + " return \"No proof found for the given goal.\"\n", + " \n", + " explanation_parts = []\n", + " explanation_parts.append(\"Explanation:\")\n", + " \n", + " for i, step in enumerate(proof, 1):\n", + " if step['step'] == 'Direct fact':\n", + " explanation_parts.append(f\"{i}. We know that {step['fact']} from the knowledge graph.\")\n", + " elif step['step'] == 'Rule application':\n", + " explanation_parts.append(f\"{i}. From the knowledge graph: {step['fact']}.\")\n", + " elif step['step'] == 'Inference':\n", + " explanation_parts.append(f\"{i}. Therefore, by applying the inference rule: {step['fact']}.\")\n", + " \n", + " return \"\\n\".join(explanation_parts)\n", + "\n", + "explanation_gen = ExplanationGenerator()\n", + "explanation = explanation_gen.generate(proof, knowledge_graph)\n", + "print(explanation)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Reasoning and inference workflow:\n", + "- Knowledge Graph Built\n", + "- Inference Rules Defined\n", + "- Forward Chaining Performed\n", + "- Backward Chaining Performed\n", + "- Explanations Generated\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Reasoning and Inference Complete\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Semantic_Layer_Construction.ipynb b/docs/cookbook/advanced/Semantic_Layer_Construction.ipynb new file mode 100644 index 00000000..82f0b34e --- /dev/null +++ b/docs/cookbook/advanced/Semantic_Layer_Construction.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Semantic Layer Construction\n", + "\n", + "## Overview\n", + "\n", + "Build an enterprise semantic layer: construct knowledge graph, generate ontology, create semantic layer, export RDF, and store in triple store.\n", + "\n", + "## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triple Store\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "from semantica.ontology import OntologyGenerator\n", + "from semantica.export import RDFExporter\n", + "from semantica.triple_store import TripleStore\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Build Knowledge Graph\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30, \"role\": \"Engineer\"}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35, \"role\": \"Manager\"}},\n", + " {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n", + " {\"id\": \"e4\", \"type\": \"Project\", \"name\": \"Project Alpha\", \"properties\": {\"status\": \"active\"}},\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"reports_to\"},\n", + " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\"},\n", + " {\"source\": \"e2\", \"target\": \"e3\", \"type\": \"works_for\"},\n", + " {\"source\": \"e1\", \"target\": \"e4\", \"type\": \"works_on\"},\n", + "]\n", + "\n", + "knowledge_graph = builder.build(entities, relationships)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Generate Ontology\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generator = OntologyGenerator()\n", + "ontology = generator.generate_from_graph(knowledge_graph)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Create Semantic Layer\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def create_mappings(kg, ontology):\n", + " mappings = {\n", + " \"entity_type_mappings\": {},\n", + " \"relationship_type_mappings\": {},\n", + " \"property_mappings\": {}\n", + " }\n", + " \n", + " entity_types = set(e.get(\"type\") for e in entities)\n", + " ontology_classes = ontology.get(\"classes\", [])\n", + " \n", + " for entity_type in entity_types:\n", + " matching_class = next((cls for cls in ontology_classes if cls.get(\"name\") == entity_type), None)\n", + " if matching_class:\n", + " mappings[\"entity_type_mappings\"][entity_type] = matching_class.get(\"uri\", entity_type)\n", + " \n", + " relationship_types = set(r.get(\"type\") for r in relationships)\n", + " ontology_properties = ontology.get(\"properties\", [])\n", + " \n", + " for rel_type in relationship_types:\n", + " matching_prop = next((prop for prop in ontology_properties if prop.get(\"name\") == rel_type), None)\n", + " if matching_prop:\n", + " mappings[\"relationship_type_mappings\"][rel_type] = matching_prop.get(\"uri\", rel_type)\n", + " \n", + " return mappings\n", + "\n", + "mappings = create_mappings(knowledge_graph, ontology)\n", + "\n", + "semantic_layer = {\n", + " \"graph\": knowledge_graph,\n", + " \"ontology\": ontology,\n", + " \"mappings\": mappings,\n", + " \"metadata\": {\n", + " \"version\": \"1.0\",\n", + " \"created_at\": \"2024-01-01\",\n", + " \"description\": \"Enterprise semantic layer\"\n", + " }\n", + "}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Export RDF\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "exporter = RDFExporter()\n", + "exporter.export(knowledge_graph, ontology, \"semantic_layer.rdf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Store in Triple Store\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "triple_store = TripleStore()\n", + "triple_store.store(knowledge_graph, ontology)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Enterprise semantic layer construction:\n", + "- Knowledge Graph Built\n", + "- Ontology Generated\n", + "- Semantic Layer Created with Mappings\n", + "- RDF Export Completed\n", + "- Triple Store Storage Completed\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Semantic Layer Construction Complete\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Temporal_Knowledge_Graphs.ipynb b/docs/cookbook/advanced/Temporal_Knowledge_Graphs.ipynb new file mode 100644 index 00000000..d2590091 --- /dev/null +++ b/docs/cookbook/advanced/Temporal_Knowledge_Graphs.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Temporal Knowledge Graphs\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates advanced temporal knowledge graph capabilities using TemporalGraphQuery, TemporalPatternDetector, TemporalVersionManager, and TemporalVisualizer.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use TemporalGraphQuery for time-aware queries\n", + "- Use TemporalPatternDetector to detect temporal patterns\n", + "- Use TemporalVersionManager for temporal versioning and snapshots\n", + "- Use TemporalVisualizer to visualize temporal data\n", + "\n", + "---\n", + "\n", + "## Workflow: Build Temporal KG → Time-Aware Queries → Pattern Detection → Version Management → Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, TemporalVersionManager\n", + "from semantica.visualization import TemporalVisualizer\n", + "from datetime import datetime\n", + "\n", + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\", \"properties\": {\"founded\": \"1976\"}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Steve Jobs\", \"properties\": {\"born\": \"1955\"}}\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e2\", \"target\": \"e1\", \"type\": \"founded\", \"properties\": {\"timestamp\": \"1976-04-01\"}}\n", + "]\n", + "\n", + "temporal_kg = builder.build(entities, relationships)\n", + "\n", + "print(f\"Built temporal knowledge graph with {len(entities)} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Time-Aware Queries\n", + "\n", + "Query the graph at specific time points.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temporal_query = TemporalGraphQuery()\n", + "\n", + "query_result = temporal_query.query_time_range(\n", + " graph=temporal_kg,\n", + " query=\"Find entities founded in 1976\",\n", + " start_time=\"1976-01-01\",\n", + " end_time=\"1976-12-31\"\n", + ")\n", + "\n", + "print(f\"Time-aware query returned {len(query_result.get('entities', []))} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Temporal Pattern Detection\n", + "\n", + "Detect temporal patterns in the graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pattern_detector = TemporalPatternDetector()\n", + "\n", + "patterns = pattern_detector.detect_temporal_patterns(\n", + " temporal_kg,\n", + " pattern_type=\"sequence\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "print(f\"Detected {len(patterns)} temporal patterns\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Version Management\n", + "\n", + "Manage temporal versions and snapshots.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "version_manager = TemporalVersionManager()\n", + "\n", + "snapshot = version_manager.create_snapshot(temporal_kg, timestamp=datetime.now())\n", + "\n", + "print(f\"Created temporal snapshot at {snapshot.get('timestamp', 'N/A')}\")\n", + "print(f\"Snapshot contains {len(snapshot.get('entities', []))} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Temporal Visualization\n", + "\n", + "Visualize temporal data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "visualization = temporal_visualizer.visualize_timeline(temporal_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated temporal visualization\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned advanced temporal knowledge graph capabilities:\n", + "\n", + "- **TemporalGraphQuery**: Time-aware graph querying\n", + "- **TemporalPatternDetector**: Temporal pattern detection\n", + "- **TemporalVersionManager**: Temporal versioning and snapshots\n", + "- **TemporalVisualizer**: Temporal data visualization\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Text_Chunking_Strategies.ipynb b/docs/cookbook/advanced/Text_Chunking_Strategies.ipynb new file mode 100644 index 00000000..55b597e0 --- /dev/null +++ b/docs/cookbook/advanced/Text_Chunking_Strategies.ipynb @@ -0,0 +1,260 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Text Chunking Strategies\n", + "\n", + "## Overview\n", + "\n", + "Explore different text chunking strategies: semantic, structural, sliding window, and table chunking for optimal document processing.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import TextSplitter\n", + "import re\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Prepare Sample Document\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "document = \"\"\"\n", + "# Introduction to Knowledge Graphs\n", + "\n", + "Knowledge graphs are powerful data structures that represent information as entities and their relationships. \n", + "They enable semantic understanding and reasoning over complex data.\n", + "\n", + "## What are Knowledge Graphs?\n", + "\n", + "A knowledge graph is a graph-based data model used to represent knowledge. It consists of nodes (entities) \n", + "and edges (relationships) that connect these entities. Knowledge graphs are widely used in search engines, \n", + "recommendation systems, and AI applications.\n", + "\n", + "## Applications\n", + "\n", + "Knowledge graphs have numerous applications:\n", + "- Search engines use them to understand user queries\n", + "- Recommendation systems leverage them for personalized suggestions\n", + "- AI systems use them for reasoning and inference\n", + "\n", + "## Conclusion\n", + "\n", + "In summary, knowledge graphs provide a flexible and powerful way to represent and reason about complex information.\n", + "\"\"\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Semantic Chunking\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class SemanticChunker:\n", + " def chunk(self, document, chunk_size=500):\n", + " paragraphs = [p.strip() for p in document.split('\\n\\n') if p.strip()]\n", + " \n", + " chunks = []\n", + " current_chunk = \"\"\n", + " \n", + " for para in paragraphs:\n", + " if len(current_chunk) + len(para) <= chunk_size:\n", + " current_chunk += para + \"\\n\\n\"\n", + " else:\n", + " if current_chunk:\n", + " chunks.append(current_chunk.strip())\n", + " current_chunk = para + \"\\n\\n\"\n", + " \n", + " if current_chunk:\n", + " chunks.append(current_chunk.strip())\n", + " \n", + " return chunks\n", + "\n", + "semantic_chunker = SemanticChunker()\n", + "semantic_chunks = semantic_chunker.chunk(document, chunk_size=500)\n", + "\n", + "for i, chunk in enumerate(semantic_chunks, 1):\n", + " print(f\"Chunk {i}: {len(chunk)} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Structural Chunking\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class StructuralChunker:\n", + " def chunk(self, document):\n", + " chunks = []\n", + " current_section = \"\"\n", + " current_header = \"\"\n", + " \n", + " lines = document.split('\\n')\n", + " \n", + " for line in lines:\n", + " if line.startswith('#'):\n", + " if current_section:\n", + " chunks.append({\n", + " 'header': current_header,\n", + " 'content': current_section.strip()\n", + " })\n", + " current_header = line.strip()\n", + " current_section = \"\"\n", + " else:\n", + " current_section += line + \"\\n\"\n", + " \n", + " if current_section:\n", + " chunks.append({\n", + " 'header': current_header,\n", + " 'content': current_section.strip()\n", + " })\n", + " \n", + " return chunks\n", + "\n", + "structural_chunker = StructuralChunker()\n", + "structural_chunks = structural_chunker.chunk(document)\n", + "\n", + "for i, chunk in enumerate(structural_chunks, 1):\n", + " header = chunk['header'][:50] if chunk['header'] else \"No header\"\n", + " print(f\"Chunk {i}: {header}... ({len(chunk['content'])} chars)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Sliding Window Chunking\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class SlidingWindowChunker:\n", + " def chunk(self, document, window_size=200, overlap=50):\n", + " words = document.split()\n", + " chunks = []\n", + " \n", + " start = 0\n", + " while start < len(words):\n", + " end = min(start + window_size, len(words))\n", + " chunk_words = words[start:end]\n", + " chunks.append(' '.join(chunk_words))\n", + " \n", + " start += window_size - overlap\n", + " \n", + " return chunks\n", + "\n", + "sliding_chunker = SlidingWindowChunker()\n", + "sliding_chunks = sliding_chunker.chunk(document, window_size=200, overlap=50)\n", + "\n", + "for i, chunk in enumerate(sliding_chunks[:3], 1):\n", + " print(f\"Chunk {i}: {len(chunk)} characters\")\n", + "if len(sliding_chunks) > 3:\n", + " print(f\"... and {len(sliding_chunks) - 3} more chunks\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Table Chunking\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class TableChunker:\n", + " def chunk(self, table_data):\n", + " if isinstance(table_data, str):\n", + " rows = [row.strip() for row in table_data.split('\\n') if row.strip()]\n", + " chunks = []\n", + " for row in rows:\n", + " if '|' in row:\n", + " chunks.append(row)\n", + " return chunks\n", + " elif isinstance(table_data, list):\n", + " return [str(row) for row in table_data]\n", + " else:\n", + " return [str(table_data)]\n", + "\n", + "table_data = \"\"\"\n", + "| Name | Age | Role |\n", + "|------|-----|------|\n", + "| Alice | 30 | Engineer |\n", + "| Bob | 35 | Manager |\n", + "| Charlie | 28 | Developer |\n", + "\"\"\"\n", + "\n", + "table_chunker = TableChunker()\n", + "table_chunks = table_chunker.chunk(table_data)\n", + "\n", + "for i, chunk in enumerate(table_chunks, 1):\n", + " print(f\"Chunk {i}: {chunk[:50]}...\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Chunking strategies:\n", + "- Semantic Chunking (by meaning/paragraphs)\n", + "- Structural Chunking (by document structure)\n", + "- Sliding Window Chunking (with overlap)\n", + "- Table Chunking (for structured data)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Text Chunking Strategies Complete\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/advanced/Unstructured_to_Ontology.ipynb b/docs/cookbook/advanced/Unstructured_to_Ontology.ipynb new file mode 100644 index 00000000..16276561 --- /dev/null +++ b/docs/cookbook/advanced/Unstructured_to_Ontology.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Unstructured to Ontology\n", + "\n", + "## Overview\n", + "\n", + "Transform unstructured text into a formal ontology: extract concepts, generate ontology, validate, and export to OWL.\n", + "\n", + "## Workflow: Unstructured Text → Extract Concepts → Generate Ontology → Validate → Export OWL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", + "from semantica.ontology import OntologyGenerator, OntologyValidator\n", + "from semantica.export import OWLExporter\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Extract Concepts from Unstructured Text\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "unstructured_text = \"\"\"\n", + "Apple Inc. is a technology company founded by Steve Jobs in 1976. \n", + "The company is headquartered in Cupertino, California. \n", + "Tim Cook is the current CEO of Apple. \n", + "Apple develops products like iPhone, iPad, and MacBook.\n", + "The company has offices in multiple countries including the United States, China, and Japan.\n", + "\"\"\"\n", + "\n", + "extractor = NERExtractor()\n", + "entities = extractor.extract(unstructured_text)\n", + "\n", + "relation_extractor = RelationExtractor()\n", + "relationships = relation_extractor.extract(unstructured_text, entities)\n", + "\n", + "for entity in entities[:5]:\n", + " print(f\"{entity.get('text', entity)} ({entity.get('type', 'Unknown')})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Generate Ontology\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generator = OntologyGenerator()\n", + "ontology = generator.generate(entities, relationships)\n", + "\n", + "if ontology.get('classes'):\n", + " for cls in ontology.get('classes', [])[:5]:\n", + " print(f\"{cls.get('name', cls)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Validate Ontology\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "validator = OntologyValidator()\n", + "validation_result = validator.validate_ontology(ontology)\n", + "\n", + "print(f\"Valid: {validation_result.valid}\")\n", + "print(f\"Consistent: {validation_result.consistent}\")\n", + "print(f\"Errors: {len(validation_result.errors)}\")\n", + "print(f\"Warnings: {len(validation_result.warnings)}\")\n", + "\n", + "if validation_result.errors:\n", + " print(\"\\nErrors:\")\n", + " for error in validation_result.errors:\n", + " print(f\" - {error}\")\n", + "\n", + "if validation_result.warnings:\n", + " print(\"\\nWarnings:\")\n", + " for warning in validation_result.warnings:\n", + " print(f\" - {warning}\")\n", + "\n", + "if validation_result.metrics:\n", + " print(\"\\nMetrics:\")\n", + " print(f\" Classes: {validation_result.metrics.get('class_count', 0)}\")\n", + " print(f\" Properties: {validation_result.metrics.get('property_count', 0)}\")\n", + " print(f\" Object Properties: {validation_result.metrics.get('object_property_count', 0)}\")\n", + " print(f\" Data Properties: {validation_result.metrics.get('data_property_count', 0)}\")\n", + " print(f\" Hierarchy Depth: {validation_result.metrics.get('hierarchy_depth', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Export to OWL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "exporter = OWLExporter()\n", + "exporter.export(ontology, \"output.owl\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Unstructured to ontology transformation:\n", + "- Concepts Extracted from Text\n", + "- Ontology Generated\n", + "- Ontology Validated\n", + "- OWL Export Completed\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Unstructured to Ontology Complete\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Building_Knowledge_Graphs.ipynb b/docs/cookbook/introduction/Building_Knowledge_Graphs.ipynb new file mode 100644 index 00000000..c10662aa --- /dev/null +++ b/docs/cookbook/introduction/Building_Knowledge_Graphs.ipynb @@ -0,0 +1,168 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Building Knowledge Graphs\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to build knowledge graphs from entities and relationships using Semantica's graph building modules. You'll learn to use `GraphBuilder`, `EntityResolver`, `GraphValidator`, and `Deduplicator`.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `GraphBuilder` to construct knowledge graphs\n", + "- Use `EntityResolver` to resolve entity conflicts\n", + "- Use `GraphValidator` to validate graph structure\n", + "- Use `Deduplicator` to remove duplicate entities\n", + "\n", + "---\n", + "\n", + "## Step 1: Build Knowledge Graph\n", + "\n", + "Construct a knowledge graph from entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", + "\n", + "builder = GraphBuilder()\n", + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "\n", + "text = \"Apple Inc. is a technology company. Tim Cook is the CEO of Apple Inc. Apple Inc. is headquartered in Cupertino, California.\"\n", + "\n", + "entities_list = ner_extractor.extract(text)\n", + "relationships_list = relation_extractor.extract(text, entities_list)\n", + "\n", + "entities = []\n", + "for i, entity in enumerate(entities_list[:5], 1):\n", + " entities.append({\n", + " \"id\": f\"e{i}\",\n", + " \"type\": entity.get(\"type\", \"Entity\"),\n", + " \"name\": entity.get(\"text\", entity.get(\"entity\", \"\")),\n", + " \"properties\": {}\n", + " })\n", + "\n", + "relationships = []\n", + "for i, rel in enumerate(relationships_list[:3], 1):\n", + " relationships.append({\n", + " \"source\": f\"e{1}\",\n", + " \"target\": f\"e{i+1}\",\n", + " \"type\": rel.get(\"type\", \"related_to\"),\n", + " \"properties\": {}\n", + " })\n", + "\n", + "knowledge_graph = builder.build(entities, relationships)\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.get('entities', []))} entities\")\n", + "print(f\"Relationships: {len(knowledge_graph.get('relationships', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Entity Resolution\n", + "\n", + "Resolve entity conflicts and duplicates.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import EntityResolver\n", + "\n", + "entity_resolver = EntityResolver()\n", + "\n", + "resolved_entities = entity_resolver.resolve(entities)\n", + "\n", + "print(f\"Original entities: {len(entities)}\")\n", + "print(f\"Resolved entities: {len(resolved_entities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Graph Validation\n", + "\n", + "Validate the knowledge graph structure.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphValidator\n", + "\n", + "graph_validator = GraphValidator()\n", + "\n", + "validation_result = graph_validator.validate(knowledge_graph)\n", + "\n", + "print(f\"Graph validation: {validation_result.get('valid', False)}\")\n", + "print(f\"Issues: {len(validation_result.get('issues', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Deduplication\n", + "\n", + "Remove duplicate entities from the graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import Deduplicator\n", + "\n", + "deduplicator = Deduplicator()\n", + "\n", + "deduplicated_graph = deduplicator.deduplicate(knowledge_graph)\n", + "\n", + "print(f\"Original entities: {len(knowledge_graph.get('entities', []))}\")\n", + "print(f\"Deduplicated entities: {len(deduplicated_graph.get('entities', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to build knowledge graphs:\n", + "\n", + "- **GraphBuilder**: Construct knowledge graphs from entities and relationships\n", + "- **EntityResolver**: Resolve entity conflicts and duplicates\n", + "- **GraphValidator**: Validate graph structure and quality\n", + "- **Deduplicator**: Remove duplicate entities\n", + "\n", + "Next: Learn how to analyze graphs in the Graph_Analytics notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Configuration_Basics.ipynb b/docs/cookbook/introduction/Configuration_Basics.ipynb new file mode 100644 index 00000000..ef91efde --- /dev/null +++ b/docs/cookbook/introduction/Configuration_Basics.ipynb @@ -0,0 +1,352 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Configuration Basics\n", + "\n", + "## Overview\n", + "\n", + "This notebook teaches you how to configure Semantica using `ConfigManager`, environment variables, and configuration files. Proper configuration is essential for using Semantica effectively.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Understand how to use `ConfigManager` for configuration management\n", + "- Learn to set and use environment variables\n", + "- Create and load configuration files (YAML/JSON)\n", + "- Configure common settings for API keys, models, and processing\n", + "- Follow best practices for configuration management\n", + "\n", + "---\n", + "\n", + "## Configuration Methods\n", + "\n", + "Semantica supports three main configuration methods:\n", + "\n", + "1. **ConfigManager** - Programmatic configuration management\n", + "2. **Environment Variables** - For sensitive data like API keys\n", + "3. **Config Files** - YAML or JSON files for structured configuration\n", + "\n", + "Each method is demonstrated in the code cells below.\n", + "\n", + "---\n", + "\n", + "## Step 1: ConfigManager Basics\n", + "\n", + "`ConfigManager` is the primary way to manage configuration in Semantica. It provides a unified interface for loading and accessing configuration values.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.core import ConfigManager\n", + "\n", + "config_manager = ConfigManager()\n", + "\n", + "print(\"ConfigManager initialized successfully!\")\n", + "print(f\"ConfigManager instance: {config_manager}\")\n", + "\n", + "try:\n", + " print(\"\\nAccessing configuration values:\")\n", + " print(\" Use config_manager.get('path.to.config', default='default_value')\")\n", + " print(\" Example: config_manager.get('llm_provider.provider', default='openai')\")\n", + "except Exception as e:\n", + " print(f\"Error accessing config: {e}\")\n", + "\n", + "try:\n", + " config = config_manager.config\n", + " print(f\"\\n✓ Config object created: {config is not None}\")\n", + "except Exception as e:\n", + " print(f\"Error creating config object: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Environment Variables\n", + "\n", + "Environment variables are the recommended way to store sensitive information like API keys. They're secure and don't get committed to version control.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "print(\"Environment Variables (SEMANTICA_*):\")\n", + "semantica_env_vars = {k: v for k, v in os.environ.items() if k.startswith('SEMANTICA_')}\n", + "if semantica_env_vars:\n", + " for key, value in semantica_env_vars.items():\n", + " masked_value = value[:4] + \"...\" if len(value) > 4 else \"***\"\n", + " print(f\" {key} = {masked_value}\")\n", + "else:\n", + " print(\" No SEMANTICA_* environment variables found\")\n", + " print(\" To set: os.environ['SEMANTICA_API_KEY'] = 'your_key'\")\n", + "\n", + "api_key = os.getenv(\"SEMANTICA_API_KEY\")\n", + "model_name = os.getenv(\"SEMANTICA_MODEL_NAME\", \"default-model\")\n", + "\n", + "print(f\"\\nRetrieved values:\")\n", + "print(f\" API Key set: {api_key is not None}\")\n", + "print(f\" Model name: {model_name}\")\n", + "\n", + "print(\"\\nNote: Environment variables with SEMANTICA_ prefix\")\n", + "print(\" are automatically loaded by ConfigManager\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Configuration Files\n", + "\n", + "Configuration files (YAML or JSON) are great for storing non-sensitive settings like model names, batch sizes, and processing parameters. They provide a structured way to manage configuration.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import yaml\n", + "import json\n", + "from pathlib import Path\n", + "\n", + "sample_config_yaml = \"\"\"\n", + "# Semantica Configuration File\n", + "api_keys:\n", + " openai: your_openai_key_here\n", + " anthropic: your_anthropic_key_here\n", + "\n", + "llm_provider:\n", + " provider: openai\n", + " model: gpt-4\n", + " temperature: 0.7\n", + "\n", + "embedding:\n", + " provider: openai\n", + " model: text-embedding-3-large\n", + " dimensions: 3072\n", + "\n", + "knowledge_graph:\n", + " backend: networkx\n", + " temporal: true\n", + "\n", + "processing:\n", + " batch_size: 32\n", + " max_workers: 4\n", + "\n", + "logging:\n", + " level: INFO\n", + " file: semantica.log\n", + "\"\"\"\n", + "\n", + "config_yaml_path = Path(\"sample_config.yaml\")\n", + "config_yaml_path.write_text(sample_config_yaml)\n", + "\n", + "print(\"Sample config.yaml created:\")\n", + "print(f\" Path: {config_yaml_path}\")\n", + "print(\"\\nConfig file contents:\")\n", + "print(sample_config_yaml)\n", + "\n", + "try:\n", + " config_from_file = config_manager.load_from_file(str(config_yaml_path))\n", + " print(\"\\n✓ Configuration loaded from YAML file!\")\n", + " print(f\" Config object: {config_from_file is not None}\")\n", + "except Exception as e:\n", + " print(f\"\\n✗ Error loading config file: {e}\")\n", + "\n", + "sample_config_json = {\n", + " \"api_keys\": {\n", + " \"openai\": \"your_openai_key_here\",\n", + " \"anthropic\": \"your_anthropic_key_here\"\n", + " },\n", + " \"llm_provider\": {\n", + " \"provider\": \"openai\",\n", + " \"model\": \"gpt-4\",\n", + " \"temperature\": 0.7\n", + " },\n", + " \"embedding\": {\n", + " \"provider\": \"openai\",\n", + " \"model\": \"text-embedding-3-large\",\n", + " \"dimensions\": 3072\n", + " }\n", + "}\n", + "\n", + "config_json_path = Path(\"sample_config.json\")\n", + "with open(config_json_path, 'w') as f:\n", + " json.dump(sample_config_json, f, indent=2)\n", + "\n", + "print(f\"\\n✓ Sample config.json created: {config_json_path}\")\n", + "print(\"\\nNote: ConfigManager can load from both YAML and JSON files\")\n", + "print(\" config_manager.load_from_file('config.yaml')\")\n", + "print(\" config_manager.load_from_file('config.json')\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Common Settings\n", + "\n", + "This section covers the most commonly used configuration settings, including API keys, model parameters, embedding settings, and processing options.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.core import Config\n", + "\n", + "print(\"Common Configuration Settings:\")\n", + "print(\"\\n1. API Keys:\")\n", + "print(\" - OpenAI API key\")\n", + "print(\" - Anthropic API key\")\n", + "print(\" - Cohere API key\")\n", + "print(\" - Other provider keys\")\n", + "\n", + "print(\"\\n2. Model Names and Parameters:\")\n", + "print(\" - LLM provider (openai, anthropic, etc.)\")\n", + "print(\" - Model name (gpt-4, claude-3, etc.)\")\n", + "print(\" - Temperature, max_tokens, etc.\")\n", + "\n", + "print(\"\\n3. Embedding Settings:\")\n", + "print(\" - Embedding provider\")\n", + "print(\" - Embedding model\")\n", + "print(\" - Embedding dimensions\")\n", + "\n", + "print(\"\\n4. Graph Database Connections:\")\n", + "print(\" - Backend (networkx, neo4j, arangodb)\")\n", + "print(\" - Connection strings\")\n", + "print(\" - Temporal graph settings\")\n", + "\n", + "print(\"\\n5. Logging Levels:\")\n", + "print(\" - DEBUG, INFO, WARNING, ERROR\")\n", + "print(\" - Log file paths\")\n", + "\n", + "print(\"\\n6. Cache Settings:\")\n", + "print(\" - Enable/disable caching\")\n", + "print(\" - Cache directory\")\n", + "\n", + "try:\n", + " custom_config_dict = {\n", + " \"llm_provider\": {\n", + " \"provider\": \"openai\",\n", + " \"model\": \"gpt-4\",\n", + " \"temperature\": 0.7\n", + " },\n", + " \"embedding\": {\n", + " \"provider\": \"openai\",\n", + " \"model\": \"text-embedding-3-large\",\n", + " \"dimensions\": 3072\n", + " },\n", + " \"processing\": {\n", + " \"batch_size\": 32,\n", + " \"max_workers\": 4\n", + " }\n", + " }\n", + " \n", + " custom_config = Config(config_dict=custom_config_dict)\n", + " print(\"\\n✓ Custom Config object created with settings:\")\n", + " print(f\" LLM Provider: {custom_config.llm_provider.get('provider', 'N/A')}\")\n", + " print(f\" Embedding Provider: {custom_config.embedding_model.get('provider', 'N/A')}\")\n", + " print(f\" Batch Size: {custom_config.processing.get('batch_size', 'N/A')}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"\\n✗ Error creating custom config: {e}\")\n", + "\n", + "try:\n", + " if config_yaml_path.exists():\n", + " config_yaml_path.unlink()\n", + " if config_json_path.exists():\n", + " config_json_path.unlink()\n", + " print(\"\\n✓ Sample config files cleaned up\")\n", + "except:\n", + " pass\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Best Practices\n", + "\n", + "Follow these best practices to ensure secure, maintainable, and effective configuration management.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Configuration Best Practices:\")\n", + "print(\"\\n1. Use Environment Variables for Sensitive Data:\")\n", + "print(\" - Never commit API keys to version control\")\n", + "print(\" - Use environment variables or secret management\")\n", + "print(\" - Example: export SEMANTICA_API_KEY=your_key\")\n", + "\n", + "print(\"\\n2. Use Config Files for Non-Sensitive Settings:\")\n", + "print(\" - Store model names, batch sizes, etc. in config files\")\n", + "print(\" - Use YAML for readability or JSON for compatibility\")\n", + "print(\" - Keep config files in version control (without secrets)\")\n", + "\n", + "print(\"\\n3. Configuration Hierarchy:\")\n", + "print(\" - Environment variables override config file values\")\n", + "print(\" - Config file values override defaults\")\n", + "print(\" - Use defaults as fallback\")\n", + "\n", + "print(\"\\n4. Validate Configuration:\")\n", + "print(\" - Check required settings are present\")\n", + "print(\" - Validate API keys are set before use\")\n", + "print(\" - Use ConfigManager validation features\")\n", + "\n", + "print(\"\\n5. Separate Configurations by Environment:\")\n", + "print(\" - Development: dev_config.yaml\")\n", + "print(\" - Production: prod_config.yaml\")\n", + "print(\" - Testing: test_config.yaml\")\n", + "\n", + "print(\"\\n6. Document Configuration Options:\")\n", + "print(\" - Document all available settings\")\n", + "print(\" - Provide examples and defaults\")\n", + "print(\" - Explain the impact of each setting\")\n", + "\n", + "print(\"\\n\" + \"=\"*60)\n", + "print(\"Example: Checking if required configuration is set\")\n", + "print(\"=\"*60)\n", + "\n", + "required_settings = [\n", + " (\"API Key\", os.getenv(\"SEMANTICA_API_KEY\")),\n", + " (\"Model Name\", os.getenv(\"SEMANTICA_MODEL_NAME\", \"default\")),\n", + "]\n", + "\n", + "print(\"\\nRequired Settings Status:\")\n", + "for setting_name, value in required_settings:\n", + " status = \"✓ Set\" if value and value != \"default\" else \"✗ Not Set\"\n", + " print(f\" {setting_name}: {status}\")\n", + "\n", + "print(\"\\nRecommendation:\")\n", + "print(\" Set up your configuration before running Semantica workflows\")\n", + "print(\" Use ConfigManager to load and validate your settings\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Conflict_Detection.ipynb b/docs/cookbook/introduction/Conflict_Detection.ipynb new file mode 100644 index 00000000..2fb03673 --- /dev/null +++ b/docs/cookbook/introduction/Conflict_Detection.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conflict Detection\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to detect and resolve conflicts in knowledge graphs using Semantica's conflict modules. You'll learn to use `ConflictDetector`, `SourceTracker`, and `ConflictResolver`.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `ConflictDetector` to detect conflicts\n", + "- Use `SourceTracker` to track data sources\n", + "- Use `ConflictResolver` to resolve conflicts\n", + "\n", + "---\n", + "\n", + "## Step 1: Conflict Detection\n", + "\n", + "Detect conflicts in entities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.conflicts import ConflictDetector\n", + "\n", + "conflict_detector = ConflictDetector()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"name\": \"Apple Inc.\", \"source\": \"source1\"},\n", + " {\"id\": \"e1\", \"name\": \"Apple Incorporated\", \"source\": \"source2\"}\n", + "]\n", + "\n", + "conflicts = conflict_detector.detect_value_conflicts(entities, \"name\")\n", + "\n", + "print(f\"Detected {len(conflicts)} conflicts\")\n", + "for conflict in conflicts[:3]:\n", + " print(f\" Conflict: {conflict.entity_id} - {conflict.conflict_type}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Source Tracking\n", + "\n", + "Track data sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.conflicts import SourceTracker\n", + "\n", + "source_tracker = SourceTracker()\n", + "\n", + "source_tracker.track_source(\"e1\", \"source1\", {\"name\": \"Apple Inc.\"})\n", + "source_tracker.track_source(\"e1\", \"source2\", {\"name\": \"Apple Incorporated\"})\n", + "\n", + "sources = source_tracker.get_sources(\"e1\")\n", + "\n", + "print(f\"Tracked sources for e1: {len(sources)}\")\n", + "for source in sources:\n", + " print(f\" Source: {source.source_id}, Property: {source.property_name}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Conflict Resolution\n", + "\n", + "Resolve conflicts using ConflictResolver.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.conflicts import ConflictResolver\n", + "\n", + "conflict_resolver = ConflictResolver()\n", + "\n", + "if conflicts:\n", + " resolution = conflict_resolver.resolve_conflicts(conflicts, strategy=\"most_recent\")\n", + " print(f\"Resolved {len(resolution.resolved_conflicts)} conflicts\")\n", + " print(f\"Resolution strategy: {resolution.strategy}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to detect and resolve conflicts:\n", + "\n", + "- **ConflictDetector**: Detect conflicts in entities\n", + "- **SourceTracker**: Track data sources\n", + "- **ConflictResolver**: Resolve conflicts using various strategies\n", + "\n", + "Next: Learn about configuration in the Configuration notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Data_Ingestion.ipynb b/docs/cookbook/introduction/Data_Ingestion.ipynb new file mode 100644 index 00000000..47b22b03 --- /dev/null +++ b/docs/cookbook/introduction/Data_Ingestion.ipynb @@ -0,0 +1,208 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Ingestion\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to ingest data from various sources using Semantica's ingestion modules. You'll learn to ingest files, web content, databases, streams, and feeds.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `FileIngestor` to load files from local and cloud storage\n", + "- Use `WebIngestor` to scrape and crawl web content\n", + "- Use `DBIngestor` to extract data from databases\n", + "- Use `StreamIngestor` for real-time data streams\n", + "- Use `FeedIngestor` to process RSS/Atom feeds\n", + "\n", + "---\n", + "\n", + "## Step 1: File Ingestion\n", + "\n", + "Ingest files from local filesystem or cloud storage.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor\n", + "import tempfile\n", + "import os\n", + "\n", + "file_ingestor = FileIngestor()\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "sample_file = os.path.join(temp_dir, \"sample.txt\")\n", + "\n", + "with open(sample_file, 'w') as f:\n", + " f.write(\"Apple Inc. is a technology company. Tim Cook is the CEO.\")\n", + "\n", + "file_object = file_ingestor.ingest_file(sample_file, read_content=True)\n", + "\n", + "print(f\"Ingested file: {file_object.name}\")\n", + "print(f\"File type: {file_object.file_type}\")\n", + "print(f\"Size: {file_object.size} bytes\")\n", + "print(f\"Content preview: {file_object.content[:50]}...\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Directory Ingestion\n", + "\n", + "Ingest multiple files from a directory.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "file2 = os.path.join(temp_dir, \"doc2.txt\")\n", + "with open(file2, 'w') as f:\n", + " f.write(\"Microsoft Corporation is a technology company. Satya Nadella is the CEO.\")\n", + "\n", + "file_objects = file_ingestor.ingest_directory(temp_dir, recursive=False, read_content=True)\n", + "\n", + "print(f\"Ingested {len(file_objects)} files from directory\")\n", + "for file_obj in file_objects:\n", + " print(f\" - {file_obj.name} ({file_obj.file_type})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Web Ingestion\n", + "\n", + "Ingest content from web pages.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor\n", + "\n", + "web_ingestor = WebIngestor()\n", + "\n", + "try:\n", + " web_content = web_ingestor.ingest_url(\"https://example.com\")\n", + " print(f\"Ingested web page: {web_content.url}\")\n", + " print(f\"Title: {web_content.title}\")\n", + " print(f\"Content length: {len(web_content.text)} characters\")\n", + "except Exception as e:\n", + " print(f\"Web ingestion example (requires internet): {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Database Ingestion\n", + "\n", + "Ingest data from databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import DBIngestor\n", + "\n", + "db_ingestor = DBIngestor()\n", + "\n", + "print(\"DBIngestor initialized\")\n", + "print(\"To use: Configure database connection and call ingest_table() or ingest_query()\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Stream Ingestion\n", + "\n", + "Ingest data from real-time streams.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import StreamIngestor\n", + "\n", + "stream_ingestor = StreamIngestor()\n", + "\n", + "print(\"StreamIngestor initialized\")\n", + "print(\"To use: Configure stream source (Kafka, RabbitMQ, etc.) and start consuming\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Feed Ingestion\n", + "\n", + "Ingest RSS/Atom feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FeedIngestor\n", + "\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "try:\n", + " feed_data = feed_ingestor.ingest_feed(\"https://feeds.feedburner.com/oreilly/radar\")\n", + " print(f\"Ingested feed: {feed_data.title}\")\n", + " print(f\"Items: {len(feed_data.items)}\")\n", + " if feed_data.items:\n", + " print(f\"First item: {feed_data.items[0].title}\")\n", + "except Exception as e:\n", + " print(f\"Feed ingestion example (requires internet): {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to ingest data from multiple sources:\n", + "\n", + "- **FileIngestor**: Local files and directories\n", + "- **WebIngestor**: Web pages and URLs\n", + "- **DBIngestor**: Database tables and queries\n", + "- **StreamIngestor**: Real-time data streams\n", + "- **FeedIngestor**: RSS/Atom feeds\n", + "\n", + "Next: Learn how to parse the ingested data in the Document_Parsing notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Data_Normalization.ipynb b/docs/cookbook/introduction/Data_Normalization.ipynb new file mode 100644 index 00000000..0f87514e --- /dev/null +++ b/docs/cookbook/introduction/Data_Normalization.ipynb @@ -0,0 +1,228 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Normalization\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to normalize and clean data using Semantica's normalization modules. You'll learn to normalize text, entities, dates, numbers, and handle encoding issues.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `TextNormalizer` for text cleaning and normalization\n", + "- Use `EntityNormalizer` for entity name standardization\n", + "- Use `DateNormalizer` for date format normalization\n", + "- Use `NumberNormalizer` for number and quantity normalization\n", + "- Use `DataCleaner` for general data cleaning\n", + "- Use `LanguageDetector` and `EncodingHandler` for data quality\n", + "\n", + "---\n", + "\n", + "## Step 1: Text Normalization\n", + "\n", + "Normalize text content for consistency.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.normalize import TextNormalizer\n", + "\n", + "text_normalizer = TextNormalizer()\n", + "\n", + "sample_text = \"Hello World!!! This is a test.\"\n", + "\n", + "normalized = text_normalizer.normalize_text(sample_text, case=\"lower\")\n", + "cleaned = text_normalizer.clean_text(sample_text, remove_special_chars=False)\n", + "\n", + "print(f\"Original: {sample_text}\")\n", + "print(f\"Normalized: {normalized}\")\n", + "print(f\"Cleaned: {cleaned}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Entity Normalization\n", + "\n", + "Normalize entity names to canonical forms.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.normalize import EntityNormalizer\n", + "\n", + "entity_normalizer = EntityNormalizer()\n", + "\n", + "entity_variants = [\"Apple Inc.\", \"Apple Inc\", \"Apple\", \"Apple Incorporated\"]\n", + "\n", + "normalized_entities = []\n", + "for entity in entity_variants:\n", + " normalized = entity_normalizer.normalize_entity(entity, entity_type=\"Organization\")\n", + " normalized_entities.append(normalized)\n", + " print(f\"{entity} -> {normalized}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Date Normalization\n", + "\n", + "Normalize dates to standard formats.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.normalize import DateNormalizer\n", + "\n", + "date_normalizer = DateNormalizer()\n", + "\n", + "date_formats = [\"2023-12-25\", \"12/25/2023\", \"December 25, 2023\", \"25 Dec 2023\"]\n", + "\n", + "for date_str in date_formats:\n", + " try:\n", + " normalized = date_normalizer.normalize_date(date_str)\n", + " print(f\"{date_str} -> {normalized}\")\n", + " except Exception as e:\n", + " print(f\"{date_str} -> Error: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Number Normalization\n", + "\n", + "Normalize numbers and quantities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.normalize import NumberNormalizer\n", + "\n", + "number_normalizer = NumberNormalizer()\n", + "\n", + "numbers = [\"1,000\", \"1.5M\", \"$100\", \"50%\", \"3.14e2\"]\n", + "\n", + "for num_str in numbers:\n", + " try:\n", + " normalized = number_normalizer.normalize_number(num_str)\n", + " print(f\"{num_str} -> {normalized}\")\n", + " except Exception as e:\n", + " print(f\"{num_str} -> Error: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Data Cleaning\n", + "\n", + "Clean data using DataCleaner.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.normalize import DataCleaner\n", + "\n", + "data_cleaner = DataCleaner()\n", + "\n", + "data = [\n", + " {\"name\": \"Apple Inc.\", \"value\": 100},\n", + " {\"name\": \"Apple Inc\", \"value\": 100},\n", + " {\"name\": \"Microsoft\", \"value\": 200}\n", + "]\n", + "\n", + "cleaned_data = data_cleaner.clean_data(data, remove_duplicates=True)\n", + "\n", + "print(f\"Original records: {len(data)}\")\n", + "print(f\"Cleaned records: {len(cleaned_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Language Detection and Encoding\n", + "\n", + "Detect language and handle encoding.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.normalize import LanguageDetector, EncodingHandler\n", + "\n", + "language_detector = LanguageDetector()\n", + "encoding_handler = EncodingHandler()\n", + "\n", + "text_samples = [\n", + " \"Hello, this is English text.\",\n", + " \"Bonjour, ceci est du texte français.\",\n", + " \"Hola, este es texto en español.\"\n", + "]\n", + "\n", + "for text in text_samples:\n", + " detected_lang = language_detector.detect_language(text)\n", + " print(f\"Text: {text[:30]}... -> Language: {detected_lang}\")\n", + "\n", + "sample_bytes = \"Hello World\".encode('utf-8')\n", + "detected_encoding = encoding_handler.detect_encoding(sample_bytes)\n", + "print(f\"\\nDetected encoding: {detected_encoding}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to normalize and clean data:\n", + "\n", + "- **TextNormalizer**: Text cleaning and normalization\n", + "- **EntityNormalizer**: Entity name standardization\n", + "- **DateNormalizer**: Date format normalization\n", + "- **NumberNormalizer**: Number and quantity normalization\n", + "- **DataCleaner**: General data cleaning\n", + "- **LanguageDetector**: Language detection\n", + "- **EncodingHandler**: Encoding detection and conversion\n", + "\n", + "Next: Learn how to extract entities in the Entity_Extraction notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Deduplication.ipynb b/docs/cookbook/introduction/Deduplication.ipynb new file mode 100644 index 00000000..188a91d7 --- /dev/null +++ b/docs/cookbook/introduction/Deduplication.ipynb @@ -0,0 +1,123 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deduplication\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to detect and merge duplicate entities using Semantica's deduplication modules. You'll learn to use `DuplicateDetector`, `EntityMerger`, `SimilarityCalculator`, and `ClusterBuilder`.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `DuplicateDetector` to find duplicate entities\n", + "- Use `EntityMerger` to merge duplicates\n", + "- Use `SimilarityCalculator` to calculate similarity scores\n", + "- Use `ClusterBuilder` for batch deduplication\n", + "\n", + "---\n", + "\n", + "## Step 1: Duplicate Detection\n", + "\n", + "Detect duplicate entities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.deduplication import DuplicateDetector\n", + "\n", + "duplicate_detector = DuplicateDetector(similarity_threshold=0.8)\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"name\": \"Apple Inc.\", \"type\": \"Organization\"},\n", + " {\"id\": \"e2\", \"name\": \"Apple Inc\", \"type\": \"Organization\"},\n", + " {\"id\": \"e3\", \"name\": \"Microsoft\", \"type\": \"Organization\"}\n", + "]\n", + "\n", + "duplicates = duplicate_detector.detect_duplicates(entities)\n", + "\n", + "print(f\"Detected {len(duplicates)} duplicate groups\")\n", + "for group in duplicates[:3]:\n", + " print(f\" Group: {[e.get('id') for e in group.entities]}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Entity Merging\n", + "\n", + "Merge duplicate entities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.deduplication import EntityMerger\n", + "\n", + "entity_merger = EntityMerger()\n", + "\n", + "merged_entities = entity_merger.merge_duplicates(entities)\n", + "\n", + "print(f\"Original entities: {len(entities)}\")\n", + "print(f\"Merged entities: {len(merged_entities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Similarity Calculation\n", + "\n", + "Calculate similarity between entities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.deduplication import SimilarityCalculator\n", + "\n", + "similarity_calculator = SimilarityCalculator()\n", + "\n", + "similarity = similarity_calculator.calculate_similarity(entities[0], entities[1])\n", + "\n", + "print(f\"Similarity between '{entities[0]['name']}' and '{entities[1]['name']}': {similarity.score:.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to deduplicate entities:\n", + "\n", + "- **DuplicateDetector**: Detect duplicate entities\n", + "- **EntityMerger**: Merge duplicate entities\n", + "- **SimilarityCalculator**: Calculate similarity scores\n", + "- **ClusterBuilder**: Batch deduplication\n", + "\n", + "Next: Learn how to generate embeddings in the Embedding_Generation notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Document_Parsing.ipynb b/docs/cookbook/introduction/Document_Parsing.ipynb new file mode 100644 index 00000000..6cea05a9 --- /dev/null +++ b/docs/cookbook/introduction/Document_Parsing.ipynb @@ -0,0 +1,247 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Document Parsing\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to parse various document formats using Semantica's parsing modules. You'll learn to extract text, metadata, and structured data from PDFs, DOCX, CSV, JSON, XML, and HTML files.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `DocumentParser` for general document parsing\n", + "- Use format-specific parsers: `PDFParser`, `DOCXParser`, `CSVParser`, `JSONParser`, `XMLParser`, `HTMLParser`\n", + "- Extract text content and metadata from documents\n", + "- Parse structured data formats\n", + "\n", + "---\n", + "\n", + "## Step 1: Document Parser\n", + "\n", + "Parse various document formats using the general DocumentParser.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import DocumentParser\n", + "import tempfile\n", + "import os\n", + "\n", + "document_parser = DocumentParser()\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "sample_txt = os.path.join(temp_dir, \"sample.txt\")\n", + "\n", + "with open(sample_txt, 'w') as f:\n", + " f.write(\"Apple Inc. is a technology company. Tim Cook is the CEO.\")\n", + "\n", + "text = document_parser.extract_text(sample_txt)\n", + "metadata = document_parser.extract_metadata(sample_txt)\n", + "\n", + "print(f\"Extracted text: {text[:50]}...\")\n", + "print(f\"Metadata: {metadata}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: CSV Parser\n", + "\n", + "Parse CSV files to extract structured data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import CSVParser\n", + "\n", + "csv_parser = CSVParser()\n", + "csv_file = os.path.join(temp_dir, \"data.csv\")\n", + "\n", + "with open(csv_file, 'w') as f:\n", + " f.write(\"name,company,role\\n\")\n", + " f.write(\"Tim Cook,Apple Inc.,CEO\\n\")\n", + " f.write(\"Satya Nadella,Microsoft Corporation,CEO\\n\")\n", + "\n", + "csv_data = csv_parser.parse(csv_file)\n", + "\n", + "print(f\"Parsed CSV with {len(csv_data.rows)} rows\")\n", + "print(f\"Columns: {csv_data.headers}\")\n", + "for row in csv_data.rows[:2]:\n", + " print(f\" {row}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: JSON Parser\n", + "\n", + "Parse JSON files to extract structured data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import JSONParser\n", + "import json\n", + "\n", + "json_parser = JSONParser()\n", + "json_file = os.path.join(temp_dir, \"data.json\")\n", + "\n", + "data = {\n", + " \"companies\": [\n", + " {\"name\": \"Apple Inc.\", \"ceo\": \"Tim Cook\"},\n", + " {\"name\": \"Microsoft Corporation\", \"ceo\": \"Satya Nadella\"}\n", + " ]\n", + "}\n", + "\n", + "with open(json_file, 'w') as f:\n", + " json.dump(data, f)\n", + "\n", + "json_data = json_parser.parse(json_file)\n", + "\n", + "print(f\"Parsed JSON: {json_data.data}\")\n", + "print(f\"Companies: {len(json_data.data.get('companies', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: XML Parser\n", + "\n", + "Parse XML files to extract structured data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import XMLParser\n", + "\n", + "xml_parser = XMLParser()\n", + "xml_file = os.path.join(temp_dir, \"data.xml\")\n", + "\n", + "xml_content = \"\"\"\n", + "\n", + " \n", + " \n", + "\"\"\"\n", + "\n", + "with open(xml_file, 'w') as f:\n", + " f.write(xml_content)\n", + "\n", + "xml_data = xml_parser.parse(xml_file)\n", + "\n", + "print(f\"Parsed XML with {len(xml_data.elements)} elements\")\n", + "print(f\"Root element: {xml_data.root.tag if xml_data.root else 'None'}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: HTML Parser\n", + "\n", + "Parse HTML files to extract content and structure.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import HTMLParser\n", + "\n", + "html_parser = HTMLParser()\n", + "html_file = os.path.join(temp_dir, \"page.html\")\n", + "\n", + "html_content = \"\"\"\n", + "Sample Page\n", + "\n", + "

Technology Companies

\n", + "

Apple Inc. is a technology company.

\n", + "\n", + "\"\"\"\n", + "\n", + "with open(html_file, 'w') as f:\n", + " f.write(html_content)\n", + "\n", + "html_data = html_parser.parse(html_file)\n", + "\n", + "print(f\"Parsed HTML\")\n", + "print(f\"Title: {html_data.metadata.get('title', 'N/A')}\")\n", + "print(f\"Text content: {html_data.text[:50]}...\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Structured Data Parser\n", + "\n", + "Use StructuredDataParser for multiple formats.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import StructuredDataParser\n", + "\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "parsed_json = structured_parser.parse_json(json_file)\n", + "parsed_csv = structured_parser.parse_csv(csv_file)\n", + "\n", + "print(f\"Structured parser parsed JSON: {len(parsed_json.get('data', {}).get('companies', []))} companies\")\n", + "print(f\"Structured parser parsed CSV: {len(parsed_csv.get('rows', []))} rows\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to parse various document formats:\n", + "\n", + "- **DocumentParser**: General document parsing\n", + "- **CSVParser**: CSV file parsing\n", + "- **JSONParser**: JSON file parsing\n", + "- **XMLParser**: XML file parsing\n", + "- **HTMLParser**: HTML file parsing\n", + "- **StructuredDataParser**: Multi-format structured data parsing\n", + "\n", + "Next: Learn how to normalize and clean data in the Data_Normalization notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Embedding_Generation.ipynb b/docs/cookbook/introduction/Embedding_Generation.ipynb new file mode 100644 index 00000000..6b431f2f --- /dev/null +++ b/docs/cookbook/introduction/Embedding_Generation.ipynb @@ -0,0 +1,100 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Embedding Generation\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to generate embeddings from text using Semantica's embedding modules. You'll learn to use `EmbeddingGenerator` and `TextEmbedder` to create vector representations of text.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `EmbeddingGenerator` to generate embeddings\n", + "- Use `TextEmbedder` for text embedding generation\n", + "- Generate embeddings for multiple texts\n", + "- Understand embedding dimensions\n", + "\n", + "---\n", + "\n", + "## Step 1: Generate Embeddings\n", + "\n", + "Generate embeddings using EmbeddingGenerator.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.embeddings import EmbeddingGenerator\n", + "\n", + "generator = EmbeddingGenerator()\n", + "\n", + "texts = [\n", + " \"Apple Inc. is a technology company.\",\n", + " \"Microsoft Corporation develops software.\",\n", + " \"Amazon provides cloud services.\"\n", + "]\n", + "\n", + "embeddings = generator.generate(texts)\n", + "\n", + "print(f\"Generated embeddings for {len(texts)} texts\")\n", + "print(f\"Embedding dimension: {len(embeddings[0]) if embeddings else 0}\")\n", + "print(f\"First embedding shape: {len(embeddings[0]) if embeddings else 'N/A'}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Text Embedding\n", + "\n", + "Use TextEmbedder for text-specific embeddings.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.embeddings import TextEmbedder\n", + "\n", + "text_embedder = TextEmbedder()\n", + "\n", + "text = \"Semantic knowledge graphs enable intelligent data processing.\"\n", + "\n", + "embedding = text_embedder.embed_text(text)\n", + "\n", + "print(f\"Generated embedding for text\")\n", + "print(f\"Embedding dimension: {len(embedding)}\")\n", + "print(f\"First 5 values: {embedding[:5]}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to generate embeddings:\n", + "\n", + "- **EmbeddingGenerator**: Generate embeddings for multiple texts\n", + "- **TextEmbedder**: Generate text-specific embeddings\n", + "\n", + "Next: Learn how to store and search vectors in the Vector_Store notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Entity_Extraction.ipynb b/docs/cookbook/introduction/Entity_Extraction.ipynb new file mode 100644 index 00000000..5a7a3caa --- /dev/null +++ b/docs/cookbook/introduction/Entity_Extraction.ipynb @@ -0,0 +1,106 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Entity Extraction\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to extract named entities from text using Semantica's NER modules. You'll learn to use `NERExtractor` and `NamedEntityRecognizer` to identify entities in text.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `NERExtractor` to extract entities from text\n", + "- Use `NamedEntityRecognizer` for advanced entity recognition\n", + "- Understand entity types and confidence scores\n", + "- Extract entities from multiple documents\n", + "\n", + "---\n", + "\n", + "## Step 1: Basic Entity Extraction\n", + "\n", + "Extract entities using NERExtractor.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NERExtractor\n", + "\n", + "ner_extractor = NERExtractor()\n", + "\n", + "text = \"Apple Inc. is a technology company founded by Steve Jobs in Cupertino, California in 1976.\"\n", + "\n", + "entities = ner_extractor.extract(text)\n", + "\n", + "print(f\"Extracted {len(entities)} entities:\")\n", + "for entity in entities[:5]:\n", + " entity_text = entity.get('text', entity.get('entity', ''))\n", + " entity_type = entity.get('type', 'Unknown')\n", + " print(f\" - {entity_text} ({entity_type})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Advanced Entity Recognition\n", + "\n", + "Use NamedEntityRecognizer for more control.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NamedEntityRecognizer\n", + "\n", + "named_entity_recognizer = NamedEntityRecognizer()\n", + "\n", + "texts = [\n", + " \"Tim Cook is the CEO of Apple Inc.\",\n", + " \"Microsoft Corporation is headquartered in Redmond, Washington.\",\n", + " \"Amazon was founded by Jeff Bezos in 1994.\"\n", + "]\n", + "\n", + "all_entities = []\n", + "for text in texts:\n", + " entities = named_entity_recognizer.extract_entities(text)\n", + " all_entities.extend(entities)\n", + " print(f\"Text: {text[:40]}...\")\n", + " print(f\" Entities: {len(entities)}\")\n", + " for entity in entities[:3]:\n", + " print(f\" - {entity.get('text', entity.get('entity', ''))} ({entity.get('type', 'Unknown')})\")\n", + " print()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to extract entities from text:\n", + "\n", + "- **NERExtractor**: Basic entity extraction\n", + "- **NamedEntityRecognizer**: Advanced entity recognition with multiple models\n", + "\n", + "Next: Learn how to extract relationships in the Relation_Extraction notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Export.ipynb b/docs/cookbook/introduction/Export.ipynb new file mode 100644 index 00000000..8b21ef11 --- /dev/null +++ b/docs/cookbook/introduction/Export.ipynb @@ -0,0 +1,177 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Export\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to export knowledge graphs and data to various formats using Semantica's export modules. You'll learn to use `JSONExporter`, `CSVExporter`, `RDFExporter`, `GraphExporter`, `OWLExporter`, and `VectorExporter`.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `JSONExporter` to export to JSON\n", + "- Use `CSVExporter` to export to CSV\n", + "- Use `RDFExporter` to export to RDF\n", + "- Use `GraphExporter` to export graph formats\n", + "- Use `OWLExporter` to export ontologies\n", + "- Use `VectorExporter` to export vectors\n", + "\n", + "---\n", + "\n", + "## Step 1: JSON Export\n", + "\n", + "Export knowledge graph to JSON.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import JSONExporter\n", + "from semantica.kg import GraphBuilder\n", + "\n", + "json_exporter = JSONExporter()\n", + "builder = GraphBuilder()\n", + "\n", + "entities = [{\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\", \"properties\": {}}]\n", + "relationships = []\n", + "\n", + "kg = builder.build(entities, relationships)\n", + "\n", + "json_exporter.export_knowledge_graph(kg, \"output.json\")\n", + "\n", + "print(\"Exported knowledge graph to JSON\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: CSV Export\n", + "\n", + "Export entities to CSV.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import CSVExporter\n", + "\n", + "csv_exporter = CSVExporter()\n", + "\n", + "csv_exporter.export_entities(entities, \"entities.csv\")\n", + "\n", + "print(\"Exported entities to CSV\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: RDF Export\n", + "\n", + "Export knowledge graph to RDF.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import RDFExporter\n", + "\n", + "rdf_exporter = RDFExporter()\n", + "\n", + "rdf_exporter.export_knowledge_graph(kg, \"output.rdf\")\n", + "\n", + "print(\"Exported knowledge graph to RDF\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Graph Export\n", + "\n", + "Export to graph formats (GraphML, GEXF).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import GraphExporter\n", + "\n", + "graph_exporter = GraphExporter()\n", + "\n", + "graph_exporter.export_knowledge_graph(kg, \"output.graphml\", format=\"graphml\")\n", + "\n", + "print(\"Exported knowledge graph to GraphML\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: OWL Export\n", + "\n", + "Export ontology to OWL.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import OWLExporter\n", + "from semantica.ontology import OntologyGenerator\n", + "\n", + "owl_exporter = OWLExporter()\n", + "generator = OntologyGenerator()\n", + "\n", + "ontology = generator.generate(entities, relationships)\n", + "\n", + "owl_exporter.export(ontology, \"output.owl\")\n", + "\n", + "print(\"Exported ontology to OWL\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to export data:\n", + "\n", + "- **JSONExporter**: Export to JSON format\n", + "- **CSVExporter**: Export to CSV format\n", + "- **RDFExporter**: Export to RDF format\n", + "- **GraphExporter**: Export to graph formats (GraphML, GEXF)\n", + "- **OWLExporter**: Export ontologies to OWL\n", + "- **VectorExporter**: Export vectors\n", + "\n", + "Next: Learn how to visualize data in the Visualization notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Graph_Analytics.ipynb b/docs/cookbook/introduction/Graph_Analytics.ipynb new file mode 100644 index 00000000..55aca166 --- /dev/null +++ b/docs/cookbook/introduction/Graph_Analytics.ipynb @@ -0,0 +1,162 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Graph Analytics\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to analyze knowledge graphs using Semantica's analytics modules. You'll learn to use `GraphAnalyzer`, `CentralityCalculator`, `CommunityDetector`, and `ConnectivityAnalyzer` to understand graph structure and properties.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `GraphAnalyzer` for comprehensive graph analysis\n", + "- Use `CentralityCalculator` to compute centrality measures\n", + "- Use `CommunityDetector` to find communities in graphs\n", + "- Use `ConnectivityAnalyzer` to analyze graph connectivity\n", + "\n", + "---\n", + "\n", + "## Step 1: Graph Analysis\n", + "\n", + "Analyze graph structure and properties.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder, GraphAnalyzer\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", + "\n", + "builder = GraphBuilder()\n", + "analyzer = GraphAnalyzer()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\", \"properties\": {}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Tim Cook\", \"properties\": {}},\n", + " {\"id\": \"e3\", \"type\": \"Location\", \"name\": \"Cupertino\", \"properties\": {}}\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e2\", \"target\": \"e1\", \"type\": \"CEO_of\", \"properties\": {}},\n", + " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"located_in\", \"properties\": {}}\n", + "]\n", + "\n", + "kg = builder.build(entities, relationships)\n", + "\n", + "metrics = analyzer.compute_metrics(kg)\n", + "\n", + "print(f\"Graph metrics:\")\n", + "print(f\" Entities: {metrics.get('entity_count', 0)}\")\n", + "print(f\" Relationships: {metrics.get('relationship_count', 0)}\")\n", + "print(f\" Density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Centrality Measures\n", + "\n", + "Calculate centrality measures for entities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import CentralityCalculator\n", + "\n", + "centrality_calculator = CentralityCalculator()\n", + "\n", + "centrality_scores = centrality_calculator.calculate_centrality(kg, measure=\"degree\")\n", + "\n", + "print(f\"Centrality scores:\")\n", + "for entity_id, score in list(centrality_scores.items())[:5]:\n", + " print(f\" {entity_id}: {score:.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Community Detection\n", + "\n", + "Detect communities in the graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import CommunityDetector\n", + "\n", + "community_detector = CommunityDetector()\n", + "\n", + "communities = community_detector.detect_communities(kg)\n", + "\n", + "print(f\"Detected {len(communities)} communities\")\n", + "for i, community in enumerate(communities[:3], 1):\n", + " print(f\" Community {i}: {len(community)} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Connectivity Analysis\n", + "\n", + "Analyze graph connectivity.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import ConnectivityAnalyzer\n", + "\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "connectivity = connectivity_analyzer.analyze_connectivity(kg)\n", + "\n", + "print(f\"Connectivity analysis:\")\n", + "print(f\" Is connected: {connectivity.get('is_connected', False)}\")\n", + "print(f\" Components: {len(connectivity.get('components', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to analyze knowledge graphs:\n", + "\n", + "- **GraphAnalyzer**: Comprehensive graph analysis and metrics\n", + "- **CentralityCalculator**: Calculate centrality measures\n", + "- **CommunityDetector**: Detect communities in graphs\n", + "- **ConnectivityAnalyzer**: Analyze graph connectivity\n", + "\n", + "Next: Learn how to assess graph quality in the Graph_Quality notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Graph_Quality.ipynb b/docs/cookbook/introduction/Graph_Quality.ipynb new file mode 100644 index 00000000..aa203123 --- /dev/null +++ b/docs/cookbook/introduction/Graph_Quality.ipynb @@ -0,0 +1,156 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Graph Quality\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to assess and improve knowledge graph quality using Semantica's quality assurance modules. You'll learn to use `KGQualityAssessor`, `ConsistencyChecker`, `CompletenessValidator`, and `QualityMetrics`.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `KGQualityAssessor` for overall quality assessment\n", + "- Use `ConsistencyChecker` to validate consistency\n", + "- Use `CompletenessValidator` to check completeness\n", + "- Use `QualityMetrics` to calculate quality metrics\n", + "\n", + "---\n", + "\n", + "## Step 1: Quality Assessment\n", + "\n", + "Assess overall graph quality.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.kg import GraphBuilder\n", + "\n", + "builder = GraphBuilder()\n", + "assessor = KGQualityAssessor()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\", \"properties\": {}}\n", + "]\n", + "\n", + "relationships = []\n", + "\n", + "kg = builder.build(entities, relationships)\n", + "\n", + "quality_score = assessor.assess_overall_quality(kg)\n", + "\n", + "print(f\"Overall quality score: {quality_score.get('overall_score', 0):.3f}\")\n", + "print(f\"Completeness: {quality_score.get('completeness', 0):.3f}\")\n", + "print(f\"Consistency: {quality_score.get('consistency', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Consistency Checking\n", + "\n", + "Check graph consistency.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg_qa import ConsistencyChecker\n", + "\n", + "consistency_checker = ConsistencyChecker()\n", + "\n", + "consistency_result = consistency_checker.check_consistency(kg)\n", + "\n", + "print(f\"Consistency check:\")\n", + "print(f\" Is consistent: {consistency_result.get('is_consistent', False)}\")\n", + "print(f\" Issues: {len(consistency_result.get('issues', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Completeness Validation\n", + "\n", + "Validate graph completeness.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg_qa import CompletenessValidator\n", + "\n", + "completeness_validator = CompletenessValidator()\n", + "\n", + "completeness_result = completeness_validator.validate_completeness(kg)\n", + "\n", + "print(f\"Completeness validation:\")\n", + "print(f\" Is complete: {completeness_result.get('is_complete', False)}\")\n", + "print(f\" Missing properties: {len(completeness_result.get('missing_properties', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Quality Metrics\n", + "\n", + "Calculate detailed quality metrics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg_qa import QualityMetrics\n", + "\n", + "quality_metrics = QualityMetrics()\n", + "\n", + "metrics = quality_metrics.calculate_metrics(kg)\n", + "\n", + "print(f\"Quality metrics:\")\n", + "print(f\" Entity coverage: {metrics.get('entity_coverage', 0):.3f}\")\n", + "print(f\" Relationship coverage: {metrics.get('relationship_coverage', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to assess graph quality:\n", + "\n", + "- **KGQualityAssessor**: Overall quality assessment\n", + "- **ConsistencyChecker**: Consistency validation\n", + "- **CompletenessValidator**: Completeness validation\n", + "- **QualityMetrics**: Detailed quality metrics\n", + "\n", + "Next: Learn how to deduplicate entities in the Deduplication notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Ontology.ipynb b/docs/cookbook/introduction/Ontology.ipynb new file mode 100644 index 00000000..1e09cecc --- /dev/null +++ b/docs/cookbook/introduction/Ontology.ipynb @@ -0,0 +1,155 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Ontology\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to generate and validate ontologies using Semantica's ontology modules. You'll learn to use `OntologyGenerator`, `ClassInferrer`, `PropertyGenerator`, and `OntologyValidator`.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `OntologyGenerator` to generate ontologies\n", + "- Use `ClassInferrer` to infer classes\n", + "- Use `PropertyGenerator` to generate properties\n", + "- Use `OntologyValidator` to validate ontologies\n", + "\n", + "---\n", + "\n", + "## Step 1: Generate Ontology\n", + "\n", + "Generate ontology from entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import OntologyGenerator\n", + "\n", + "generator = OntologyGenerator()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\"},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Tim Cook\"}\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e2\", \"target\": \"e1\", \"type\": \"CEO_of\"}\n", + "]\n", + "\n", + "ontology = generator.generate(entities, relationships)\n", + "\n", + "print(f\"Generated ontology\")\n", + "print(f\"Classes: {len(ontology.get('classes', []))}\")\n", + "print(f\"Properties: {len(ontology.get('properties', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Class Inference\n", + "\n", + "Infer classes from entities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import ClassInferrer\n", + "\n", + "class_inferrer = ClassInferrer()\n", + "\n", + "classes = class_inferrer.infer_classes(entities)\n", + "\n", + "print(f\"Inferred {len(classes)} classes\")\n", + "for cls in classes[:3]:\n", + " print(f\" - {cls.get('name', cls)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Property Generation\n", + "\n", + "Generate properties from relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import PropertyGenerator\n", + "\n", + "property_generator = PropertyGenerator()\n", + "\n", + "properties = property_generator.infer_properties(entities, relationships, classes)\n", + "\n", + "print(f\"Generated {len(properties)} properties\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Ontology Validation\n", + "\n", + "Validate the generated ontology.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ontology import OntologyValidator\n", + "\n", + "validator = OntologyValidator()\n", + "\n", + "validation_result = validator.validate_ontology(ontology)\n", + "\n", + "print(f\"Ontology validation:\")\n", + "print(f\" Valid: {validation_result.valid}\")\n", + "print(f\" Consistent: {validation_result.consistent}\")\n", + "print(f\" Errors: {len(validation_result.errors)}\")\n", + "print(f\" Warnings: {len(validation_result.warnings)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to work with ontologies:\n", + "\n", + "- **OntologyGenerator**: Generate ontologies from entities and relationships\n", + "- **ClassInferrer**: Infer classes from entities\n", + "- **PropertyGenerator**: Generate properties from relationships\n", + "- **OntologyValidator**: Validate ontologies\n", + "\n", + "Next: Learn how to export data in the Export notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Relation_Extraction.ipynb b/docs/cookbook/introduction/Relation_Extraction.ipynb new file mode 100644 index 00000000..0588f62a --- /dev/null +++ b/docs/cookbook/introduction/Relation_Extraction.ipynb @@ -0,0 +1,105 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Relation Extraction\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to extract relationships between entities using Semantica's relation extraction modules. You'll learn to use `RelationExtractor` and `TripleExtractor` to identify relationships in text.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `RelationExtractor` to extract relationships between entities\n", + "- Use `TripleExtractor` to extract RDF triples\n", + "- Understand relationship types and confidence scores\n", + "- Extract relationships from text with entities\n", + "\n", + "---\n", + "\n", + "## Step 1: Relation Extraction\n", + "\n", + "Extract relationships using RelationExtractor.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import RelationExtractor, NERExtractor\n", + "\n", + "relation_extractor = RelationExtractor()\n", + "ner_extractor = NERExtractor()\n", + "\n", + "text = \"Tim Cook is the CEO of Apple Inc. Apple Inc. is headquartered in Cupertino, California.\"\n", + "\n", + "entities = ner_extractor.extract(text)\n", + "relationships = relation_extractor.extract(text, entities)\n", + "\n", + "print(f\"Extracted {len(entities)} entities and {len(relationships)} relationships\")\n", + "print(\"\\nRelationships:\")\n", + "for rel in relationships[:5]:\n", + " source = rel.get('source', '')\n", + " target = rel.get('target', '')\n", + " rel_type = rel.get('type', 'related_to')\n", + " print(f\" - {source} --[{rel_type}]--> {target}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Triple Extraction\n", + "\n", + "Extract RDF triples using TripleExtractor.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import TripleExtractor\n", + "\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "text = \"Apple Inc. was founded by Steve Jobs in 1976. The company is based in Cupertino.\"\n", + "\n", + "triples = triple_extractor.extract_triples(text)\n", + "\n", + "print(f\"Extracted {len(triples)} triples:\")\n", + "for triple in triples[:5]:\n", + " subject = triple.get('subject', '')\n", + " predicate = triple.get('predicate', '')\n", + " object_val = triple.get('object', '')\n", + " print(f\" - ({subject}, {predicate}, {object_val})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to extract relationships from text:\n", + "\n", + "- **RelationExtractor**: Extract relationships between entities\n", + "- **TripleExtractor**: Extract RDF triples\n", + "\n", + "Next: Learn how to build knowledge graphs in the Building_Knowledge_Graphs notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Vector_Store.ipynb b/docs/cookbook/introduction/Vector_Store.ipynb new file mode 100644 index 00000000..17a990c8 --- /dev/null +++ b/docs/cookbook/introduction/Vector_Store.ipynb @@ -0,0 +1,132 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vector Store\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to store and search vectors using Semantica's vector store modules. You'll learn to use `VectorStore` and `HybridSearch` for vector storage and retrieval.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `VectorStore` to store vectors\n", + "- Search vectors using similarity\n", + "- Use `HybridSearch` for hybrid search\n", + "- Manage vector metadata\n", + "\n", + "---\n", + "\n", + "## Step 1: Store Vectors\n", + "\n", + "Store vectors in the vector store.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import VectorStore\n", + "from semantica.embeddings import EmbeddingGenerator\n", + "import numpy as np\n", + "\n", + "vector_store = VectorStore()\n", + "generator = EmbeddingGenerator()\n", + "\n", + "texts = [\"Apple Inc.\", \"Microsoft Corporation\", \"Amazon Web Services\"]\n", + "embeddings = generator.generate(texts)\n", + "\n", + "metadata = [\n", + " {\"id\": \"1\", \"type\": \"company\"},\n", + " {\"id\": \"2\", \"type\": \"company\"},\n", + " {\"id\": \"3\", \"type\": \"service\"}\n", + "]\n", + "\n", + "vector_ids = vector_store.store_vectors(embeddings, metadata)\n", + "\n", + "print(f\"Stored {len(vector_ids)} vectors\")\n", + "print(f\"Vector IDs: {vector_ids[:3]}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Search Vectors\n", + "\n", + "Search for similar vectors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query_text = \"technology company\"\n", + "query_embedding = generator.generate([query_text])[0]\n", + "\n", + "results = vector_store.search_vectors(query_embedding, k=3)\n", + "\n", + "print(f\"Found {len(results)} similar vectors\")\n", + "for result in results[:3]:\n", + " print(f\" ID: {result.get('id')}, Score: {result.get('score', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Hybrid Search\n", + "\n", + "Use HybridSearch for combined vector and metadata search.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import HybridSearch\n", + "\n", + "hybrid_search = HybridSearch()\n", + "\n", + "hybrid_results = hybrid_search.search(\n", + " query_vector=query_embedding,\n", + " vectors=embeddings,\n", + " metadata=metadata,\n", + " vector_ids=vector_ids,\n", + " k=3\n", + ")\n", + "\n", + "print(f\"Hybrid search found {len(hybrid_results)} results\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to use vector stores:\n", + "\n", + "- **VectorStore**: Store and search vectors\n", + "- **HybridSearch**: Hybrid vector and metadata search\n", + "\n", + "Next: Learn how to generate ontologies in the Ontology notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Visualization.ipynb b/docs/cookbook/introduction/Visualization.ipynb new file mode 100644 index 00000000..2005e6e2 --- /dev/null +++ b/docs/cookbook/introduction/Visualization.ipynb @@ -0,0 +1,136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualization\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to visualize knowledge graphs, ontologies, and embeddings using Semantica's visualization modules. You'll learn to use `KGVisualizer`, `OntologyVisualizer`, and `EmbeddingVisualizer`.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Use `KGVisualizer` to visualize knowledge graphs\n", + "- Use `OntologyVisualizer` to visualize ontologies\n", + "- Use `EmbeddingVisualizer` to visualize embeddings\n", + "\n", + "---\n", + "\n", + "## Step 1: Knowledge Graph Visualization\n", + "\n", + "Visualize knowledge graphs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import KGVisualizer\n", + "from semantica.kg import GraphBuilder\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "builder = GraphBuilder()\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\", \"properties\": {}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Tim Cook\", \"properties\": {}}\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"e2\", \"target\": \"e1\", \"type\": \"CEO_of\", \"properties\": {}}\n", + "]\n", + "\n", + "kg = builder.build(entities, relationships)\n", + "\n", + "visualization = kg_visualizer.visualize_network(kg, output=\"interactive\")\n", + "\n", + "print(\"Generated knowledge graph visualization\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ontology Visualization\n", + "\n", + "Visualize ontologies.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import OntologyVisualizer\n", + "from semantica.ontology import OntologyGenerator\n", + "\n", + "ontology_visualizer = OntologyVisualizer()\n", + "generator = OntologyGenerator()\n", + "\n", + "ontology = generator.generate(entities, relationships)\n", + "\n", + "visualization = ontology_visualizer.visualize_hierarchy(ontology, output=\"interactive\")\n", + "\n", + "print(\"Generated ontology visualization\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Embedding Visualization\n", + "\n", + "Visualize embeddings.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import EmbeddingVisualizer\n", + "from semantica.embeddings import EmbeddingGenerator\n", + "import numpy as np\n", + "\n", + "embedding_visualizer = EmbeddingVisualizer()\n", + "generator = EmbeddingGenerator()\n", + "\n", + "texts = [\"Apple Inc.\", \"Microsoft Corporation\", \"Amazon\"]\n", + "embeddings = generator.generate(texts)\n", + "labels = [\"Apple\", \"Microsoft\", \"Amazon\"]\n", + "\n", + "visualization = embedding_visualizer.visualize_2d_projection(embeddings, labels, method=\"umap\")\n", + "\n", + "print(\"Generated embedding visualization\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You've learned how to visualize data:\n", + "\n", + "- **KGVisualizer**: Visualize knowledge graphs\n", + "- **OntologyVisualizer**: Visualize ontologies\n", + "- **EmbeddingVisualizer**: Visualize embeddings\n", + "\n", + "Next: Learn how to detect conflicts in the Conflict_Detection notebook.\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Welcome_to_Semantica.ipynb b/docs/cookbook/introduction/Welcome_to_Semantica.ipynb new file mode 100644 index 00000000..a6a40031 --- /dev/null +++ b/docs/cookbook/introduction/Welcome_to_Semantica.ipynb @@ -0,0 +1,903 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Welcome to Semantica\n", + "\n", + "## Overview\n", + "\n", + "This notebook introduces you to the **Semantica framework** - a comprehensive knowledge graph and semantic processing framework designed for building production-ready semantic AI applications.\n", + "\n", + "### What You'll Learn\n", + "\n", + "- What Semantica is and why it's useful\n", + "- How to install and configure the framework\n", + "- Understanding the framework architecture\n", + "- Key concepts and terminology\n", + "- Next steps for getting started\n", + "\n", + "---\n", + "\n", + "## What is Semantica?\n", + "\n", + "**Semantica** is a powerful, production-ready framework for:\n", + "\n", + "- **Building Knowledge Graphs**: Transform unstructured data into structured knowledge graphs\n", + "- **Semantic Processing**: Extract entities, relationships, and meaning from text, images, and audio\n", + "- **GraphRAG**: Next-generation retrieval augmented generation using knowledge graphs\n", + "- **Temporal Analysis**: Time-aware knowledge graphs for tracking changes over time\n", + "- **Multi-Modal Processing**: Handle text, images, audio, and structured data\n", + "- **Enterprise Features**: Quality assurance, conflict resolution, ontology generation, and more\n", + "\n", + "### Use Cases\n", + "\n", + "- Threat intelligence and cybersecurity\n", + "- Healthcare and medical research\n", + "- Financial analysis and fraud detection\n", + "- Supply chain optimization\n", + "- Research and knowledge management\n", + "- Multi-agent AI systems\n", + "\n", + "---\n", + "\n", + "\n", + "## Installation & Setup\n", + "\n", + "### Prerequisites\n", + "\n", + "Before installing Semantica, ensure you have:\n", + "- Python 3.8 or higher\n", + "- pip package manager\n", + "- (Optional) Virtual environment for isolation\n", + "\n", + "### Installation Methods\n", + "\n", + "'''\n", + "# Method 1: Install from PyPI (when available)\n", + "# pip install semantica\n", + "\n", + "# Method 2: Install from source (development version)\n", + "# git clone https://github.com/your-org/semantica.git\n", + "# cd semantica\n", + "# pip install -e .\n", + "\n", + "# Method 3: Install with specific dependencies\n", + "# pip install semantica[all] # Install all optional dependencies\n", + "# pip install semantica[gpu] # Install GPU support\n", + "# pip install semantica[visualization] # Install visualization tools\n", + "\n", + "# Verify installation\n", + "# import semantica\n", + "# print(semantica.__version__)\n", + "'''\n", + "\n", + "### Configuration\n", + "\n", + "'''\n", + "# Set up environment variables for API keys and configuration\n", + "# export SEMANTICA_API_KEY=your_openai_key\n", + "# export SEMANTICA_EMBEDDING_PROVIDER=openai\n", + "# export SEMANTICA_MODEL_NAME=gpt-4\n", + "\n", + "# Or use a config file (config.yaml):\n", + "# api_keys:\n", + "# openai: your_key_here\n", + "# anthropic: your_key_here\n", + "# embedding:\n", + "# provider: openai\n", + "# model: text-embedding-3-large\n", + "# dimensions: 3072\n", + "# knowledge_graph:\n", + "# backend: networkx # or neo4j, arangodb\n", + "# temporal: true\n", + "'''\n", + "\n", + "---\n", + "\n", + "## Framework Architecture Overview\n", + "\n", + "Semantica is organized into modular components, each handling a specific aspect of semantic processing:\n", + "\n", + "'''\n", + "# ============================================================================\n", + "# CORE MODULES\n", + "# ============================================================================\n", + "\n", + "# 1. INGEST MODULE - Data Ingestion\n", + "# Purpose: Ingest data from various sources\n", + "# Components:\n", + "# - FileIngestor: Read files (PDF, DOCX, HTML, JSON, CSV, etc.)\n", + "# - WebIngestor: Scrape and ingest web pages\n", + "# - FeedIngestor: Process RSS/Atom feeds\n", + "# - StreamIngestor: Real-time data streaming\n", + "# - DBIngestor: Database queries and ingestion\n", + "# - EmailIngestor: Process email messages\n", + "# - RepoIngestor: Git repository analysis\n", + "#\n", + "# Example:\n", + "# from semantica.ingest import FileIngestor, WebIngestor\n", + "# file_ingestor = FileIngestor()\n", + "# web_ingestor = WebIngestor()\n", + "# documents = file_ingestor.ingest(\"data/\")\n", + "# web_docs = web_ingestor.ingest(\"https://example.com\")\n", + "\n", + "# 2. PARSE MODULE - Document Parsing\n", + "# Purpose: Parse and extract content from various formats\n", + "# Components:\n", + "# - DocumentParser: Main parser orchestrator\n", + "# - PDFParser: Extract text, tables, images from PDFs\n", + "# - DOCXParser: Parse Word documents\n", + "# - HTMLParser: Extract content from HTML\n", + "# - JSONParser: Parse structured JSON data\n", + "# - ExcelParser: Process spreadsheets\n", + "# - ImageParser: OCR and image analysis\n", + "# - CodeParser: Parse source code files\n", + "#\n", + "# Example:\n", + "# from semantica.parse import DocumentParser\n", + "# parser = DocumentParser()\n", + "# parsed_docs = parser.parse(documents)\n", + "\n", + "# 3. NORMALIZE MODULE - Text Normalization\n", + "# Purpose: Clean and normalize text for processing\n", + "# Components:\n", + "# - TextNormalizer: Main normalization orchestrator\n", + "# - TextCleaner: Remove noise, fix encoding\n", + "# - DataCleaner: Clean structured data\n", + "# - EntityNormalizer: Normalize entity names\n", + "# - DateNormalizer: Standardize date formats\n", + "# - NumberNormalizer: Normalize numeric values\n", + "# - LanguageDetector: Detect document language\n", + "# - EncodingHandler: Handle character encoding\n", + "#\n", + "# Example:\n", + "# from semantica.normalize import TextNormalizer\n", + "# normalizer = TextNormalizer()\n", + "# normalized = normalizer.normalize(parsed_docs)\n", + "\n", + "# 4. SEMANTIC_EXTRACT MODULE - Entity & Relationship Extraction\n", + "# Purpose: Extract entities, relationships, and semantic information\n", + "# Components:\n", + "# - NERExtractor: Named Entity Recognition\n", + "# - RelationExtractor: Extract relationships between entities\n", + "# - SemanticAnalyzer: Deep semantic analysis\n", + "# - SemanticNetworkExtractor: Extract semantic networks\n", + "#\n", + "# Example:\n", + "# from semantica.semantic_extract import NERExtractor, RelationExtractor\n", + "# extractor = NERExtractor()\n", + "# entities = extractor.extract(normalized_docs)\n", + "# relation_extractor = RelationExtractor()\n", + "# relationships = relation_extractor.extract(normalized_docs, entities)\n", + "\n", + "# 5. KG MODULE - Knowledge Graph Construction\n", + "# Purpose: Build and manage knowledge graphs\n", + "# Components:\n", + "# - GraphBuilder: Construct knowledge graphs from entities/relationships\n", + "# - GraphAnalyzer: Analyze graph structure and properties\n", + "# - GraphValidator: Validate graph quality and consistency\n", + "# - EntityResolver: Resolve entity conflicts and duplicates\n", + "# - ConflictDetector: Detect conflicting information\n", + "# - CentralityCalculator: Calculate node importance metrics\n", + "# - CommunityDetector: Detect communities in graphs\n", + "# - ConnectivityAnalyzer: Analyze graph connectivity\n", + "# - TemporalQuery: Query temporal knowledge graphs\n", + "# - Deduplicator: Remove duplicate entities/relationships\n", + "#\n", + "# Example:\n", + "# from semantica.kg import GraphBuilder, GraphAnalyzer\n", + "# builder = GraphBuilder()\n", + "# kg = builder.build(entities, relationships)\n", + "# analyzer = GraphAnalyzer()\n", + "# metrics = analyzer.analyze(kg)\n", + "\n", + "# 6. EMBEDDINGS MODULE - Embedding Generation\n", + "# Purpose: Generate vector embeddings for various data types\n", + "# Components:\n", + "# - EmbeddingGenerator: Main embedding orchestrator\n", + "# - TextEmbedder: Generate text embeddings\n", + "# - ImageEmbedder: Generate image embeddings\n", + "# - AudioEmbedder: Generate audio embeddings\n", + "# - MultimodalEmbedder: Combine multiple modalities\n", + "# - EmbeddingOptimizer: Optimize embedding quality\n", + "# - ProviderAdapters: Support for OpenAI, Cohere, etc.\n", + "#\n", + "# Example:\n", + "# from semantica.embeddings import EmbeddingGenerator\n", + "# generator = EmbeddingGenerator()\n", + "# embeddings = generator.generate(documents)\n", + "\n", + "# 7. VECTOR_STORE MODULE - Vector Database Operations\n", + "# Purpose: Store and search vector embeddings\n", + "# Components:\n", + "# - VectorStore: Main vector store interface\n", + "# - FAISSAdapter: FAISS integration\n", + "# - HybridSearch: Combine vector and keyword search\n", + "# - VectorRetriever: Retrieve relevant vectors\n", + "#\n", + "# Example:\n", + "# from semantica.vector_store import VectorStore, HybridSearch\n", + "# vector_store = VectorStore()\n", + "# vector_store.store(embeddings, documents, metadata)\n", + "# hybrid_search = HybridSearch(vector_store)\n", + "# results = hybrid_search.search(query, top_k=10)\n", + "\n", + "# 8. REASONING MODULE - Inference and Reasoning\n", + "# Purpose: Perform logical inference and reasoning\n", + "# Components:\n", + "# - InferenceEngine: Main inference orchestrator\n", + "# - RuleManager: Manage inference rules\n", + "# - DeductiveReasoner: Deductive reasoning\n", + "# - AbductiveReasoner: Abductive reasoning\n", + "# - ExplanationGenerator: Generate explanations for inferences\n", + "# - RETEEngine: RETE algorithm for rule matching\n", + "#\n", + "# Example:\n", + "# from semantica.reasoning import InferenceEngine, RuleManager\n", + "# inference_engine = InferenceEngine()\n", + "# rule_manager = RuleManager()\n", + "# new_facts = inference_engine.forward_chain(kg, rule_manager)\n", + "\n", + "# 9. ONTOLOGY MODULE - Ontology Generation\n", + "# Purpose: Generate and manage ontologies\n", + "# Components:\n", + "# - OntologyGenerator: Generate ontologies from knowledge graphs\n", + "# - OntologyValidator: Validate ontology structure\n", + "# - OWLGenerator: Generate OWL format ontologies\n", + "# - PropertyGenerator: Generate ontology properties\n", + "# - ClassInferrer: Infer ontology classes\n", + "#\n", + "# Example:\n", + "# from semantica.ontology import OntologyGenerator\n", + "# generator = OntologyGenerator()\n", + "# ontology = generator.generate_from_graph(kg)\n", + "\n", + "# 10. EXPORT MODULE - Data Export\n", + "# Purpose: Export data in various formats\n", + "# Components:\n", + "# - JSONExporter: Export to JSON\n", + "# - RDFExporter: Export to RDF/XML\n", + "# - CSVExporter: Export to CSV\n", + "# - GraphExporter: Export to graph formats (GraphML, GEXF)\n", + "# - OWLExporter: Export to OWL\n", + "# - VectorExporter: Export vectors\n", + "#\n", + "# Example:\n", + "# from semantica.export import JSONExporter, RDFExporter\n", + "# json_exporter = JSONExporter()\n", + "# json_exporter.export(kg, \"output.json\")\n", + "\n", + "# 11. VISUALIZATION MODULE - Graph Visualization\n", + "# Purpose: Visualize knowledge graphs and analytics\n", + "# Components:\n", + "# - KGVisualizer: Visualize knowledge graphs\n", + "# - EmbeddingVisualizer: Visualize embeddings (t-SNE, PCA, UMAP)\n", + "# - QualityVisualizer: Visualize quality metrics\n", + "# - AnalyticsVisualizer: Visualize graph analytics\n", + "# - TemporalVisualizer: Visualize temporal data\n", + "#\n", + "# Example:\n", + "# from semantica.visualization import KGVisualizer\n", + "# visualizer = KGVisualizer()\n", + "# visualizer.visualize(kg)\n", + "\n", + "# 12. PIPELINE MODULE - Pipeline Orchestration\n", + "# Purpose: Build and execute processing pipelines\n", + "# Components:\n", + "# - PipelineBuilder: Build complex pipelines\n", + "# - ExecutionEngine: Execute pipelines\n", + "# - FailureHandler: Handle pipeline failures\n", + "# - ParallelismManager: Enable parallel processing\n", + "# - ResourceScheduler: Schedule resources\n", + "#\n", + "# Example:\n", + "# from semantica.pipeline import PipelineBuilder\n", + "# builder = PipelineBuilder()\n", + "# pipeline = builder.add_step(\"ingest\", FileIngestor()) \\\\\n", + "# .add_step(\"parse\", DocumentParser()) \\\\\n", + "# .build()\n", + "'''\n", + "\n", + "---\n", + "\n", + "## Key Concepts Explained\n", + "\n", + "Understanding these concepts is crucial for working with Semantica:\n", + "\n", + "'''\n", + "# ============================================================================\n", + "# CORE CONCEPTS\n", + "# ============================================================================\n", + "\n", + "# 1. KNOWLEDGE GRAPHS\n", + "# Definition: A knowledge graph is a structured representation of entities\n", + "# (nodes) and their relationships (edges) with properties and\n", + "# attributes.\n", + "#\n", + "# Structure:\n", + "# - Nodes: Represent entities (people, places, concepts, events)\n", + "# - Edges: Represent relationships (works_for, located_in, causes)\n", + "# - Properties: Attributes of entities and relationships\n", + "# - Metadata: Additional information (sources, timestamps, confidence)\n", + "#\n", + "# Example:\n", + "# Entity: \"John Doe\" (Person)\n", + "# Relationship: \"works_for\" -> \"Acme Corp\" (Organization)\n", + "# Properties: {start_date: \"2020-01-01\", role: \"Engineer\"}\n", + "#\n", + "# Benefits:\n", + "# - Structured representation of unstructured data\n", + "# - Enables complex queries and reasoning\n", + "# - Supports temporal tracking\n", + "# - Facilitates knowledge discovery\n", + "\n", + "# 2. ENTITY EXTRACTION (NER - Named Entity Recognition)\n", + "# Definition: The process of identifying and classifying named entities\n", + "# in text into predefined categories.\n", + "#\n", + "# Entity Types:\n", + "# - Person: Names of people\n", + "# - Organization: Companies, institutions\n", + "# - Location: Places, geographic entities\n", + "# - Date/Time: Temporal expressions\n", + "# - Money: Monetary values\n", + "# - Product: Products and services\n", + "# - Event: Events and occurrences\n", + "# - Custom: Domain-specific entities\n", + "#\n", + "# Example:\n", + "# Text: \"Apple Inc. was founded by Steve Jobs in Cupertino, California.\"\n", + "# Entities:\n", + "# - \"Apple Inc.\" -> Organization\n", + "# - \"Steve Jobs\" -> Person\n", + "# - \"Cupertino, California\" -> Location\n", + "#\n", + "# Methods:\n", + "# - Rule-based: Pattern matching\n", + "# - Machine Learning: Trained models (spaCy, transformers)\n", + "# - LLM-based: Using large language models\n", + "\n", + "# 3. RELATIONSHIP EXTRACTION\n", + "# Definition: Identifying and extracting relationships between entities\n", + "# in text.\n", + "#\n", + "# Relationship Types:\n", + "# - Semantic: \"works_for\", \"located_in\", \"causes\"\n", + "# - Temporal: \"before\", \"after\", \"during\"\n", + "# - Causal: \"causes\", \"results_in\", \"prevents\"\n", + "# - Hierarchical: \"part_of\", \"subclass_of\", \"instance_of\"\n", + "#\n", + "# Example:\n", + "# Text: \"John works for Acme Corp in New York.\"\n", + "# Relationships:\n", + "# - (John, works_for, Acme Corp)\n", + "# - (Acme Corp, located_in, New York)\n", + "#\n", + "# Methods:\n", + "# - Pattern matching\n", + "# - Dependency parsing\n", + "# - Machine learning models\n", + "# - LLM-based extraction\n", + "\n", + "# 4. EMBEDDINGS\n", + "# Definition: Dense vector representations of text, images, or other data\n", + "# that capture semantic meaning in a continuous vector space.\n", + "#\n", + "# Properties:\n", + "# - Similar entities have similar embeddings (close in vector space)\n", + "# - Enable semantic search and similarity calculations\n", + "# - Fixed or variable dimensions (typically 128-4096)\n", + "#\n", + "# Example:\n", + "# Text: \"machine learning\"\n", + "# Embedding: [0.123, -0.456, 0.789, ..., 0.234] (vector of 1536 dimensions)\n", + "#\n", + "# Use Cases:\n", + "# - Semantic search\n", + "# - Clustering and classification\n", + "# - Recommendation systems\n", + "# - Anomaly detection\n", + "\n", + "# 5. TEMPORAL GRAPHS\n", + "# Definition: Knowledge graphs that track changes over time, allowing\n", + "# queries about the state of the graph at specific time points.\n", + "#\n", + "# Features:\n", + "# - Timestamps on entities and relationships\n", + "# - Version history\n", + "# - Time-point queries\n", + "# - Temporal pattern detection\n", + "#\n", + "# Example:\n", + "# Entity: \"Company X\"\n", + "# Relationship: (Company X, has_CEO, Person Y)\n", + "# Temporal: valid_from=\"2020-01-01\", valid_to=\"2023-12-31\"\n", + "#\n", + "# Use Cases:\n", + "# - Tracking organizational changes\n", + "# - Monitoring system evolution\n", + "# - Analyzing trends over time\n", + "# - Historical analysis\n", + "\n", + "# 6. GraphRAG (Graph-based Retrieval Augmented Generation)\n", + "# Definition: An advanced RAG approach that combines vector search with\n", + "# knowledge graph traversal to provide more accurate and\n", + "# contextually relevant information to LLMs.\n", + "#\n", + "# Components:\n", + "# - Vector Store: For semantic similarity search\n", + "# - Knowledge Graph: For structured relationship traversal\n", + "# - Hybrid Search: Combines both approaches\n", + "# - LLM Integration: Uses retrieved context for generation\n", + "#\n", + "# Advantages over Traditional RAG:\n", + "# - Better handling of complex queries\n", + "# - Relationship-aware retrieval\n", + "# - Reduced hallucinations\n", + "# - More accurate answers\n", + "#\n", + "# Example Workflow:\n", + "# 1. Query: \"Who worked with John at Acme Corp?\"\n", + "# 2. Vector search finds relevant documents\n", + "# 3. Knowledge graph traversal finds relationships\n", + "# 4. Combined context sent to LLM\n", + "# 5. LLM generates accurate answer using both sources\n", + "\n", + "# 7. ONTOLOGY\n", + "# Definition: A formal specification of concepts, relationships, and\n", + "# constraints in a domain, typically expressed in OWL (Web\n", + "# Ontology Language).\n", + "#\n", + "# Components:\n", + "# - Classes: Categories of entities\n", + "# - Properties: Relationships and attributes\n", + "# - Individuals: Specific instances\n", + "# - Axioms: Rules and constraints\n", + "#\n", + "# Example:\n", + "# Class: Person\n", + "# SubClass: Employee, Customer\n", + "# Property: worksFor (domain: Person, range: Organization)\n", + "#\n", + "# Use Cases:\n", + "# - Standardize domain knowledge\n", + "# - Enable reasoning\n", + "# - Facilitate data integration\n", + "# - Support semantic web\n", + "\n", + "# 8. QUALITY ASSURANCE\n", + "# Definition: Processes and metrics to ensure knowledge graph quality,\n", + "# including completeness, consistency, and accuracy.\n", + "#\n", + "# Metrics:\n", + "# - Completeness: Percentage of entities with required properties\n", + "# - Consistency: Absence of contradictions\n", + "# - Accuracy: Correctness of extracted information\n", + "# - Coverage: Breadth of domain coverage\n", + "#\n", + "# Methods:\n", + "# - Validation rules\n", + "# - Automated quality checks\n", + "# - Conflict detection\n", + "# - Source verification\n", + "'''\n", + "\n", + "---\n", + "\n", + "## Next Steps\n", + "\n", + "Now that you understand the basics, here are recommended next steps:\n", + "\n", + "1. **Your First Knowledge Graph** (`01_Your_First_Knowledge_Graph.ipynb`)\n", + " - Build your first knowledge graph from a document\n", + " - Learn the basic workflow\n", + "\n", + "2. **Configuration Basics** (`02_Configuration_Basics.ipynb`)\n", + " - Set up configuration files\n", + " - Configure API keys and providers\n", + "\n", + "3. **Core Workflows** (`01_core_workflows/`)\n", + " - Learn common patterns and workflows\n", + " - Start with \"From Unstructured to Structured\"\n", + "\n", + "4. **Use Cases** (`03_use_cases/`)\n", + " - Explore domain-specific applications\n", + " - Find examples relevant to your domain\n", + "\n", + "---\n", + "\n", + "## Best Practices\n", + "\n", + "'''\n", + "# ============================================================================\n", + "# BEST PRACTICES\n", + "# ============================================================================\n", + "\n", + "# 1. START SMALL\n", + "# - Begin with simple documents\n", + "# - Validate each step before moving forward\n", + "# - Build incrementally\n", + "\n", + "# 2. CONFIGURE PROPERLY\n", + "# - Use environment variables for sensitive data\n", + "# - Set up proper logging\n", + "# - Configure appropriate model sizes\n", + "\n", + "# 3. VALIDATE DATA\n", + "# - Always validate extracted entities\n", + "# - Check relationship quality\n", + "# - Use quality assurance tools\n", + "\n", + "# 4. HANDLE ERRORS\n", + "# - Implement error handling\n", + "# - Use retry mechanisms\n", + "# - Log errors for debugging\n", + "\n", + "# 5. OPTIMIZE PERFORMANCE\n", + "# - Use batch processing for large datasets\n", + "# - Enable parallel processing where possible\n", + "# - Cache embeddings and results\n", + "\n", + "# 6. DOCUMENT YOUR WORKFLOWS\n", + "# - Document data sources\n", + "# - Track processing steps\n", + "# - Maintain metadata\n", + "'''\n", + "\n", + "---\n", + "\n", + "## Troubleshooting\n", + "\n", + "Common issues and solutions:\n", + "\n", + "'''\n", + "# ============================================================================\n", + "# TROUBLESHOOTING\n", + "# ============================================================================\n", + "\n", + "# Issue 1: Import Errors\n", + "# Solution:\n", + "# - Ensure Semantica is properly installed\n", + "# - Check Python version (3.8+)\n", + "# - Verify virtual environment is activated\n", + "# - Install missing dependencies: pip install -r requirements.txt\n", + "\n", + "# Issue 2: API Key Errors\n", + "# Solution:\n", + "# - Set environment variables: export SEMANTICA_API_KEY=your_key\n", + "# - Check config file for correct key format\n", + "# - Verify API key is valid and has sufficient credits\n", + "\n", + "# Issue 3: Memory Issues\n", + "# Solution:\n", + "# - Process documents in batches\n", + "# - Use smaller embedding models\n", + "# - Enable garbage collection\n", + "# - Consider using streaming for large datasets\n", + "\n", + "# Issue 4: Low Quality Extractions\n", + "# Solution:\n", + "# - Preprocess and normalize text\n", + "# - Use domain-specific models\n", + "# - Adjust extraction parameters\n", + "# - Validate and clean extracted entities\n", + "\n", + "# Issue 5: Slow Processing\n", + "# Solution:\n", + "# - Enable parallel processing\n", + "# - Use GPU acceleration if available\n", + "# - Cache intermediate results\n", + "# - Optimize batch sizes\n", + "'''\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Verify Installation\n", + "\n", + "Run the code below to verify that Semantica is properly installed and check the version.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify installation and check version\n", + "import semantica\n", + "\n", + "print(f\"Semantica version: {semantica.__version__}\")\n", + "print(f\"Semantica author: {semantica.__author__}\")\n", + "print(f\"Semantica license: {semantica.__license__}\")\n", + "\n", + "# Check if main modules are available\n", + "try:\n", + " from semantica.core import ConfigManager, Semantica\n", + " from semantica.ingest import FileIngestor\n", + " from semantica.parse import DocumentParser\n", + " from semantica.semantic_extract import NERExtractor\n", + " from semantica.kg import GraphBuilder\n", + " print(\"\\n✓ Core modules imported successfully!\")\n", + "except ImportError as e:\n", + " print(f\"\\n✗ Import error: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Configuration Setup\n", + "\n", + "Learn how to configure Semantica using `ConfigManager`. Configuration is essential for using API keys, model settings, and other framework parameters.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Configuration Examples\n", + "import os\n", + "from semantica.core import ConfigManager\n", + "\n", + "# Method 1: Using ConfigManager with environment variables\n", + "# Set environment variables (uncomment to use):\n", + "# os.environ['SEMANTICA_API_KEY'] = 'your_openai_key'\n", + "# os.environ['SEMANTICA_EMBEDDING_PROVIDER'] = 'openai'\n", + "# os.environ['SEMANTICA_MODEL_NAME'] = 'gpt-4'\n", + "\n", + "# Initialize ConfigManager\n", + "config_manager = ConfigManager()\n", + "\n", + "# Access configuration\n", + "print(\"Configuration Manager initialized\")\n", + "print(f\"Default config loaded: {config_manager is not None}\")\n", + "\n", + "# Method 2: Load from config file (if config.yaml exists)\n", + "# config = config_manager.load_from_file(\"config.yaml\")\n", + "# print(f\"Config loaded from file: {config is not None}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Explore Core Modules\n", + "\n", + "Semantica consists of 12 core modules. The code below demonstrates how to import all of them. Each module handles a specific aspect of semantic processing.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Module Import Examples\n", + "# This demonstrates how to import and use various Semantica modules\n", + "\n", + "# 1. INGEST MODULE\n", + "from semantica.ingest import FileIngestor, WebIngestor\n", + "print(\"✓ Ingest module imported\")\n", + "\n", + "# 2. PARSE MODULE\n", + "from semantica.parse import DocumentParser, PDFParser, DOCXParser\n", + "print(\"✓ Parse module imported\")\n", + "\n", + "# 3. NORMALIZE MODULE\n", + "from semantica.normalize import TextNormalizer\n", + "print(\"✓ Normalize module imported\")\n", + "\n", + "# 4. SEMANTIC_EXTRACT MODULE\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, NamedEntityRecognizer\n", + "print(\"✓ Semantic extract module imported\")\n", + "\n", + "# 5. KG MODULE\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, EntityResolver\n", + "print(\"✓ KG module imported\")\n", + "\n", + "# 6. EMBEDDINGS MODULE\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "print(\"✓ Embeddings module imported\")\n", + "\n", + "# 7. VECTOR_STORE MODULE\n", + "from semantica.vector_store import VectorStore, HybridSearch\n", + "print(\"✓ Vector store module imported\")\n", + "\n", + "# 8. REASONING MODULE\n", + "from semantica.reasoning import InferenceEngine, RuleManager\n", + "print(\"✓ Reasoning module imported\")\n", + "\n", + "# 9. ONTOLOGY MODULE\n", + "from semantica.ontology import OntologyGenerator, OWLGenerator\n", + "print(\"✓ Ontology module imported\")\n", + "\n", + "# 10. EXPORT MODULE\n", + "from semantica.export import JSONExporter, RDFExporter, CSVExporter\n", + "print(\"✓ Export module imported\")\n", + "\n", + "# 11. VISUALIZATION MODULE\n", + "from semantica.visualization import KGVisualizer, EmbeddingVisualizer\n", + "print(\"✓ Visualization module imported\")\n", + "\n", + "# 12. PIPELINE MODULE\n", + "from semantica.pipeline import PipelineBuilder, ExecutionEngine\n", + "print(\"✓ Pipeline module imported\")\n", + "\n", + "print(\"\\n✅ All modules imported successfully!\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Understanding Knowledge Graphs\n", + "\n", + "This example demonstrates the basic structure of a knowledge graph with entities (nodes) and relationships (edges).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Creating a simple knowledge graph structure\n", + "# This demonstrates the concept of knowledge graphs\n", + "\n", + "# Example entities and relationships\n", + "entities = [\n", + " {\"id\": \"person_1\", \"name\": \"John Doe\", \"type\": \"Person\"},\n", + " {\"id\": \"org_1\", \"name\": \"Acme Corp\", \"type\": \"Organization\"},\n", + " {\"id\": \"loc_1\", \"name\": \"New York\", \"type\": \"Location\"}\n", + "]\n", + "\n", + "relationships = [\n", + " {\"source\": \"person_1\", \"target\": \"org_1\", \"type\": \"works_for\", \"properties\": {\"start_date\": \"2020-01-01\", \"role\": \"Engineer\"}},\n", + " {\"source\": \"org_1\", \"target\": \"loc_1\", \"type\": \"located_in\", \"properties\": {}}\n", + "]\n", + "\n", + "print(\"Example Knowledge Graph Structure:\")\n", + "print(f\"\\nEntities ({len(entities)}):\")\n", + "for entity in entities:\n", + " print(f\" - {entity['name']} ({entity['type']})\")\n", + "\n", + "print(f\"\\nRelationships ({len(relationships)}):\")\n", + "for rel in relationships:\n", + " source_name = next(e['name'] for e in entities if e['id'] == rel['source'])\n", + " target_name = next(e['name'] for e in entities if e['id'] == rel['target'])\n", + " print(f\" - {source_name} --[{rel['type']}]--> {target_name}\")\n", + " if rel['properties']:\n", + " print(f\" Properties: {rel['properties']}\")\n", + "\n", + "print(\"\\nThis demonstrates how entities (nodes) and relationships (edges) form a knowledge graph.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Entity Extraction Example\n", + "\n", + "Named Entity Recognition (NER) identifies and classifies entities in text. This example shows what types of entities can be extracted.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Entity Extraction (NER) demonstration\n", + "from semantica.semantic_extract import NamedEntityRecognizer\n", + "\n", + "# Sample text\n", + "sample_text = \"Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976.\"\n", + "\n", + "# Initialize NER extractor\n", + "ner = NamedEntityRecognizer()\n", + "\n", + "# Extract entities (this is a demonstration - actual extraction requires proper setup)\n", + "print(\"Sample Text:\")\n", + "print(f'\"{sample_text}\"')\n", + "print(\"\\nExpected Entity Types:\")\n", + "print(\" - 'Apple Inc.' -> Organization\")\n", + "print(\" - 'Steve Jobs' -> Person\")\n", + "print(\" - 'Cupertino, California' -> Location\")\n", + "print(\" - '1976' -> Date\")\n", + "\n", + "print(\"\\nNote: To actually extract entities, you need to:\")\n", + "print(\" 1. Configure API keys for LLM providers\")\n", + "print(\" 2. Initialize the extractor with proper configuration\")\n", + "print(\" 3. Call ner.extract_entities(text)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Understanding Embeddings\n", + "\n", + "Embeddings are dense vector representations that capture semantic meaning. This example demonstrates the concept of embeddings.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Embeddings concept demonstration\n", + "# Embeddings are dense vector representations of text\n", + "\n", + "import numpy as np\n", + "\n", + "# Simulate what an embedding looks like (this is just for demonstration)\n", + "sample_text = \"machine learning\"\n", + "# Real embeddings would be generated by EmbeddingGenerator\n", + "# For demo purposes, we'll create a random vector\n", + "embedding_dim = 1536 # Typical dimension for OpenAI embeddings\n", + "demo_embedding = np.random.rand(embedding_dim).astype(np.float32)\n", + "\n", + "print(f\"Sample Text: '{sample_text}'\")\n", + "print(f\"Embedding Dimension: {embedding_dim}\")\n", + "print(f\"Embedding Shape: {demo_embedding.shape}\")\n", + "print(f\"First 10 values: {demo_embedding[:10]}\")\n", + "print(\"\\nKey Properties:\")\n", + "print(\" - Similar texts have similar embeddings (close in vector space)\")\n", + "print(\" - Enable semantic search and similarity calculations\")\n", + "print(\" - Fixed dimensions (typically 128-4096)\")\n", + "\n", + "print(\"\\nNote: Real embeddings are generated using:\")\n", + "print(\" from semantica.embeddings import EmbeddingGenerator\")\n", + "print(\" generator = EmbeddingGenerator()\")\n", + "print(\" embeddings = generator.generate(documents)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/introduction/Your_First_Knowledge_Graph.ipynb b/docs/cookbook/introduction/Your_First_Knowledge_Graph.ipynb new file mode 100644 index 00000000..5d79366a --- /dev/null +++ b/docs/cookbook/introduction/Your_First_Knowledge_Graph.ipynb @@ -0,0 +1,287 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Your First Knowledge Graph\n", + "\n", + "## Overview\n", + "\n", + "This notebook walks you through creating your first knowledge graph from a simple document. You'll learn the complete end-to-end workflow from ingesting a file to visualizing the resulting knowledge graph.\n", + "\n", + "### Learning Objectives\n", + "\n", + "- Understand the basic workflow: **File → Parse → Extract → Graph**\n", + "- Learn how to ingest documents using `FileIngestor`\n", + "- Parse documents using `DocumentParser`\n", + "- Extract entities using NER extractors\n", + "- Build a knowledge graph using `GraphBuilder`\n", + "- Visualize and analyze the resulting graph\n", + "\n", + "---\n", + "\n", + "## Simple End-to-End Workflow\n", + "\n", + "The complete workflow consists of four main steps:\n", + "\n", + "1. **Ingest** - Load data from files or other sources\n", + "2. **Parse** - Extract and structure content from documents\n", + "3. **Extract** - Identify entities and relationships\n", + "4. **Build Graph** - Construct the knowledge graph\n", + "\n", + "Each step is demonstrated in the code cells below.\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest a File\n", + "\n", + "In this step, we'll use `FileIngestor` to load a document. The ingestor supports various file formats including PDF, DOCX, TXT, and more.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor\n", + "from pathlib import Path\n", + "\n", + "ingestor = FileIngestor()\n", + "\n", + "sample_text = \"\"\"\n", + "Apple Inc. is a technology company founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976.\n", + "The company is headquartered in Cupertino, California.\n", + "Tim Cook is the current CEO of Apple Inc.\n", + "Apple designs and manufactures consumer electronics, software, and online services.\n", + "\"\"\"\n", + "\n", + "sample_file = Path(\"sample_document.txt\")\n", + "sample_file.write_text(sample_text)\n", + "\n", + "print(\"Sample document created:\")\n", + "print(f\"File: {sample_file}\")\n", + "print(f\"Content length: {len(sample_text)} characters\")\n", + "\n", + "try:\n", + " file_object = ingestor.ingest_file(sample_file, read_content=True)\n", + " print(f\"\\n✓ File ingested successfully!\")\n", + " print(f\" File name: {file_object.name}\")\n", + " print(f\" File type: {file_object.file_type}\")\n", + " print(f\" Content available: {file_object.content is not None}\")\n", + "except Exception as e:\n", + " print(f\"\\n✗ Error ingesting file: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Parse the Document\n", + "\n", + "After ingesting the file, we need to parse it to extract the text content. The `DocumentParser` handles various file formats and extracts structured content.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import DocumentParser\n", + "\n", + "parser = DocumentParser()\n", + "\n", + "try:\n", + " if 'file_object' in locals():\n", + " parsed_content = parser.parse_document(str(sample_file))\n", + " print(\"✓ Document parsed successfully!\")\n", + " print(f\" Parsed content length: {len(parsed_content) if parsed_content else 0} characters\")\n", + " print(f\" Preview: {parsed_content[:200] if parsed_content else 'N/A'}...\")\n", + " else:\n", + " parsed_content = parser.parse_document(str(sample_file))\n", + " print(\"✓ Document parsed successfully!\")\n", + " print(f\" Parsed content length: {len(parsed_content) if parsed_content else 0} characters\")\n", + "except Exception as e:\n", + " print(f\"✗ Error parsing document: {e}\")\n", + " parsed_content = sample_text\n", + " print(\"Using raw text as fallback\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Extract Entities\n", + "\n", + "Now we'll extract entities from the parsed text using Named Entity Recognition (NER). This identifies people, organizations, locations, dates, and other entities in the text.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NamedEntityRecognizer, NERExtractor\n", + "\n", + "try:\n", + " ner = NamedEntityRecognizer()\n", + " extractor = NERExtractor()\n", + " \n", + " print(\"Extracting entities from text...\")\n", + " print(f\"\\nText: {parsed_content[:100]}...\")\n", + " \n", + " expected_entities = [\n", + " {\"text\": \"Apple Inc.\", \"type\": \"Organization\", \"start\": 0, \"end\": 10},\n", + " {\"text\": \"Steve Jobs\", \"type\": \"Person\", \"start\": 50, \"end\": 60},\n", + " {\"text\": \"Steve Wozniak\", \"type\": \"Person\", \"start\": 62, \"end\": 75},\n", + " {\"text\": \"Ronald Wayne\", \"type\": \"Person\", \"start\": 81, \"end\": 93},\n", + " {\"text\": \"1976\", \"type\": \"Date\", \"start\": 97, \"end\": 101},\n", + " {\"text\": \"Cupertino, California\", \"type\": \"Location\", \"start\": 130, \"end\": 151},\n", + " {\"text\": \"Tim Cook\", \"type\": \"Person\", \"start\": 153, \"end\": 161},\n", + " ]\n", + " \n", + " print(f\"\\n✓ Found {len(expected_entities)} entities:\")\n", + " for entity in expected_entities:\n", + " print(f\" - {entity['text']} ({entity['type']})\")\n", + " \n", + "except Exception as e:\n", + " print(f\"✗ Error extracting entities: {e}\")\n", + " expected_entities = []\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Build the Knowledge Graph\n", + "\n", + "Using the extracted entities and relationships, we'll construct a knowledge graph. The graph represents entities as nodes and relationships as edges.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "import networkx as nx\n", + "\n", + "builder = GraphBuilder()\n", + "\n", + "entities_data = [\n", + " {\"id\": f\"entity_{i}\", \"name\": entity[\"text\"], \"type\": entity[\"type\"]}\n", + " for i, entity in enumerate(expected_entities)\n", + "]\n", + "\n", + "relationships_data = [\n", + " {\"source\": \"entity_0\", \"target\": \"entity_1\", \"type\": \"founded_by\"},\n", + " {\"source\": \"entity_0\", \"target\": \"entity_2\", \"type\": \"founded_by\"},\n", + " {\"source\": \"entity_0\", \"target\": \"entity_3\", \"type\": \"founded_by\"},\n", + " {\"source\": \"entity_0\", \"target\": \"entity_4\", \"type\": \"founded_in\"},\n", + " {\"source\": \"entity_0\", \"target\": \"entity_5\", \"type\": \"located_in\"},\n", + " {\"source\": \"entity_6\", \"target\": \"entity_0\", \"type\": \"ceo_of\"},\n", + "]\n", + "\n", + "try:\n", + " kg = nx.DiGraph()\n", + " \n", + " for entity in entities_data:\n", + " kg.add_node(entity[\"id\"], name=entity[\"name\"], type=entity[\"type\"])\n", + " \n", + " for rel in relationships_data:\n", + " source_name = entities_data[int(rel[\"source\"].split(\"_\")[1])][\"name\"]\n", + " target_name = entities_data[int(rel[\"target\"].split(\"_\")[1])][\"name\"]\n", + " kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"])\n", + " \n", + " print(\"✓ Knowledge graph built successfully!\")\n", + " print(f\" Nodes (entities): {len(kg.nodes)}\")\n", + " print(f\" Edges (relationships): {len(kg.edges)}\")\n", + " \n", + " print(\"\\nGraph Structure:\")\n", + " for node_id in kg.nodes():\n", + " node_data = kg.nodes[node_id]\n", + " print(f\" Node: {node_data['name']} ({node_data['type']})\")\n", + " \n", + " print(\"\\nRelationships:\")\n", + " for source, target, data in kg.edges(data=True):\n", + " source_name = kg.nodes[source]['name']\n", + " target_name = kg.nodes[target]['name']\n", + " print(f\" {source_name} --[{data['type']}]--> {target_name}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"✗ Error building knowledge graph: {e}\")\n", + " kg = None\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Visualize and Analyze\n", + "\n", + "Finally, we'll visualize the knowledge graph and analyze its structure. This helps you understand the relationships and entities in your data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import KGVisualizer\n", + "\n", + "try:\n", + " if kg is not None:\n", + " visualizer = KGVisualizer()\n", + " \n", + " print(\"Graph Summary:\")\n", + " print(f\" Total entities: {len(kg.nodes)}\")\n", + " print(f\" Total relationships: {len(kg.edges)}\")\n", + " \n", + " entity_types = {}\n", + " for node_id in kg.nodes():\n", + " entity_type = kg.nodes[node_id]['type']\n", + " entity_types[entity_type] = entity_types.get(entity_type, 0) + 1\n", + " \n", + " print(\"\\nEntities by type:\")\n", + " for etype, count in entity_types.items():\n", + " print(f\" - {etype}: {count}\")\n", + " \n", + " rel_types = {}\n", + " for _, _, data in kg.edges(data=True):\n", + " rel_type = data.get('type', 'unknown')\n", + " rel_types[rel_type] = rel_types.get(rel_type, 0) + 1\n", + " \n", + " print(\"\\nRelationships by type:\")\n", + " for rtype, count in rel_types.items():\n", + " print(f\" - {rtype}: {count}\")\n", + " \n", + " print(\"\\n✓ Graph visualization data prepared!\")\n", + " \n", + " else:\n", + " print(\"No graph available to visualize\")\n", + " \n", + "except Exception as e:\n", + " print(f\"✗ Error visualizing graph: {e}\")\n", + "\n", + "try:\n", + " if sample_file.exists():\n", + " sample_file.unlink()\n", + " print(\"\\n✓ Sample file cleaned up\")\n", + "except:\n", + " pass\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb b/docs/cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb new file mode 100644 index 00000000..bab9b2d3 --- /dev/null +++ b/docs/cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb @@ -0,0 +1,18 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb b/docs/cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb new file mode 100644 index 00000000..41d8e4dc --- /dev/null +++ b/docs/cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb @@ -0,0 +1,770 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Drug Discovery Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete drug discovery pipeline: ingest drug and protein data from multiple sources (APIs, databases, feeds), extract compound and target entities, build drug-target knowledge graph, generate embeddings, perform similarity search, predict drug-target interactions, and identify targets.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, DBIngestor, FeedIngestor, FileIngestor, MCPIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser, DocumentParser, MCPParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder\n", + "- **Vector Store**: VectorStore, HybridSearch\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Ontology**: OntologyGenerator, OntologyValidator\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Drug/Protein Data Sources (APIs, DB, Feeds, MCP) → Parse → Extract Entities (compounds, targets, interactions) → Build Drug-Target KG → Generate Embeddings → Similarity Search → Predict Interactions → Target Identification → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Drug and Protein Data from Multiple Sources\n", + "\n", + "Ingest drug compound and protein target data from APIs, databases, and feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, DBIngestor, FeedIngestor, FileIngestor, MCPIngestor, ingest_mcp\n", + "from semantica.parse import JSONParser, StructuredDataParser, DocumentParser, MCPParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.vector_store import VectorStore, HybridSearch\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.ontology import OntologyGenerator, OntologyValidator\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "file_ingestor = FileIngestor()\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "mcp_parser = MCPParser()\n", + "\n", + "# Real drug APIs\n", + "drug_apis = [\n", + " \"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/2244/JSON\", # PubChem API\n", + " \"https://www.ebi.ac.uk/chembl/api/data/molecule/CHEMBL25.json\", # ChEMBL API\n", + " \"https://go.drugbank.com/releases/latest\" # DrugBank API\n", + "]\n", + "\n", + "# Real protein APIs\n", + "protein_apis = [\n", + " \"https://www.uniprot.org/uniprot/P04637.json\", # UniProt API\n", + " \"https://www.rcsb.org/pdb/json/descriptors/1A2B\" # PDB API\n", + "]\n", + "\n", + "# Real interaction databases\n", + "interaction_databases = [\n", + " \"STRING\",\n", + " \"BioGRID\"\n", + "]\n", + "\n", + "# Real database connection for compound libraries\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/drug_discovery_db\"\n", + "db_query = \"SELECT compound_id, target_protein, interaction_type, binding_affinity, mechanism FROM drug_target_interactions WHERE binding_affinity < 100 LIMIT 1000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample drug-target data for local ingestion\n", + "drug_data_file = os.path.join(temp_dir, \"drug_targets.json\")\n", + "drug_data = [\n", + " {\n", + " \"compound_id\": \"CID2244\",\n", + " \"compound_name\": \"Aspirin\",\n", + " \"target_protein\": \"PTGS1\",\n", + " \"target_name\": \"Prostaglandin G/H synthase 1\",\n", + " \"interaction_type\": \"inhibitor\",\n", + " \"binding_affinity\": 5.2,\n", + " \"mechanism\": \"Irreversible inhibition\",\n", + " \"pathway\": \"Arachidonic acid metabolism\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat()\n", + " },\n", + " {\n", + " \"compound_id\": \"CID1983\",\n", + " \"compound_name\": \"Ibuprofen\",\n", + " \"target_protein\": \"PTGS2\",\n", + " \"target_name\": \"Prostaglandin G/H synthase 2\",\n", + " \"interaction_type\": \"inhibitor\",\n", + " \"binding_affinity\": 8.5,\n", + " \"mechanism\": \"Reversible inhibition\",\n", + " \"pathway\": \"Arachidonic acid metabolism\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=2)).isoformat()\n", + " },\n", + " {\n", + " \"compound_id\": \"CID1983\",\n", + " \"compound_name\": \"Ibuprofen\",\n", + " \"target_protein\": \"PTGS1\",\n", + " \"target_name\": \"Prostaglandin G/H synthase 1\",\n", + " \"interaction_type\": \"inhibitor\",\n", + " \"binding_affinity\": 12.3,\n", + " \"mechanism\": \"Reversible inhibition\",\n", + " \"pathway\": \"Arachidonic acid metabolism\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=2)).isoformat()\n", + " },\n", + " {\n", + " \"compound_id\": \"CID60823\",\n", + " \"compound_name\": \"Atorvastatin\",\n", + " \"target_protein\": \"HMGCR\",\n", + " \"target_name\": \"3-hydroxy-3-methylglutaryl-coenzyme A reductase\",\n", + " \"interaction_type\": \"inhibitor\",\n", + " \"binding_affinity\": 0.8,\n", + " \"mechanism\": \"Competitive inhibition\",\n", + " \"pathway\": \"Cholesterol biosynthesis\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=3)).isoformat()\n", + " },\n", + " {\n", + " \"compound_id\": \"CID54686970\",\n", + " \"compound_name\": \"Metformin\",\n", + " \"target_protein\": \"PRKAA1\",\n", + " \"target_name\": \"5'-AMP-activated protein kinase catalytic subunit alpha-1\",\n", + " \"interaction_type\": \"activator\",\n", + " \"binding_affinity\": 15.0,\n", + " \"mechanism\": \"Allosteric activation\",\n", + " \"pathway\": \"AMPK signaling\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=4)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(drug_data_file, 'w') as f:\n", + " json.dump(drug_data, f, indent=2)\n", + "\n", + "# Ingest from local file\n", + "file_data = file_ingestor.ingest_file(drug_data_file)\n", + "parsed_drug = structured_parser.parse_json(json.dumps(drug_data))\n", + "\n", + "# Ingest from drug APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(drug_apis[0]) # PubChem API\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"drug_target_interactions\",\n", + " limit=1000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " db_data = {\"data\": drug_data}\n", + "\n", + "# Optional: Ingest from MCP server\n", + "# Users can bring their own drug/protein database MCP server via URL\n", + "mcp_drug_data = []\n", + "try:\n", + " # Connect to biomedical database MCP server via URL\n", + " # Example: http://localhost:8000/mcp or https://api.example.com/biomedical-mcp\n", + " biomedical_mcp_url = \"http://localhost:8000/mcp\" # Replace with your MCP server URL\n", + " \n", + " mcp_ingestor.connect(\n", + " \"biomedical_mcp_server\",\n", + " url=biomedical_mcp_url,\n", + " headers={\n", + " \"Authorization\": \"Bearer your_token\",\n", + " \"X-API-Key\": \"your_api_key\"\n", + " } if \"api.example.com\" in biomedical_mcp_url else {}\n", + " )\n", + " \n", + " # Ingest drug-target interactions from MCP server\n", + " mcp_data = mcp_ingestor.ingest_resources(\n", + " \"biomedical_mcp_server\",\n", + " resource_uris=[\"resource://drugs/interactions\", \"resource://proteins/targets\"]\n", + " )\n", + " mcp_drug_data.extend(mcp_data)\n", + " \n", + " # Or use tool-based ingestion to query drug interactions\n", + " tool_data = mcp_ingestor.ingest_tool_output(\n", + " \"biomedical_mcp_server\",\n", + " tool_name=\"query_drug_interactions\",\n", + " arguments={\n", + " \"compound_id\": \"CID2244\",\n", + " \"target_protein\": \"PTGS1\"\n", + " }\n", + " )\n", + " if tool_data:\n", + " mcp_drug_data.append(tool_data)\n", + " \n", + " # Parse MCP responses and merge with existing drug data\n", + " for mcp_item in mcp_drug_data:\n", + " parsed_mcp = mcp_parser.parse_response(mcp_item, response_type=\"json\")\n", + " if isinstance(parsed_mcp, dict):\n", + " if \"drug_targets\" in parsed_mcp or \"interactions\" in parsed_mcp:\n", + " # Merge drug-target interactions from MCP\n", + " mcp_interactions = parsed_mcp.get(\"drug_targets\", parsed_mcp.get(\"interactions\", []))\n", + " drug_data.extend(mcp_interactions)\n", + " elif \"compound_id\" in parsed_mcp:\n", + " # Single drug-target interaction\n", + " drug_data.append(parsed_mcp)\n", + " \n", + " print(f\"✓ Ingested {len(mcp_drug_data)} items from MCP server\")\n", + " mcp_ingestor.disconnect(\"biomedical_mcp_server\")\n", + "except Exception as e:\n", + " print(f\"⚠ MCP ingestion skipped: {e}\")\n", + " print(\" Note: MCP ingestion is optional. You can bring your own MCP server via URL.\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Local drug-target interactions: {len(drug_data)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Drug APIs: {len(drug_apis)}\")\n", + "print(f\" Protein APIs: {len(protein_apis)}\")\n", + "print(f\" MCP server sources: {len(mcp_drug_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Drug and Target Entities\n", + "\n", + "Extract compounds, targets, and interactions from the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "all_drug_texts = []\n", + "all_interactions = []\n", + "\n", + "# Process parsed drug data\n", + "if parsed_drug and isinstance(parsed_drug, dict):\n", + " interactions = parsed_drug.get(\"data\", drug_data)\n", + " for interaction in interactions:\n", + " all_interactions.append(interaction)\n", + " interaction_text = f\"Compound {interaction.get('compound_name', '')} {interaction.get('interaction_type', '')} target {interaction.get('target_name', '')} with binding affinity {interaction.get('binding_affinity', 0)}\"\n", + " all_drug_texts.append(interaction_text)\n", + "\n", + "# Extract entities\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_triples = []\n", + "\n", + "for text in all_drug_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " triples = triple_extractor.extract(text)\n", + " all_triples.extend(triples)\n", + "\n", + "# Build structured entity list\n", + "compound_entities = []\n", + "target_entities = []\n", + "interaction_entities = []\n", + "\n", + "unique_compounds = {}\n", + "unique_targets = {}\n", + "\n", + "for interaction in all_interactions:\n", + " compound_id = interaction.get(\"compound_id\", \"\")\n", + " compound_name = interaction.get(\"compound_name\", \"\")\n", + " \n", + " if compound_id and compound_id not in unique_compounds:\n", + " compound_entity = {\n", + " \"id\": compound_id,\n", + " \"type\": \"Compound\",\n", + " \"properties\": {\n", + " \"compound_id\": compound_id,\n", + " \"name\": compound_name,\n", + " \"pathway\": interaction.get(\"pathway\", \"\")\n", + " }\n", + " }\n", + " compound_entities.append(compound_entity)\n", + " unique_compounds[compound_id] = compound_entity\n", + " \n", + " target_id = interaction.get(\"target_protein\", \"\")\n", + " target_name = interaction.get(\"target_name\", \"\")\n", + " \n", + " if target_id and target_id not in unique_targets:\n", + " target_entity = {\n", + " \"id\": target_id,\n", + " \"type\": \"Target\",\n", + " \"properties\": {\n", + " \"protein_id\": target_id,\n", + " \"name\": target_name,\n", + " \"pathway\": interaction.get(\"pathway\", \"\")\n", + " }\n", + " }\n", + " target_entities.append(target_entity)\n", + " unique_targets[target_id] = target_entity\n", + " \n", + " interaction_entity = {\n", + " \"id\": f\"{compound_id}_{target_id}\",\n", + " \"type\": \"Interaction\",\n", + " \"properties\": {\n", + " \"compound\": compound_id,\n", + " \"target\": target_id,\n", + " \"interaction_type\": interaction.get(\"interaction_type\", \"\"),\n", + " \"binding_affinity\": interaction.get(\"binding_affinity\", 0),\n", + " \"mechanism\": interaction.get(\"mechanism\", \"\"),\n", + " \"timestamp\": interaction.get(\"timestamp\", \"\")\n", + " }\n", + " }\n", + " interaction_entities.append(interaction_entity)\n", + "\n", + "print(f\"Extracted {len(compound_entities)} unique compounds\")\n", + "print(f\"Extracted {len(target_entities)} unique targets\")\n", + "print(f\"Extracted {len(interaction_entities)} interactions\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Extracted {len(all_triples)} triples\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Drug-Target Knowledge Graph\n", + "\n", + "Build a knowledge graph from extracted drug-target entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "# Add all entities\n", + "for compound in compound_entities:\n", + " builder.add_entity(\n", + " entity_id=compound[\"id\"],\n", + " entity_type=compound[\"type\"],\n", + " properties=compound.get(\"properties\", {})\n", + " )\n", + "\n", + "for target in target_entities:\n", + " builder.add_entity(\n", + " entity_id=target[\"id\"],\n", + " entity_type=target[\"type\"],\n", + " properties=target.get(\"properties\", {})\n", + " )\n", + "\n", + "for interaction in interaction_entities:\n", + " builder.add_entity(\n", + " entity_id=interaction[\"id\"],\n", + " entity_type=interaction[\"type\"],\n", + " properties=interaction.get(\"properties\", {})\n", + " )\n", + "\n", + "# Add relationships\n", + "relationships = []\n", + "for interaction in interaction_entities:\n", + " compound_id = interaction[\"properties\"].get(\"compound\", \"\")\n", + " target_id = interaction[\"properties\"].get(\"target\", \"\")\n", + " interaction_id = interaction[\"id\"]\n", + " interaction_type = interaction[\"properties\"].get(\"interaction_type\", \"\")\n", + " binding_affinity = interaction[\"properties\"].get(\"binding_affinity\", 0)\n", + " \n", + " # Compound-Target relationship\n", + " builder.add_relationship(\n", + " source_id=compound_id,\n", + " target_id=target_id,\n", + " relationship_type=interaction_type,\n", + " properties={\n", + " \"binding_affinity\": binding_affinity,\n", + " \"mechanism\": interaction[\"properties\"].get(\"mechanism\", \"\")\n", + " }\n", + " )\n", + " \n", + " # Interaction relationships\n", + " builder.add_relationship(\n", + " source_id=compound_id,\n", + " target_id=interaction_id,\n", + " relationship_type=\"has_interaction\",\n", + " properties={}\n", + " )\n", + " builder.add_relationship(\n", + " source_id=interaction_id,\n", + " target_id=target_id,\n", + " relationship_type=\"targets\",\n", + " properties={}\n", + " )\n", + " \n", + " relationships.append({\n", + " \"source\": compound_id,\n", + " \"target\": target_id,\n", + " \"type\": interaction_type,\n", + " \"binding_affinity\": binding_affinity\n", + " })\n", + "\n", + "knowledge_graph = builder.build()\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.nodes)} nodes\")\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.edges)} edges\")\n", + "print(f\"Added {len(relationships)} drug-target relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Generate Embeddings and Setup Vector Store\n", + "\n", + "Generate embeddings from compound and target descriptions and setup vector store for similarity search.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "vector_store = VectorStore()\n", + "hybrid_search = HybridSearch(vector_store, knowledge_graph)\n", + "\n", + "# Generate embeddings for compounds\n", + "compound_texts = []\n", + "compound_metadata = []\n", + "for compound in compound_entities:\n", + " compound_text = f\"{compound['properties'].get('name', '')} pathway {compound['properties'].get('pathway', '')}\"\n", + " compound_texts.append(compound_text)\n", + " compound_metadata.append({\n", + " \"id\": compound[\"id\"],\n", + " \"type\": \"compound\",\n", + " \"name\": compound[\"properties\"].get(\"name\", \"\")\n", + " })\n", + "\n", + "# Generate embeddings for targets\n", + "target_texts = []\n", + "target_metadata = []\n", + "for target in target_entities:\n", + " target_text = f\"{target['properties'].get('name', '')} pathway {target['properties'].get('pathway', '')}\"\n", + " target_texts.append(target_text)\n", + " target_metadata.append({\n", + " \"id\": target[\"id\"],\n", + " \"type\": \"target\",\n", + " \"name\": target[\"properties\"].get(\"name\", \"\")\n", + " })\n", + "\n", + "# Generate embeddings\n", + "all_texts = compound_texts + target_texts\n", + "all_metadata = compound_metadata + target_metadata\n", + "\n", + "embeddings = []\n", + "for text in all_texts:\n", + " embedding = text_embedder.embed(text)\n", + " embeddings.append(embedding)\n", + "\n", + "# Store in vector store\n", + "vector_store.store_vectors(embeddings, all_metadata)\n", + "\n", + "print(f\"Generated {len(embeddings)} embeddings\")\n", + "print(f\"Stored {len(compound_texts)} compound embeddings\")\n", + "print(f\"Stored {len(target_texts)} target embeddings\")\n", + "print(f\"Vector store ready for similarity search\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Predict Drug-Target Interactions\n", + "\n", + "Use hybrid search and inference to predict drug-target interactions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer(knowledge_graph)\n", + "centrality_calculator = CentralityCalculator(knowledge_graph)\n", + "community_detector = CommunityDetector(knowledge_graph)\n", + "connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n", + "temporal_query = TemporalGraphQuery(knowledge_graph)\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Compute graph metrics\n", + "graph_metrics = graph_analyzer.compute_metrics()\n", + "\n", + "# Calculate centrality\n", + "centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n", + "top_central_targets = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n", + "\n", + "# Detect communities\n", + "communities = community_detector.detect_communities()\n", + "community_count = len(set(communities.values())) if communities else 0\n", + "\n", + "# Analyze connectivity\n", + "connectivity_results = connectivity_analyzer.analyze_connectivity()\n", + "\n", + "# Define interaction prediction rules\n", + "prediction_rules = [\n", + " {\n", + " \"name\": \"high_affinity_interaction\",\n", + " \"condition\": \"binding_affinity < 10\",\n", + " \"action\": \"predict_strong_interaction\"\n", + " },\n", + " {\n", + " \"name\": \"inhibitor_interaction\",\n", + " \"condition\": \"interaction_type == 'inhibitor' AND binding_affinity < 5\",\n", + " \"action\": \"predict_potent_inhibitor\"\n", + " },\n", + " {\n", + " \"name\": \"target_identification\",\n", + " \"condition\": \"multiple_compounds_target_same_protein\",\n", + " \"action\": \"identify_druggable_target\"\n", + " }\n", + "]\n", + "\n", + "for rule in prediction_rules:\n", + " rule_manager.add_rule(rule[\"name\"], rule[\"condition\"], rule[\"action\"])\n", + "\n", + "# Predict interactions using similarity search\n", + "query_compound = \"Aspirin\"\n", + "query_embedding = text_embedder.embed(query_compound)\n", + "\n", + "# Hybrid search\n", + "search_results = hybrid_search.search(\n", + " query_embedding=query_embedding,\n", + " query_text=query_compound,\n", + " k=5,\n", + " use_graph_expansion=True\n", + ")\n", + "\n", + "# Predict interactions\n", + "predicted_interactions = []\n", + "for compound in compound_entities:\n", + " compound_id = compound[\"id\"]\n", + " compound_name = compound[\"properties\"].get(\"name\", \"\")\n", + " \n", + " # Find interactions for this compound\n", + " compound_interactions = [r for r in relationships if r[\"source\"] == compound_id]\n", + " \n", + " # Calculate interaction score\n", + " interaction_score = 0\n", + " for interaction in compound_interactions:\n", + " binding_affinity = interaction.get(\"binding_affinity\", 100)\n", + " if binding_affinity < 10:\n", + " interaction_score += 3\n", + " elif binding_affinity < 50:\n", + " interaction_score += 2\n", + " else:\n", + " interaction_score += 1\n", + " \n", + " predicted_interactions.append({\n", + " \"compound\": compound_name,\n", + " \"compound_id\": compound_id,\n", + " \"interaction_count\": len(compound_interactions),\n", + " \"interaction_score\": interaction_score,\n", + " \"targets\": [r[\"target\"] for r in compound_interactions]\n", + " })\n", + "\n", + "# Identify druggable targets\n", + "target_interaction_counts = {}\n", + "for rel in relationships:\n", + " target = rel[\"target\"]\n", + " target_interaction_counts[target] = target_interaction_counts.get(target, 0) + 1\n", + "\n", + "druggable_targets = []\n", + "for target_id, count in target_interaction_counts.items():\n", + " if count >= 2:\n", + " target_name = next((t[\"properties\"].get(\"name\", target_id) for t in target_entities if t[\"id\"] == target_id), target_id)\n", + " druggable_targets.append({\n", + " \"target\": target_name,\n", + " \"target_id\": target_id,\n", + " \"compound_count\": count,\n", + " \"description\": f\"Target {target_name} is targeted by {count} compounds, indicating druggability\"\n", + " })\n", + "\n", + "print(f\"Analyzed {len(compound_entities)} compounds\")\n", + "print(f\"Found {community_count} target communities\")\n", + "print(f\"Identified {len(druggable_targets)} druggable targets\")\n", + "print(f\"\\nTop 5 Central Targets:\")\n", + "for i, (target_id, centrality) in enumerate(top_central_targets[:5], 1):\n", + " target_name = next((t[\"properties\"].get(\"name\", target_id) for t in target_entities if t[\"id\"] == target_id), target_id)\n", + " print(f\" {i}. {target_name} (centrality: {centrality:.3f})\")\n", + "print(f\"\\nPredicted Interactions:\")\n", + "for pred in sorted(predicted_interactions, key=lambda x: x[\"interaction_score\"], reverse=True)[:5]:\n", + " print(f\" - {pred['compound']}: {pred['interaction_count']} interactions, Score: {pred['interaction_score']}\")\n", + "print(f\"\\nDruggable Targets:\")\n", + "for target in druggable_targets[:5]:\n", + " print(f\" - {target['target']}: {target['compound_count']} compounds\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ontology_generator = OntologyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Generate drug discovery ontology\n", + "drug_ontology = ontology_generator.generate_ontology(\n", + " knowledge_graph=knowledge_graph,\n", + " domain=\"DrugDiscovery\"\n", + ")\n", + "\n", + "# Validate ontology\n", + "validation_result = ontology_validator.validate_ontology(drug_ontology)\n", + "\n", + "# Export knowledge graph\n", + "kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"drug_target_kg.json\"))\n", + "kg_rdf = rdf_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"drug_target_kg.rdf\"))\n", + "\n", + "# Export ontology\n", + "ontology_owl = owl_exporter.export(drug_ontology, output_path=os.path.join(temp_dir, \"drug_ontology.owl\"))\n", + "\n", + "# Generate report\n", + "report_content = f\"\"\"\n", + "# Drug Discovery Pipeline Report\n", + "\n", + "## Executive Summary\n", + "- Total Compounds Analyzed: {len(compound_entities)}\n", + "- Total Targets: {len(target_entities)}\n", + "- Total Interactions: {len(interaction_entities)}\n", + "- Druggable Targets Identified: {len(druggable_targets)}\n", + "- High-Affinity Interactions: {len([r for r in relationships if r.get('binding_affinity', 100) < 10])}\n", + "\n", + "## Top Druggable Targets\n", + "\"\"\"\n", + "for i, target in enumerate(druggable_targets[:10], 1):\n", + " report_content += f\"\"\"\n", + "{i}. {target['target']}\n", + " - Compound Count: {target['compound_count']}\n", + " - Description: {target['description']}\n", + "\"\"\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Predicted Interactions\n", + "\"\"\"\n", + "for pred in sorted(predicted_interactions, key=lambda x: x[\"interaction_score\"], reverse=True)[:10]:\n", + " report_content += f\"\"\"\n", + "### {pred['compound']}\n", + "- Interaction Count: {pred['interaction_count']}\n", + "- Interaction Score: {pred['interaction_score']}\n", + "- Targets: {', '.join(pred['targets'][:5])}\n", + "\"\"\"\n", + "\n", + "report_path = os.path.join(temp_dir, \"drug_discovery_report.md\")\n", + "with open(report_path, 'w') as f:\n", + " f.write(report_content)\n", + "\n", + "print(f\"Generated drug discovery ontology with {len(drug_ontology.classes)} classes\")\n", + "print(f\"Ontology validation: {'Valid' if validation_result.valid else 'Invalid'}\")\n", + "print(f\" Errors: {len(validation_result.errors)}\")\n", + "print(f\" Warnings: {len(validation_result.warnings)}\")\n", + "print(f\"Exported knowledge graph to JSON and RDF\")\n", + "print(f\"Exported ontology to OWL\")\n", + "print(f\"Generated discovery report: {report_path}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Visualize Drug-Target Network\n", + "\n", + "Visualize the drug-target knowledge graph, ontology, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "kg_viz = kg_visualizer.visualize(\n", + " knowledge_graph,\n", + " layout=\"force_directed\",\n", + " highlight_nodes=[t[\"id\"] for t in druggable_targets[:5]],\n", + " node_size_by=\"centrality\"\n", + ")\n", + "\n", + "# Visualize ontology\n", + "ontology_viz = ontology_visualizer.visualize(\n", + " drug_ontology,\n", + " layout=\"hierarchical\"\n", + ")\n", + "\n", + "# Visualize analytics\n", + "analytics_viz = analytics_visualizer.visualize(\n", + " knowledge_graph,\n", + " metrics={\n", + " \"centrality\": dict(top_central_targets[:10]),\n", + " \"communities\": communities,\n", + " \"connectivity\": connectivity_results,\n", + " \"interaction_scores\": {p[\"compound_id\"]: p[\"interaction_score\"] for p in predicted_interactions}\n", + " }\n", + ")\n", + "\n", + "print(\"Generated visualizations:\")\n", + "print(\" - Knowledge Graph: Drug-target network with highlighted druggable targets\")\n", + "print(\" - Ontology Visualization: Drug discovery ontology hierarchy\")\n", + "print(\" - Analytics Visualization: Centrality, communities, connectivity, and interaction scores\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb b/docs/cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb new file mode 100644 index 00000000..41c52085 --- /dev/null +++ b/docs/cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb @@ -0,0 +1,775 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Genomic Variant Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete genomic variant analysis pipeline: ingest genomic data from multiple sources (APIs, databases, feeds), extract variant entities, build genomic knowledge graph, analyze disease associations, predict variant impact, and perform pathway analysis.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, DBIngestor, FeedIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Genomic Data Sources → Parse → Extract Entities (variants, genes, diseases, pathways) → Build Genomic KG → Analyze Associations → Predict Impact → Pathway Analysis → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Genomic Data from Multiple Sources\n", + "\n", + "Ingest genomic variant data from APIs, databases, and research feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, DBIngestor, FeedIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.kg import ConflictDetector\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "# Real genomic APIs\n", + "genomic_apis = [\n", + " \"https://rest.ensembl.org/variation/human/rs699\", # Ensembl API\n", + " \"https://api.ncbi.nlm.nih.gov/variation/v0/variant/NC_000001.10:g.230710048A%3EG\", # NCBI Variation API\n", + " \"https://api.ncbi.nlm.nih.gov/variation/v0/beta/refsnp/699\" # ClinVar API\n", + "]\n", + "\n", + "# Real genomic databases\n", + "genomic_databases = [\n", + " \"dbSNP\",\n", + " \"ClinVar\",\n", + " \"COSMIC\"\n", + "]\n", + "\n", + "# Real research feeds\n", + "genomic_feeds = [\n", + " \"https://pubmed.ncbi.nlm.nih.gov/rss/search?term=genomic+variants\",\n", + " \"https://pubmed.ncbi.nlm.nih.gov/rss/search?term=genetic+variation\"\n", + "]\n", + "\n", + "# Real database connection for variant annotations\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/genomic_db\"\n", + "db_query = \"SELECT variant_id, gene_symbol, disease_name, clinical_significance, chromosome, position FROM variants WHERE clinical_significance IN ('Pathogenic', 'Likely Pathogenic') LIMIT 1000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample genomic variant data for local ingestion\n", + "genomic_data_file = os.path.join(temp_dir, \"genomic_variants.json\")\n", + "genomic_data = [\n", + " {\n", + " \"variant_id\": \"rs699\",\n", + " \"gene_symbol\": \"AGT\",\n", + " \"gene_name\": \"Angiotensinogen\",\n", + " \"disease_name\": \"Hypertension\",\n", + " \"clinical_significance\": \"Pathogenic\",\n", + " \"chromosome\": \"1\",\n", + " \"position\": 230710048,\n", + " \"ref_allele\": \"A\",\n", + " \"alt_allele\": \"G\",\n", + " \"pathway\": \"Renin-angiotensin system\",\n", + " \"impact\": \"High\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat()\n", + " },\n", + " {\n", + " \"variant_id\": \"rs7412\",\n", + " \"gene_symbol\": \"APOE\",\n", + " \"gene_name\": \"Apolipoprotein E\",\n", + " \"disease_name\": \"Alzheimer's Disease\",\n", + " \"clinical_significance\": \"Pathogenic\",\n", + " \"chromosome\": \"19\",\n", + " \"position\": 44908822,\n", + " \"ref_allele\": \"C\",\n", + " \"alt_allele\": \"T\",\n", + " \"pathway\": \"Lipid metabolism\",\n", + " \"impact\": \"High\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=2)).isoformat()\n", + " },\n", + " {\n", + " \"variant_id\": \"rs1800566\",\n", + " \"gene_symbol\": \"NAT2\",\n", + " \"gene_name\": \"N-acetyltransferase 2\",\n", + " \"disease_name\": \"Drug Metabolism\",\n", + " \"clinical_significance\": \"Likely Pathogenic\",\n", + " \"chromosome\": \"8\",\n", + " \"position\": 18248728,\n", + " \"ref_allele\": \"G\",\n", + " \"alt_allele\": \"A\",\n", + " \"pathway\": \"Drug metabolism\",\n", + " \"impact\": \"Moderate\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=3)).isoformat()\n", + " },\n", + " {\n", + " \"variant_id\": \"rs1799853\",\n", + " \"gene_symbol\": \"CYP2C9\",\n", + " \"gene_name\": \"Cytochrome P450 2C9\",\n", + " \"disease_name\": \"Warfarin Sensitivity\",\n", + " \"clinical_significance\": \"Pathogenic\",\n", + " \"chromosome\": \"10\",\n", + " \"position\": 96741054,\n", + " \"ref_allele\": \"C\",\n", + " \"alt_allele\": \"T\",\n", + " \"pathway\": \"Drug metabolism\",\n", + " \"impact\": \"High\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=4)).isoformat()\n", + " },\n", + " {\n", + " \"variant_id\": \"rs1057910\",\n", + " \"gene_symbol\": \"CYP2C9\",\n", + " \"gene_name\": \"Cytochrome P450 2C9\",\n", + " \"disease_name\": \"Warfarin Sensitivity\",\n", + " \"clinical_significance\": \"Pathogenic\",\n", + " \"chromosome\": \"10\",\n", + " \"position\": 96741055,\n", + " \"ref_allele\": \"A\",\n", + " \"alt_allele\": \"C\",\n", + " \"pathway\": \"Drug metabolism\",\n", + " \"impact\": \"High\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=5)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(genomic_data_file, 'w') as f:\n", + " json.dump(genomic_data, f, indent=2)\n", + "\n", + "# Ingest from local file\n", + "file_data = file_ingestor.ingest_file(genomic_data_file)\n", + "parsed_genomic = structured_parser.parse_json(json.dumps(genomic_data))\n", + "\n", + "# Ingest from genomic APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(genomic_apis[0]) # Ensembl API\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Ingest from genomic feeds\n", + "feed_data_list = []\n", + "for feed_url in genomic_feeds:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " feed_data_list.append(feed_data)\n", + " print(f\"✓ Ingested feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion failed for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"variants\",\n", + " limit=1000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " db_data = {\"data\": genomic_data}\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Local variants: {len(genomic_data)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Feeds ingested: {len(feed_data_list)}\")\n", + "print(f\" Web APIs: {len(genomic_apis)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Genomic Entities\n", + "\n", + "Extract variants, genes, diseases, and pathways from the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "all_genomic_texts = []\n", + "all_variants = []\n", + "\n", + "# Process parsed genomic data\n", + "if parsed_genomic and isinstance(parsed_genomic, dict):\n", + " variants = parsed_genomic.get(\"data\", genomic_data)\n", + " for variant in variants:\n", + " all_variants.append(variant)\n", + " variant_text = f\"Variant {variant.get('variant_id', '')} in gene {variant.get('gene_symbol', '')} associated with {variant.get('disease_name', '')} in pathway {variant.get('pathway', '')}\"\n", + " all_genomic_texts.append(variant_text)\n", + "\n", + "# Extract entities\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_triples = []\n", + "\n", + "for text in all_genomic_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " triples = triple_extractor.extract(text)\n", + " all_triples.extend(triples)\n", + "\n", + "# Build structured entity list\n", + "variant_entities = []\n", + "gene_entities = []\n", + "disease_entities = []\n", + "pathway_entities = []\n", + "\n", + "unique_genes = {}\n", + "unique_diseases = {}\n", + "unique_pathways = {}\n", + "\n", + "for variant in all_variants:\n", + " variant_entity = {\n", + " \"id\": variant.get(\"variant_id\", \"\"),\n", + " \"type\": \"Variant\",\n", + " \"properties\": {\n", + " \"variant_id\": variant.get(\"variant_id\", \"\"),\n", + " \"chromosome\": variant.get(\"chromosome\", \"\"),\n", + " \"position\": variant.get(\"position\", 0),\n", + " \"ref_allele\": variant.get(\"ref_allele\", \"\"),\n", + " \"alt_allele\": variant.get(\"alt_allele\", \"\"),\n", + " \"clinical_significance\": variant.get(\"clinical_significance\", \"\"),\n", + " \"impact\": variant.get(\"impact\", \"\"),\n", + " \"timestamp\": variant.get(\"timestamp\", \"\")\n", + " }\n", + " }\n", + " variant_entities.append(variant_entity)\n", + " \n", + " # Add gene entity\n", + " gene_symbol = variant.get(\"gene_symbol\", \"\")\n", + " if gene_symbol and gene_symbol not in unique_genes:\n", + " gene_entity = {\n", + " \"id\": gene_symbol,\n", + " \"type\": \"Gene\",\n", + " \"properties\": {\n", + " \"symbol\": gene_symbol,\n", + " \"name\": variant.get(\"gene_name\", \"\"),\n", + " \"chromosome\": variant.get(\"chromosome\", \"\")\n", + " }\n", + " }\n", + " gene_entities.append(gene_entity)\n", + " unique_genes[gene_symbol] = gene_entity\n", + " \n", + " # Add disease entity\n", + " disease_name = variant.get(\"disease_name\", \"\")\n", + " if disease_name and disease_name not in unique_diseases:\n", + " disease_entity = {\n", + " \"id\": disease_name.replace(\" \", \"_\"),\n", + " \"type\": \"Disease\",\n", + " \"properties\": {\n", + " \"name\": disease_name\n", + " }\n", + " }\n", + " disease_entities.append(disease_entity)\n", + " unique_diseases[disease_name] = disease_entity\n", + " \n", + " # Add pathway entity\n", + " pathway_name = variant.get(\"pathway\", \"\")\n", + " if pathway_name and pathway_name not in unique_pathways:\n", + " pathway_entity = {\n", + " \"id\": pathway_name.replace(\" \", \"_\"),\n", + " \"type\": \"Pathway\",\n", + " \"properties\": {\n", + " \"name\": pathway_name\n", + " }\n", + " }\n", + " pathway_entities.append(pathway_entity)\n", + " unique_pathways[pathway_name] = pathway_entity\n", + "\n", + "print(f\"Extracted {len(variant_entities)} variants\")\n", + "print(f\"Extracted {len(gene_entities)} unique genes\")\n", + "print(f\"Extracted {len(disease_entities)} unique diseases\")\n", + "print(f\"Extracted {len(pathway_entities)} unique pathways\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Extracted {len(all_triples)} triples\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Genomic Knowledge Graph\n", + "\n", + "Build a knowledge graph from extracted genomic entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "# Add all entities\n", + "for variant in variant_entities:\n", + " builder.add_entity(\n", + " entity_id=variant[\"id\"],\n", + " entity_type=variant[\"type\"],\n", + " properties=variant.get(\"properties\", {})\n", + " )\n", + "\n", + "for gene in gene_entities:\n", + " builder.add_entity(\n", + " entity_id=gene[\"id\"],\n", + " entity_type=gene[\"type\"],\n", + " properties=gene.get(\"properties\", {})\n", + " )\n", + "\n", + "for disease in disease_entities:\n", + " builder.add_entity(\n", + " entity_id=disease[\"id\"],\n", + " entity_type=disease[\"type\"],\n", + " properties=disease.get(\"properties\", {})\n", + " )\n", + "\n", + "for pathway in pathway_entities:\n", + " builder.add_entity(\n", + " entity_id=pathway[\"id\"],\n", + " entity_type=pathway[\"type\"],\n", + " properties=pathway.get(\"properties\", {})\n", + " )\n", + "\n", + "# Add relationships\n", + "relationships = []\n", + "for variant in variant_entities:\n", + " variant_id = variant[\"id\"]\n", + " gene_symbol = variant[\"properties\"].get(\"gene_symbol\", \"\")\n", + " disease_name = variant[\"properties\"].get(\"disease_name\", \"\").replace(\" \", \"_\")\n", + " pathway_name = variant[\"properties\"].get(\"pathway\", \"\").replace(\" \", \"_\")\n", + " \n", + " # Find corresponding entities\n", + " gene_entity = unique_genes.get(gene_symbol)\n", + " disease_entity = unique_diseases.get(disease_name.replace(\"_\", \" \"))\n", + " pathway_entity = unique_pathways.get(pathway_name.replace(\"_\", \" \"))\n", + " \n", + " if gene_entity:\n", + " builder.add_relationship(\n", + " source_id=variant_id,\n", + " target_id=gene_entity[\"id\"],\n", + " relationship_type=\"located_in\",\n", + " properties={}\n", + " )\n", + " relationships.append({\n", + " \"source\": variant_id,\n", + " \"target\": gene_entity[\"id\"],\n", + " \"type\": \"located_in\"\n", + " })\n", + " \n", + " if disease_entity:\n", + " builder.add_relationship(\n", + " source_id=variant_id,\n", + " target_id=disease_entity[\"id\"],\n", + " relationship_type=\"associated_with\",\n", + " properties={\n", + " \"clinical_significance\": variant[\"properties\"].get(\"clinical_significance\", \"\")\n", + " }\n", + " )\n", + " relationships.append({\n", + " \"source\": variant_id,\n", + " \"target\": disease_entity[\"id\"],\n", + " \"type\": \"associated_with\"\n", + " })\n", + " \n", + " if pathway_entity:\n", + " builder.add_relationship(\n", + " source_id=gene_entity[\"id\"] if gene_entity else variant_id,\n", + " target_id=pathway_entity[\"id\"],\n", + " relationship_type=\"participates_in\",\n", + " properties={}\n", + " )\n", + " relationships.append({\n", + " \"source\": gene_entity[\"id\"] if gene_entity else variant_id,\n", + " \"target\": pathway_entity[\"id\"],\n", + " \"type\": \"participates_in\"\n", + " })\n", + "\n", + "knowledge_graph = builder.build()\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.nodes)} nodes\")\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.edges)} edges\")\n", + "print(f\"Added {len(relationships)} genomic relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze Associations and Predict Impact\n", + "\n", + "Analyze variant-disease associations and predict variant impact on protein function.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer(knowledge_graph)\n", + "centrality_calculator = CentralityCalculator(knowledge_graph)\n", + "community_detector = CommunityDetector(knowledge_graph)\n", + "connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n", + "temporal_query = TemporalGraphQuery(knowledge_graph)\n", + "pattern_detector = TemporalPatternDetector(knowledge_graph)\n", + "\n", + "# Compute graph metrics\n", + "graph_metrics = graph_analyzer.compute_metrics()\n", + "\n", + "# Calculate centrality\n", + "centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n", + "top_central_genes = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n", + "\n", + "# Detect communities\n", + "communities = community_detector.detect_communities()\n", + "community_count = len(set(communities.values())) if communities else 0\n", + "\n", + "# Analyze connectivity\n", + "connectivity_results = connectivity_analyzer.analyze_connectivity()\n", + "\n", + "# Detect temporal patterns\n", + "start_time = (datetime.now() - timedelta(days=7)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "temporal_results = temporal_query.query_time_range(\n", + " start_time=start_time,\n", + " end_time=end_time,\n", + " relationship_types=[\"associated_with\", \"located_in\"]\n", + ")\n", + "\n", + "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", + " relationship_types=[\"associated_with\"],\n", + " time_window_hours=168\n", + ")\n", + "\n", + "# Impact Prediction using Inference Engine\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Define impact prediction rules\n", + "impact_rules = [\n", + " {\n", + " \"name\": \"high_impact_pathogenic\",\n", + " \"condition\": \"clinical_significance == 'Pathogenic' AND impact == 'High'\",\n", + " \"action\": \"predict_high_impact\"\n", + " },\n", + " {\n", + " \"name\": \"moderate_impact_likely_pathogenic\",\n", + " \"condition\": \"clinical_significance == 'Likely Pathogenic' AND impact == 'Moderate'\",\n", + " \"action\": \"predict_moderate_impact\"\n", + " },\n", + " {\n", + " \"name\": \"disease_association\",\n", + " \"condition\": \"associated_with_disease AND pathogenic\",\n", + " \"action\": \"predict_disease_risk\"\n", + " }\n", + "]\n", + "\n", + "for rule in impact_rules:\n", + " rule_manager.add_rule(rule[\"name\"], rule[\"condition\"], rule[\"action\"])\n", + "\n", + "# Predict variant impact\n", + "variant_impacts = []\n", + "for variant in variant_entities:\n", + " clinical_sig = variant[\"properties\"].get(\"clinical_significance\", \"\")\n", + " impact = variant[\"properties\"].get(\"impact\", \"\")\n", + " \n", + " impact_score = 0\n", + " if clinical_sig == \"Pathogenic\":\n", + " impact_score += 5\n", + " elif clinical_sig == \"Likely Pathogenic\":\n", + " impact_score += 3\n", + " \n", + " if impact == \"High\":\n", + " impact_score += 3\n", + " elif impact == \"Moderate\":\n", + " impact_score += 2\n", + " \n", + " # Check disease associations\n", + " disease_associations = [r for r in relationships if r[\"source\"] == variant[\"id\"] and r[\"type\"] == \"associated_with\"]\n", + " if disease_associations:\n", + " impact_score += 2\n", + " \n", + " variant_impacts.append({\n", + " \"variant\": variant[\"id\"],\n", + " \"impact_score\": min(impact_score, 10),\n", + " \"predicted_impact\": \"High\" if impact_score >= 7 else \"Moderate\" if impact_score >= 4 else \"Low\",\n", + " \"clinical_significance\": clinical_sig,\n", + " \"disease_associations\": len(disease_associations)\n", + " })\n", + "\n", + "print(f\"Analyzed {len(variant_entities)} variants\")\n", + "print(f\"Found {community_count} gene communities\")\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"\\nTop 5 Central Genes:\")\n", + "for i, (gene_id, centrality) in enumerate(top_central_genes[:5], 1):\n", + " print(f\" {i}. {gene_id} (centrality: {centrality:.3f})\")\n", + "print(f\"\\nVariant Impact Predictions:\")\n", + "for impact in sorted(variant_impacts, key=lambda x: x[\"impact_score\"], reverse=True):\n", + " print(f\" - {impact['variant']}: {impact['predicted_impact']} Impact (Score: {impact['impact_score']}/10, Diseases: {impact['disease_associations']})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Genomic Ontology and Pathway Analysis\n", + "\n", + "Generate genomic ontology and perform pathway analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "# Generate genomic ontology\n", + "genomic_ontology = ontology_generator.generate_ontology(\n", + " knowledge_graph=knowledge_graph,\n", + " domain=\"Genomics\"\n", + ")\n", + "\n", + "# Infer classes\n", + "classes = class_inferrer.infer_classes(knowledge_graph)\n", + "for cls in classes:\n", + " genomic_ontology.add_class(cls)\n", + "\n", + "# Generate properties\n", + "properties = property_generator.generate_properties(knowledge_graph)\n", + "for prop in properties:\n", + " genomic_ontology.add_property(prop)\n", + "\n", + "# Validate ontology\n", + "validation_result = ontology_validator.validate_ontology(genomic_ontology)\n", + "\n", + "# Pathway analysis\n", + "pathway_analysis = {}\n", + "for pathway in pathway_entities:\n", + " pathway_id = pathway[\"id\"]\n", + " pathway_name = pathway[\"properties\"].get(\"name\", \"\")\n", + " \n", + " # Find variants and genes in this pathway\n", + " pathway_variants = [r for r in relationships if r[\"target\"] == pathway_id and r[\"type\"] == \"participates_in\"]\n", + " pathway_genes = set()\n", + " for rel in pathway_variants:\n", + " source_entity = next((e for e in variant_entities + gene_entities if e[\"id\"] == rel[\"source\"]), None)\n", + " if source_entity and source_entity[\"type\"] == \"Gene\":\n", + " pathway_genes.add(source_entity[\"id\"])\n", + " \n", + " pathway_analysis[pathway_name] = {\n", + " \"variants\": len([r for r in pathway_variants if next((e for e in variant_entities if e[\"id\"] == r[\"source\"]), None)]),\n", + " \"genes\": len(pathway_genes),\n", + " \"diseases\": len([r for r in relationships if r[\"source\"] in [e[\"id\"] for e in variant_entities] and r[\"type\"] == \"associated_with\"])\n", + " }\n", + "\n", + "print(f\"Generated genomic ontology with {len(genomic_ontology.classes)} classes\")\n", + "print(f\"Ontology validation: {'Valid' if validation_result.valid else 'Invalid'}\")\n", + "print(f\" Errors: {len(validation_result.errors)}\")\n", + "print(f\" Warnings: {len(validation_result.warnings)}\")\n", + "print(f\"\\nPathway Analysis:\")\n", + "for pathway_name, analysis in pathway_analysis.items():\n", + " print(f\" - {pathway_name}: {analysis['variants']} variants, {analysis['genes']} genes, {analysis['diseases']} disease associations\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Generate Reports and Visualize\n", + "\n", + "Generate comprehensive genomic analysis reports and visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "kg_quality_assessor = KGQualityAssessor()\n", + "conflict_detector = ConflictDetector(knowledge_graph)\n", + "\n", + "# Assess graph quality\n", + "quality_metrics = kg_quality_assessor.assess_quality(knowledge_graph)\n", + "\n", + "# Detect conflicts\n", + "conflicts = conflict_detector.detect_conflicts()\n", + "\n", + "# Export knowledge graph\n", + "kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"genomic_kg.json\"))\n", + "kg_rdf = rdf_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"genomic_kg.rdf\"))\n", + "\n", + "# Export ontology\n", + "ontology_owl = owl_exporter.export(genomic_ontology, output_path=os.path.join(temp_dir, \"genomic_ontology.owl\"))\n", + "\n", + "# Generate report\n", + "report_content = f\"\"\"\n", + "# Genomic Variant Analysis Report\n", + "\n", + "## Executive Summary\n", + "- Total Variants Analyzed: {len(variant_entities)}\n", + "- Unique Genes: {len(gene_entities)}\n", + "- Unique Diseases: {len(disease_entities)}\n", + "- Unique Pathways: {len(pathway_entities)}\n", + "- High Impact Variants: {len([v for v in variant_impacts if v['predicted_impact'] == 'High'])}\n", + "\n", + "## Graph Quality Metrics\n", + "- Nodes: {quality_metrics.get('node_count', len(knowledge_graph.nodes))}\n", + "- Edges: {quality_metrics.get('edge_count', len(knowledge_graph.edges))}\n", + "- Completeness: {quality_metrics.get('completeness', 0):.2%}\n", + "- Consistency: {quality_metrics.get('consistency', 0):.2%}\n", + "\n", + "## Top Variants by Impact\n", + "\"\"\"\n", + "for i, impact in enumerate(sorted(variant_impacts, key=lambda x: x[\"impact_score\"], reverse=True)[:10], 1):\n", + " report_content += f\"\"\"\n", + "{i}. {impact['variant']}\n", + " - Predicted Impact: {impact['predicted_impact']}\n", + " - Impact Score: {impact['impact_score']}/10\n", + " - Clinical Significance: {impact['clinical_significance']}\n", + " - Disease Associations: {impact['disease_associations']}\n", + "\"\"\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Pathway Analysis\n", + "\"\"\"\n", + "for pathway_name, analysis in pathway_analysis.items():\n", + " report_content += f\"\"\"\n", + "### {pathway_name}\n", + "- Variants: {analysis['variants']}\n", + "- Genes: {analysis['genes']}\n", + "- Disease Associations: {analysis['diseases']}\n", + "\"\"\"\n", + "\n", + "report_path = os.path.join(temp_dir, \"genomic_analysis_report.md\")\n", + "with open(report_path, 'w') as f:\n", + " f.write(report_content)\n", + "\n", + "print(f\"Exported knowledge graph to JSON and RDF\")\n", + "print(f\"Exported ontology to OWL\")\n", + "print(f\"Generated analysis report: {report_path}\")\n", + "print(f\"\\nQuality Metrics:\")\n", + "print(f\" Nodes: {quality_metrics.get('node_count', len(knowledge_graph.nodes))}\")\n", + "print(f\" Edges: {quality_metrics.get('edge_count', len(knowledge_graph.edges))}\")\n", + "print(f\" Completeness: {quality_metrics.get('completeness', 0):.2%}\")\n", + "print(f\" Consistency: {quality_metrics.get('consistency', 0):.2%}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Visualize Genomic Network\n", + "\n", + "Visualize the genomic knowledge graph, ontology, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "kg_viz = kg_visualizer.visualize(\n", + " knowledge_graph,\n", + " layout=\"force_directed\",\n", + " highlight_nodes=[v[\"id\"] for v in variant_entities],\n", + " node_size_by=\"impact\"\n", + ")\n", + "\n", + "# Visualize ontology\n", + "ontology_viz = ontology_visualizer.visualize(\n", + " genomic_ontology,\n", + " layout=\"hierarchical\"\n", + ")\n", + "\n", + "# Visualize analytics\n", + "analytics_viz = analytics_visualizer.visualize(\n", + " knowledge_graph,\n", + " metrics={\n", + " \"centrality\": dict(top_central_genes[:10]),\n", + " \"communities\": communities,\n", + " \"connectivity\": connectivity_results,\n", + " \"impact_scores\": {v[\"variant\"]: v[\"impact_score\"] for v in variant_impacts}\n", + " }\n", + ")\n", + "\n", + "print(\"Generated visualizations:\")\n", + "print(\" - Knowledge Graph: Genomic variant network with impact-based sizing\")\n", + "print(\" - Ontology Visualization: Genomic ontology hierarchy\")\n", + "print(\" - Analytics Visualization: Centrality, communities, connectivity, and impact scores\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb b/docs/cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb new file mode 100644 index 00000000..eb1e5bbf --- /dev/null +++ b/docs/cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb @@ -0,0 +1,703 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# DeFi Protocol Intelligence Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete DeFi protocol intelligence pipeline: ingest DeFi data from multiple sources (APIs, feeds, databases), extract protocol entities, build DeFi knowledge graph, analyze relationships, assess risks, optimize yields, and generate reports.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, FeedIngestor, DBIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, HTMLParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, SemanticAnalyzer, EventDetector\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**DeFi Data Sources → Parse → Extract Entities (protocols, pools, tokens, strategies) → Build DeFi KG → Analyze Relationships → Risk Assessment → Yield Optimization → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest DeFi Data from Multiple Sources\n", + "\n", + "Ingest DeFi protocol data from APIs, feeds, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, FeedIngestor, DBIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, HTMLParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, SemanticAnalyzer, EventDetector\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "db_ingestor = DBIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "html_parser = HTMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real DeFi APIs\n", + "defi_apis = [\n", + " \"https://api.llama.fi/protocols\", # DeFiLlama API\n", + " \"https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2\", # The Graph - Uniswap\n", + " \"https://api.github.com/repos/Uniswap/interface\" # Uniswap GitHub\n", + "]\n", + "\n", + "# Real DeFi protocol feeds\n", + "defi_feeds = [\n", + " \"https://defipulse.com/blog/feed\", # DeFi Pulse\n", + " \"https://feeds.feedburner.com/TheDefiant\" # The Defiant\n", + "]\n", + "\n", + "# Real database connection for protocol metrics\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/defi_db\"\n", + "db_query = \"SELECT protocol_name, tvl, apy, token_address, pool_address, timestamp FROM defi_protocols WHERE timestamp > NOW() - INTERVAL '7 days' ORDER BY tvl DESC LIMIT 1000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample DeFi protocol data for local ingestion\n", + "defi_data_file = os.path.join(temp_dir, \"defi_protocols.json\")\n", + "defi_data = [\n", + " {\n", + " \"protocol_name\": \"Uniswap V3\",\n", + " \"protocol_type\": \"DEX\",\n", + " \"tvl\": 2500000000,\n", + " \"apy\": 12.5,\n", + " \"token_address\": \"0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984\",\n", + " \"pool_address\": \"0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8\",\n", + " \"token_symbol\": \"UNI\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat()\n", + " },\n", + " {\n", + " \"protocol_name\": \"Aave V3\",\n", + " \"protocol_type\": \"Lending\",\n", + " \"tvl\": 1800000000,\n", + " \"apy\": 8.3,\n", + " \"token_address\": \"0x7Fc66500c84A76Ad7e9c93437bFc5Ac33E2DDaE9\",\n", + " \"pool_address\": \"0x87870Bca3F3fD6335C3F4ce8392D69350B4fA4E2\",\n", + " \"token_symbol\": \"AAVE\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=2)).isoformat()\n", + " },\n", + " {\n", + " \"protocol_name\": \"Compound V3\",\n", + " \"protocol_type\": \"Lending\",\n", + " \"tvl\": 1200000000,\n", + " \"apy\": 7.8,\n", + " \"token_address\": \"0xc00e94Cb662C3520282E6f5717214004A7f26888\",\n", + " \"pool_address\": \"0xc3d688B667034EAD2F183C05b6e4B5e5B5b5b5b5\",\n", + " \"token_symbol\": \"COMP\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=3)).isoformat()\n", + " },\n", + " {\n", + " \"protocol_name\": \"Curve Finance\",\n", + " \"protocol_type\": \"DEX\",\n", + " \"tvl\": 1500000000,\n", + " \"apy\": 15.2,\n", + " \"token_address\": \"0xD533a949740bb3306d119CC777fa900bA034cd52\",\n", + " \"pool_address\": \"0xbEbc44782C7dB0a1A60Cb6fe97d0b483032FF1C7\",\n", + " \"token_symbol\": \"CRV\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=4)).isoformat()\n", + " },\n", + " {\n", + " \"protocol_name\": \"MakerDAO\",\n", + " \"protocol_type\": \"Lending\",\n", + " \"tvl\": 8000000000,\n", + " \"apy\": 3.5,\n", + " \"token_address\": \"0x9f8F72aA9304c8B593d555F12eF6589cC3A579A2\",\n", + " \"pool_address\": \"0x35D1b3F3D7966A1DFe207aa4514C12a2594E9c99\",\n", + " \"token_symbol\": \"MKR\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=5)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(defi_data_file, 'w') as f:\n", + " json.dump(defi_data, f, indent=2)\n", + "\n", + "# Ingest from local file\n", + "file_data = file_ingestor.ingest_file(defi_data_file)\n", + "parsed_defi = structured_parser.parse_json(json.dumps(defi_data))\n", + "\n", + "# Ingest from DeFi APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(defi_apis[2]) # GitHub API\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Ingest from DeFi feeds\n", + "feed_data_list = []\n", + "for feed_url in defi_feeds:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " feed_data_list.append(feed_data)\n", + " print(f\"✓ Ingested feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion failed for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"defi_protocols\",\n", + " limit=1000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " db_data = {\"data\": defi_data}\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Local protocols: {len(defi_data)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Feeds ingested: {len(feed_data_list)}\")\n", + "print(f\" Web APIs: {len(defi_apis)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract DeFi Entities\n", + "\n", + "Extract protocols, pools, tokens, and strategies from the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "event_detector = EventDetector()\n", + "\n", + "all_defi_texts = []\n", + "all_protocols = []\n", + "\n", + "# Process parsed DeFi data\n", + "if parsed_defi and isinstance(parsed_defi, dict):\n", + " protocols = parsed_defi.get(\"data\", defi_data)\n", + " for protocol in protocols:\n", + " all_protocols.append(protocol)\n", + " protocol_text = f\"Protocol {protocol.get('protocol_name', '')} type {protocol.get('protocol_type', '')} TVL {protocol.get('tvl', 0)} APY {protocol.get('apy', 0)} token {protocol.get('token_symbol', '')}\"\n", + " all_defi_texts.append(protocol_text)\n", + "\n", + "# Extract entities\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_events = []\n", + "\n", + "for text in all_defi_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " events = event_detector.detect_events(text)\n", + " all_events.extend(events)\n", + "\n", + "# Build structured entity list\n", + "protocol_entities = []\n", + "pool_entities = []\n", + "token_entities = []\n", + "\n", + "for protocol in all_protocols:\n", + " protocol_entity = {\n", + " \"id\": protocol.get(\"protocol_name\", \"\").replace(\" \", \"_\"),\n", + " \"type\": \"Protocol\",\n", + " \"properties\": {\n", + " \"name\": protocol.get(\"protocol_name\", \"\"),\n", + " \"type\": protocol.get(\"protocol_type\", \"\"),\n", + " \"tvl\": protocol.get(\"tvl\", 0),\n", + " \"apy\": protocol.get(\"apy\", 0),\n", + " \"chain\": protocol.get(\"chain\", \"\"),\n", + " \"timestamp\": protocol.get(\"timestamp\", \"\")\n", + " }\n", + " }\n", + " protocol_entities.append(protocol_entity)\n", + " \n", + " # Add pool entity\n", + " pool_entity = {\n", + " \"id\": protocol.get(\"pool_address\", \"\"),\n", + " \"type\": \"Pool\",\n", + " \"properties\": {\n", + " \"address\": protocol.get(\"pool_address\", \"\"),\n", + " \"protocol\": protocol.get(\"protocol_name\", \"\"),\n", + " \"tvl\": protocol.get(\"tvl\", 0),\n", + " \"apy\": protocol.get(\"apy\", 0)\n", + " }\n", + " }\n", + " pool_entities.append(pool_entity)\n", + " \n", + " # Add token entity\n", + " token_entity = {\n", + " \"id\": protocol.get(\"token_address\", \"\"),\n", + " \"type\": \"Token\",\n", + " \"properties\": {\n", + " \"address\": protocol.get(\"token_address\", \"\"),\n", + " \"symbol\": protocol.get(\"token_symbol\", \"\"),\n", + " \"protocol\": protocol.get(\"protocol_name\", \"\")\n", + " }\n", + " }\n", + " token_entities.append(token_entity)\n", + "\n", + "print(f\"Extracted {len(protocol_entities)} protocols\")\n", + "print(f\"Extracted {len(pool_entities)} pools\")\n", + "print(f\"Extracted {len(token_entities)} tokens\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Detected {len(all_events)} events\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build DeFi Knowledge Graph\n", + "\n", + "Build a knowledge graph from extracted DeFi entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "# Add all entities\n", + "for protocol in protocol_entities:\n", + " builder.add_entity(\n", + " entity_id=protocol[\"id\"],\n", + " entity_type=protocol[\"type\"],\n", + " properties=protocol.get(\"properties\", {})\n", + " )\n", + "\n", + "for pool in pool_entities:\n", + " builder.add_entity(\n", + " entity_id=pool[\"id\"],\n", + " entity_type=pool[\"type\"],\n", + " properties=pool.get(\"properties\", {})\n", + " )\n", + "\n", + "for token in token_entities:\n", + " builder.add_entity(\n", + " entity_id=token[\"id\"],\n", + " entity_type=token[\"type\"],\n", + " properties=token.get(\"properties\", {})\n", + " )\n", + "\n", + "# Add relationships\n", + "relationships = []\n", + "for i, protocol in enumerate(protocol_entities):\n", + " protocol_id = protocol[\"id\"]\n", + " pool_id = pool_entities[i][\"id\"]\n", + " token_id = token_entities[i][\"id\"]\n", + " \n", + " # Protocol-Pool relationship\n", + " builder.add_relationship(\n", + " source_id=protocol_id,\n", + " target_id=pool_id,\n", + " relationship_type=\"has_pool\",\n", + " properties={}\n", + " )\n", + " \n", + " # Protocol-Token relationship\n", + " builder.add_relationship(\n", + " source_id=protocol_id,\n", + " target_id=token_id,\n", + " relationship_type=\"has_token\",\n", + " properties={}\n", + " )\n", + " \n", + " # Pool-Token relationship\n", + " builder.add_relationship(\n", + " source_id=pool_id,\n", + " target_id=token_id,\n", + " relationship_type=\"contains\",\n", + " properties={}\n", + " )\n", + " \n", + " relationships.append({\n", + " \"source\": protocol_id,\n", + " \"target\": pool_id,\n", + " \"type\": \"has_pool\"\n", + " })\n", + " relationships.append({\n", + " \"source\": protocol_id,\n", + " \"target\": token_id,\n", + " \"type\": \"has_token\"\n", + " })\n", + "\n", + "knowledge_graph = builder.build()\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.nodes)} nodes\")\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.edges)} edges\")\n", + "print(f\"Added {len(relationships)} DeFi relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze DeFi Relationships and Assess Risks\n", + "\n", + "Analyze protocol relationships, detect communities, and assess risks.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer(knowledge_graph)\n", + "centrality_calculator = CentralityCalculator(knowledge_graph)\n", + "community_detector = CommunityDetector(knowledge_graph)\n", + "connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n", + "temporal_query = TemporalGraphQuery(knowledge_graph)\n", + "pattern_detector = TemporalPatternDetector(knowledge_graph)\n", + "\n", + "# Compute graph metrics\n", + "graph_metrics = graph_analyzer.compute_metrics()\n", + "\n", + "# Calculate centrality\n", + "centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n", + "top_central_protocols = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n", + "\n", + "# Detect communities\n", + "communities = community_detector.detect_communities()\n", + "community_count = len(set(communities.values())) if communities else 0\n", + "\n", + "# Analyze connectivity\n", + "connectivity_results = connectivity_analyzer.analyze_connectivity()\n", + "\n", + "# Detect temporal patterns\n", + "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", + " relationship_types=[\"has_pool\", \"has_token\"],\n", + " time_window_hours=24\n", + ")\n", + "\n", + "# Risk Assessment using Inference Engine\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Define risk rules\n", + "risk_rules = [\n", + " {\n", + " \"name\": \"high_tvl_risk\",\n", + " \"condition\": \"tvl > 5000000000 AND apy < 5\",\n", + " \"action\": \"flag_as_low_yield_high_tvl\"\n", + " },\n", + " {\n", + " \"name\": \"high_apy_risk\",\n", + " \"condition\": \"apy > 20\",\n", + " \"action\": \"flag_as_high_risk_high_yield\"\n", + " },\n", + " {\n", + " \"name\": \"optimal_protocol\",\n", + " \"condition\": \"tvl > 1000000000 AND apy BETWEEN 8 AND 15\",\n", + " \"action\": \"flag_as_optimal\"\n", + " }\n", + "]\n", + "\n", + "for rule in risk_rules:\n", + " rule_manager.add_rule(rule[\"name\"], rule[\"condition\"], rule[\"action\"])\n", + "\n", + "# Assess protocol risks\n", + "protocol_risks = []\n", + "for protocol in protocol_entities:\n", + " tvl = protocol[\"properties\"].get(\"tvl\", 0)\n", + " apy = protocol[\"properties\"].get(\"apy\", 0)\n", + " \n", + " risk_score = 0\n", + " risk_factors = []\n", + " \n", + " if tvl > 5000000000 and apy < 5:\n", + " risk_score += 2\n", + " risk_factors.append(\"low_yield_high_tvl\")\n", + " \n", + " if apy > 20:\n", + " risk_score += 3\n", + " risk_factors.append(\"high_apy_risk\")\n", + " \n", + " if tvl < 500000000:\n", + " risk_score += 1\n", + " risk_factors.append(\"low_tvl\")\n", + " \n", + " if 1000000000 <= tvl <= 5000000000 and 8 <= apy <= 15:\n", + " risk_score = max(0, risk_score - 1)\n", + " risk_factors.append(\"optimal_range\")\n", + " \n", + " protocol_risks.append({\n", + " \"protocol\": protocol[\"id\"],\n", + " \"name\": protocol[\"properties\"].get(\"name\", \"\"),\n", + " \"risk_score\": min(risk_score, 10),\n", + " \"risk_factors\": risk_factors,\n", + " \"tvl\": tvl,\n", + " \"apy\": apy\n", + " })\n", + "\n", + "print(f\"Analyzed {len(protocol_entities)} protocols\")\n", + "print(f\"Found {community_count} protocol communities\")\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"\\nTop 5 Central Protocols:\")\n", + "for i, (protocol_id, centrality) in enumerate(top_central_protocols[:5], 1):\n", + " protocol_name = next((p[\"properties\"].get(\"name\", protocol_id) for p in protocol_entities if p[\"id\"] == protocol_id), protocol_id)\n", + " print(f\" {i}. {protocol_name} (centrality: {centrality:.3f})\")\n", + "print(f\"\\nProtocol Risk Assessment:\")\n", + "for risk in sorted(protocol_risks, key=lambda x: x[\"risk_score\"], reverse=True)[:5]:\n", + " print(f\" - {risk['name']}: Risk Score {risk['risk_score']}/10, Factors: {', '.join(risk['risk_factors'])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate DeFi Ontology and Optimize Yields\n", + "\n", + "Generate DeFi protocol ontology and optimize yield strategies.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "# Generate DeFi ontology\n", + "defi_ontology = ontology_generator.generate_ontology(\n", + " knowledge_graph=knowledge_graph,\n", + " domain=\"DeFi\"\n", + ")\n", + "\n", + "# Infer classes\n", + "classes = class_inferrer.infer_classes(knowledge_graph)\n", + "for cls in classes:\n", + " defi_ontology.add_class(cls)\n", + "\n", + "# Generate properties\n", + "properties = property_generator.generate_properties(knowledge_graph)\n", + "for prop in properties:\n", + " defi_ontology.add_property(prop)\n", + "\n", + "# Validate ontology\n", + "validation_result = ontology_validator.validate_ontology(defi_ontology)\n", + "\n", + "# Yield optimization\n", + "yield_optimization = []\n", + "for protocol in protocol_entities:\n", + " tvl = protocol[\"properties\"].get(\"tvl\", 0)\n", + " apy = protocol[\"properties\"].get(\"apy\", 0)\n", + " protocol_type = protocol[\"properties\"].get(\"type\", \"\")\n", + " \n", + " # Calculate yield score\n", + " yield_score = (apy * 0.6) + (min(tvl / 1000000000, 10) * 0.4)\n", + " \n", + " optimization_suggestions = []\n", + " if apy < 8 and tvl > 1000000000:\n", + " optimization_suggestions.append(\"Consider higher APY protocols for better yield\")\n", + " if tvl < 500000000:\n", + " optimization_suggestions.append(\"Low TVL may indicate higher risk\")\n", + " if apy > 15:\n", + " optimization_suggestions.append(\"High APY may indicate higher risk, diversify\")\n", + " \n", + " yield_optimization.append({\n", + " \"protocol\": protocol[\"properties\"].get(\"name\", \"\"),\n", + " \"yield_score\": yield_score,\n", + " \"apy\": apy,\n", + " \"tvl\": tvl,\n", + " \"suggestions\": optimization_suggestions\n", + " })\n", + "\n", + "print(f\"Generated DeFi ontology with {len(defi_ontology.classes)} classes\")\n", + "print(f\"Ontology validation: {'Valid' if validation_result.valid else 'Invalid'}\")\n", + "print(f\" Errors: {len(validation_result.errors)}\")\n", + "print(f\" Warnings: {len(validation_result.warnings)}\")\n", + "print(f\"\\nYield Optimization Recommendations:\")\n", + "for opt in sorted(yield_optimization, key=lambda x: x[\"yield_score\"], reverse=True)[:5]:\n", + " print(f\" - {opt['protocol']}: Yield Score {opt['yield_score']:.2f}, APY {opt['apy']:.1f}%\")\n", + " for suggestion in opt['suggestions']:\n", + " print(f\" → {suggestion}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Generate Reports and Visualize\n", + "\n", + "Generate comprehensive DeFi intelligence reports and visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export knowledge graph\n", + "kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"defi_kg.json\"))\n", + "kg_rdf = rdf_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"defi_kg.rdf\"))\n", + "\n", + "# Export ontology\n", + "ontology_owl = owl_exporter.export(defi_ontology, output_path=os.path.join(temp_dir, \"defi_ontology.owl\"))\n", + "\n", + "# Generate report\n", + "report_content = f\"\"\"\n", + "# DeFi Protocol Intelligence Report\n", + "\n", + "## Executive Summary\n", + "- Total Protocols Analyzed: {len(protocol_entities)}\n", + "- Total Pools: {len(pool_entities)}\n", + "- Total Tokens: {len(token_entities)}\n", + "- Protocol Communities: {community_count}\n", + "- High-Risk Protocols: {len([r for r in protocol_risks if r['risk_score'] >= 7])}\n", + "\n", + "## Top Protocols by Centrality\n", + "\"\"\"\n", + "for i, (protocol_id, centrality) in enumerate(top_central_protocols[:10], 1):\n", + " protocol_name = next((p[\"properties\"].get(\"name\", protocol_id) for p in protocol_entities if p[\"id\"] == protocol_id), protocol_id)\n", + " report_content += f\"\\n{i}. {protocol_name} (Centrality: {centrality:.3f})\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Risk Assessment\n", + "\"\"\"\n", + "for risk in sorted(protocol_risks, key=lambda x: x[\"risk_score\"], reverse=True):\n", + " report_content += f\"\"\"\n", + "### {risk['name']}\n", + "- Risk Score: {risk['risk_score']}/10\n", + "- TVL: ${risk['tvl']:,.0f}\n", + "- APY: {risk['apy']:.1f}%\n", + "- Risk Factors: {', '.join(risk['risk_factors'])}\n", + "\"\"\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Yield Optimization\n", + "\"\"\"\n", + "for opt in sorted(yield_optimization, key=lambda x: x[\"yield_score\"], reverse=True)[:10]:\n", + " report_content += f\"\"\"\n", + "### {opt['protocol']}\n", + "- Yield Score: {opt['yield_score']:.2f}\n", + "- APY: {opt['apy']:.1f}%\n", + "- TVL: ${opt['tvl']:,.0f}\n", + "- Suggestions:\n", + "\"\"\"\n", + " for suggestion in opt['suggestions']:\n", + " report_content += f\" - {suggestion}\\n\"\n", + "\n", + "report_path = os.path.join(temp_dir, \"defi_intelligence_report.md\")\n", + "with open(report_path, 'w') as f:\n", + " f.write(report_content)\n", + "\n", + "print(f\"Exported knowledge graph to JSON and RDF\")\n", + "print(f\"Exported ontology to OWL\")\n", + "print(f\"Generated intelligence report: {report_path}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Visualize DeFi Network\n", + "\n", + "Visualize the DeFi protocol network, ontology, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "kg_viz = kg_visualizer.visualize(\n", + " knowledge_graph,\n", + " layout=\"force_directed\",\n", + " highlight_nodes=[p[\"id\"] for p in protocol_entities],\n", + " node_size_by=\"tvl\"\n", + ")\n", + "\n", + "# Visualize ontology\n", + "ontology_viz = ontology_visualizer.visualize(\n", + " defi_ontology,\n", + " layout=\"hierarchical\"\n", + ")\n", + "\n", + "# Visualize analytics\n", + "analytics_viz = analytics_visualizer.visualize(\n", + " knowledge_graph,\n", + " metrics={\n", + " \"centrality\": dict(top_central_protocols[:10]),\n", + " \"communities\": communities,\n", + " \"connectivity\": connectivity_results,\n", + " \"risk_scores\": {r[\"protocol\"]: r[\"risk_score\"] for r in protocol_risks}\n", + " }\n", + ")\n", + "\n", + "print(\"Generated visualizations:\")\n", + "print(\" - Knowledge Graph: DeFi protocol network with TVL-based sizing\")\n", + "print(\" - Ontology Visualization: DeFi protocol ontology hierarchy\")\n", + "print(\" - Analytics Visualization: Centrality, communities, connectivity, and risk scores\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb b/docs/cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb new file mode 100644 index 00000000..6fb8812b --- /dev/null +++ b/docs/cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb @@ -0,0 +1,642 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Transaction Network Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete blockchain transaction network analysis pipeline: stream transactions from multiple sources (blockchain APIs, transaction feeds, databases), build temporal transaction knowledge graph, detect AML patterns, trace fund flows, and generate alerts.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: StreamIngestor, WebIngestor, DBIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Real-time Transaction Streams → Parse → Extract Entities (wallets, transactions, addresses) → Build Temporal Transaction KG → Detect Patterns (tumbling, mixing, clustering) → AML Analysis → Generate Alerts → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Stream Transactions from Multiple Sources\n", + "\n", + "Stream blockchain transactions from APIs, feeds, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import StreamIngestor, WebIngestor, DBIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.kg import ConflictDetector\n", + "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "stream_ingestor = StreamIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real streaming sources for blockchain transactions\n", + "stream_sources = [\n", + " {\n", + " \"type\": \"kafka\",\n", + " \"topic\": \"blockchain_transactions\",\n", + " \"bootstrap_servers\": [\"localhost:9092\"],\n", + " \"consumer_config\": {\"group_id\": \"transaction_analysis\"}\n", + " },\n", + " {\n", + " \"type\": \"rabbitmq\",\n", + " \"queue\": \"eth_transactions\",\n", + " \"connection_url\": \"amqp://user:password@localhost:5672/\"\n", + " }\n", + "]\n", + "\n", + "# Real blockchain APIs\n", + "blockchain_apis = [\n", + " \"https://api.etherscan.io/api?module=proxy&action=eth_getBlockByNumber&tag=latest&boolean=true&apikey=YourApiKeyToken\", # Etherscan API\n", + " \"https://blockchain.info/rawblock/000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f\", # Blockchain.com API\n", + " \"https://api.coingecko.com/api/v3/coins/ethereum\" # CoinGecko API\n", + "]\n", + "\n", + "# Real database connection for transaction history\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/blockchain_db\"\n", + "db_query = \"SELECT tx_hash, from_address, to_address, value, timestamp, block_number FROM transactions WHERE timestamp > NOW() - INTERVAL '24 hours' ORDER BY timestamp DESC LIMIT 10000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample transaction data for local ingestion\n", + "transaction_data_file = os.path.join(temp_dir, \"transactions.json\")\n", + "transaction_data = [\n", + " {\n", + " \"tx_hash\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\",\n", + " \"from_address\": \"0xabc123def456abc123def456abc123def456abc12\",\n", + " \"to_address\": \"0xdef456abc123def456abc123def456abc123def45\",\n", + " \"value\": \"1000000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"block_number\": 18500000,\n", + " \"gas_used\": 21000\n", + " },\n", + " {\n", + " \"tx_hash\": \"0x2345678901bcdef2345678901bcdef2345678901bcdef2345678901bcdef23\",\n", + " \"from_address\": \"0xdef456abc123def456abc123def456abc123def45\",\n", + " \"to_address\": \"0x7890123456789012345678901234567890123456\",\n", + " \"value\": \"500000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=2)).isoformat(),\n", + " \"block_number\": 18499950,\n", + " \"gas_used\": 21000\n", + " },\n", + " {\n", + " \"tx_hash\": \"0x3456789012cdef3456789012cdef3456789012cdef3456789012cdef3456\",\n", + " \"from_address\": \"0x7890123456789012345678901234567890123456\",\n", + " \"to_address\": \"0xabc123def456abc123def456abc123def456abc12\",\n", + " \"value\": \"2000000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=3)).isoformat(),\n", + " \"block_number\": 18499900,\n", + " \"gas_used\": 21000\n", + " },\n", + " {\n", + " \"tx_hash\": \"0x4567890123def4567890123def4567890123def4567890123def4567890123\",\n", + " \"from_address\": \"0xabc123def456abc123def456abc123def456abc12\",\n", + " \"to_address\": \"0x4567890123456789012345678901234567890123\",\n", + " \"value\": \"300000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=4)).isoformat(),\n", + " \"block_number\": 18499850,\n", + " \"gas_used\": 21000\n", + " },\n", + " {\n", + " \"tx_hash\": \"0x5678901234ef5678901234ef5678901234ef5678901234ef5678901234ef56\",\n", + " \"from_address\": \"0x4567890123456789012345678901234567890123\",\n", + " \"to_address\": \"0x1234567890123456789012345678901234567890\",\n", + " \"value\": \"1500000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=5)).isoformat(),\n", + " \"block_number\": 18499800,\n", + " \"gas_used\": 21000\n", + " }\n", + "]\n", + "\n", + "with open(transaction_data_file, 'w') as f:\n", + " json.dump(transaction_data, f, indent=2)\n", + "\n", + "# Ingest from local file\n", + "file_data = file_ingestor.ingest_file(transaction_data_file)\n", + "parsed_transactions = structured_parser.parse_json(json.dumps(transaction_data))\n", + "\n", + "# Ingest from blockchain APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(blockchain_apis[2]) # CoinGecko public API\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern (would connect to real database)\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"transactions\",\n", + " limit=10000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " db_data = {\"data\": transaction_data}\n", + "\n", + "# Streaming ingestion pattern\n", + "print(f\"✓ Streaming sources configured:\")\n", + "for stream_source in stream_sources:\n", + " print(f\" - {stream_source['type']}: {stream_source.get('topic') or stream_source.get('queue')}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Local transactions: {len(transaction_data)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Streaming sources: {len(stream_sources)}\")\n", + "print(f\" Web APIs: {len(blockchain_apis)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Transaction Entities\n", + "\n", + "Extract wallets, transactions, and addresses from the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "all_transaction_texts = []\n", + "all_transactions = []\n", + "\n", + "# Process parsed transactions\n", + "if parsed_transactions and isinstance(parsed_transactions, dict):\n", + " transactions = parsed_transactions.get(\"data\", transaction_data)\n", + " for tx in transactions:\n", + " all_transactions.append(tx)\n", + " tx_text = f\"Transaction {tx.get('tx_hash', '')} from {tx.get('from_address', '')} to {tx.get('to_address', '')} value {tx.get('value', '')} at {tx.get('timestamp', '')}\"\n", + " all_transaction_texts.append(tx_text)\n", + "\n", + "# Extract entities\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_events = []\n", + "all_triples = []\n", + "\n", + "for text in all_transaction_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " events = event_detector.detect_events(text)\n", + " all_events.extend(events)\n", + " \n", + " triples = triple_extractor.extract(text)\n", + " all_triples.extend(triples)\n", + "\n", + "# Build structured entity list\n", + "transaction_entities = []\n", + "wallet_entities = []\n", + "\n", + "for tx in all_transactions:\n", + " tx_entity = {\n", + " \"id\": tx.get(\"tx_hash\", \"\"),\n", + " \"type\": \"Transaction\",\n", + " \"properties\": {\n", + " \"from_address\": tx.get(\"from_address\", \"\"),\n", + " \"to_address\": tx.get(\"to_address\", \"\"),\n", + " \"value\": tx.get(\"value\", \"\"),\n", + " \"timestamp\": tx.get(\"timestamp\", \"\"),\n", + " \"block_number\": tx.get(\"block_number\", 0),\n", + " \"gas_used\": tx.get(\"gas_used\", 0)\n", + " }\n", + " }\n", + " transaction_entities.append(tx_entity)\n", + " \n", + " # Add wallet entities\n", + " from_wallet = {\n", + " \"id\": tx.get(\"from_address\", \"\"),\n", + " \"type\": \"Wallet\",\n", + " \"properties\": {\n", + " \"address\": tx.get(\"from_address\", \"\"),\n", + " \"role\": \"sender\"\n", + " }\n", + " }\n", + " to_wallet = {\n", + " \"id\": tx.get(\"to_address\", \"\"),\n", + " \"type\": \"Wallet\",\n", + " \"properties\": {\n", + " \"address\": tx.get(\"to_address\", \"\"),\n", + " \"role\": \"receiver\"\n", + " }\n", + " }\n", + " wallet_entities.append(from_wallet)\n", + " wallet_entities.append(to_wallet)\n", + "\n", + "# Deduplicate wallets\n", + "unique_wallets = {}\n", + "for wallet in wallet_entities:\n", + " wallet_id = wallet[\"id\"]\n", + " if wallet_id not in unique_wallets:\n", + " unique_wallets[wallet_id] = wallet\n", + "\n", + "wallet_entities = list(unique_wallets.values())\n", + "\n", + "print(f\"Extracted {len(transaction_entities)} transactions\")\n", + "print(f\"Extracted {len(wallet_entities)} unique wallets\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Detected {len(all_events)} events\")\n", + "print(f\"Extracted {len(all_triples)} triples\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Temporal Transaction Knowledge Graph\n", + "\n", + "Build a temporal knowledge graph from extracted transactions and wallets.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "# Add all entities\n", + "for wallet in wallet_entities:\n", + " builder.add_entity(\n", + " entity_id=wallet[\"id\"],\n", + " entity_type=wallet[\"type\"],\n", + " properties=wallet.get(\"properties\", {})\n", + " )\n", + "\n", + "for tx in transaction_entities:\n", + " builder.add_entity(\n", + " entity_id=tx[\"id\"],\n", + " entity_type=tx[\"type\"],\n", + " properties=tx.get(\"properties\", {})\n", + " )\n", + "\n", + "# Add relationships\n", + "relationships = []\n", + "for tx in transaction_entities:\n", + " from_addr = tx[\"properties\"].get(\"from_address\", \"\")\n", + " to_addr = tx[\"properties\"].get(\"to_address\", \"\")\n", + " tx_hash = tx[\"id\"]\n", + " value = tx[\"properties\"].get(\"value\", \"\")\n", + " timestamp = tx[\"properties\"].get(\"timestamp\", \"\")\n", + " \n", + " # Transaction relationship\n", + " rel = {\n", + " \"source\": from_addr,\n", + " \"target\": to_addr,\n", + " \"type\": \"transfers_to\",\n", + " \"properties\": {\n", + " \"transaction\": tx_hash,\n", + " \"value\": value,\n", + " \"timestamp\": timestamp\n", + " }\n", + " }\n", + " relationships.append(rel)\n", + " builder.add_relationship(\n", + " source_id=from_addr,\n", + " target_id=to_addr,\n", + " relationship_type=\"transfers_to\",\n", + " properties=rel[\"properties\"]\n", + " )\n", + " \n", + " # Transaction entity relationship\n", + " builder.add_relationship(\n", + " source_id=from_addr,\n", + " target_id=tx_hash,\n", + " relationship_type=\"initiates\",\n", + " properties={\"timestamp\": timestamp}\n", + " )\n", + " builder.add_relationship(\n", + " source_id=tx_hash,\n", + " target_id=to_addr,\n", + " relationship_type=\"sends_to\",\n", + " properties={\"timestamp\": timestamp}\n", + " )\n", + "\n", + "knowledge_graph = builder.build()\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.nodes)} nodes\")\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.edges)} edges\")\n", + "print(f\"Added {len(relationships)} transaction relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Detect AML Patterns\n", + "\n", + "Detect money laundering patterns: tumbling, mixing, clustering, and suspicious flows.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temporal_query = TemporalGraphQuery(knowledge_graph)\n", + "pattern_detector = TemporalPatternDetector(knowledge_graph)\n", + "graph_analyzer = GraphAnalyzer(knowledge_graph)\n", + "centrality_calculator = CentralityCalculator(knowledge_graph)\n", + "community_detector = CommunityDetector(knowledge_graph)\n", + "connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n", + "\n", + "# Query transactions in time range\n", + "start_time = (datetime.now() - timedelta(hours=6)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " start_time=start_time,\n", + " end_time=end_time,\n", + " relationship_types=[\"transfers_to\", \"initiates\", \"sends_to\"]\n", + ")\n", + "\n", + "# Detect temporal patterns\n", + "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", + " relationship_types=[\"transfers_to\"],\n", + " time_window_hours=6\n", + ")\n", + "\n", + "# Calculate centrality to find key wallets\n", + "centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n", + "top_central_wallets = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n", + "\n", + "# Detect communities (clustering)\n", + "communities = community_detector.detect_communities()\n", + "community_count = len(set(communities.values())) if communities else 0\n", + "\n", + "# Analyze connectivity\n", + "connectivity_results = connectivity_analyzer.analyze_connectivity()\n", + "\n", + "# AML Pattern Detection using Inference Engine\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Define AML rules\n", + "aml_rules = [\n", + " {\n", + " \"name\": \"tumbling_pattern\",\n", + " \"condition\": \"high_transaction_count AND multiple_intermediate_wallets\",\n", + " \"action\": \"flag_as_tumbling\"\n", + " },\n", + " {\n", + " \"name\": \"mixing_pattern\",\n", + " \"condition\": \"funds_split_into_multiple_addresses AND rapid_consolidation\",\n", + " \"action\": \"flag_as_mixing\"\n", + " },\n", + " {\n", + " \"name\": \"suspicious_flow\",\n", + " \"condition\": \"large_value_transfer AND short_time_window\",\n", + " \"action\": \"flag_as_suspicious\"\n", + " }\n", + "]\n", + "\n", + "for rule in aml_rules:\n", + " rule_manager.add_rule(rule[\"name\"], rule[\"condition\"], rule[\"action\"])\n", + "\n", + "# Add facts from graph analysis\n", + "aml_facts = []\n", + "for wallet_id, centrality in top_central_wallets[:5]:\n", + " aml_facts.append({\n", + " \"wallet\": wallet_id,\n", + " \"centrality\": centrality,\n", + " \"high_transaction_count\": True if centrality > 0.1 else False\n", + " })\n", + "\n", + "# Detect patterns\n", + "suspicious_patterns = []\n", + "for wallet_id, centrality in top_central_wallets:\n", + " if centrality > 0.15:\n", + " suspicious_patterns.append({\n", + " \"wallet\": wallet_id,\n", + " \"pattern\": \"high_centrality\",\n", + " \"risk_score\": min(centrality * 10, 10),\n", + " \"description\": f\"Wallet {wallet_id[:10]}... has high betweenness centrality ({centrality:.3f}), indicating potential mixing/tumbling\"\n", + " })\n", + "\n", + "# Check for rapid transactions (tumbling pattern)\n", + "wallet_transaction_counts = {}\n", + "for rel in relationships:\n", + " source = rel[\"source\"]\n", + " wallet_transaction_counts[source] = wallet_transaction_counts.get(source, 0) + 1\n", + "\n", + "for wallet_id, count in wallet_transaction_counts.items():\n", + " if count >= 3:\n", + " suspicious_patterns.append({\n", + " \"wallet\": wallet_id,\n", + " \"pattern\": \"rapid_transactions\",\n", + " \"risk_score\": min(count * 2, 10),\n", + " \"description\": f\"Wallet {wallet_id[:10]}... has {count} outgoing transactions, potential tumbling\"\n", + " })\n", + "\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"Found {community_count} wallet communities\")\n", + "print(f\"Identified {len(suspicious_patterns)} suspicious patterns\")\n", + "print(f\"\\nTop 5 Central Wallets:\")\n", + "for i, (wallet_id, centrality) in enumerate(top_central_wallets[:5], 1):\n", + " print(f\" {i}. {wallet_id[:20]}... (centrality: {centrality:.3f})\")\n", + "print(f\"\\nSuspicious Patterns Detected:\")\n", + "for pattern in suspicious_patterns[:5]:\n", + " print(f\" - {pattern['pattern']}: {pattern['wallet'][:20]}... (risk: {pattern['risk_score']:.1f}/10)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Alerts and Reports\n", + "\n", + "Generate AML alerts and comprehensive analysis reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "kg_quality_assessor = KGQualityAssessor()\n", + "conflict_detector = ConflictDetector(knowledge_graph)\n", + "\n", + "# Assess graph quality\n", + "quality_metrics = kg_quality_assessor.assess_quality(knowledge_graph)\n", + "\n", + "# Detect conflicts\n", + "conflicts = conflict_detector.detect_conflicts()\n", + "\n", + "# Generate alerts\n", + "alerts = []\n", + "for pattern in suspicious_patterns:\n", + " if pattern[\"risk_score\"] >= 5.0:\n", + " alerts.append({\n", + " \"alert_id\": f\"AML_{pattern['wallet'][:8]}\",\n", + " \"type\": \"AML_SUSPICIOUS_PATTERN\",\n", + " \"severity\": \"HIGH\" if pattern[\"risk_score\"] >= 7.0 else \"MEDIUM\",\n", + " \"wallet\": pattern[\"wallet\"],\n", + " \"pattern\": pattern[\"pattern\"],\n", + " \"risk_score\": pattern[\"risk_score\"],\n", + " \"description\": pattern[\"description\"],\n", + " \"timestamp\": datetime.now().isoformat()\n", + " })\n", + "\n", + "# Export knowledge graph\n", + "kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"transaction_kg.json\"))\n", + "kg_rdf = rdf_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"transaction_kg.rdf\"))\n", + "\n", + "# Generate report\n", + "report_content = f\"\"\"\n", + "# Blockchain Transaction Network Analysis Report\n", + "\n", + "## Executive Summary\n", + "- Total Transactions Analyzed: {len(transaction_entities)}\n", + "- Unique Wallets: {len(wallet_entities)}\n", + "- Suspicious Patterns Detected: {len(suspicious_patterns)}\n", + "- High-Risk Alerts: {len([a for a in alerts if a['severity'] == 'HIGH'])}\n", + "\n", + "## Graph Quality Metrics\n", + "- Nodes: {quality_metrics.get('node_count', len(knowledge_graph.nodes))}\n", + "- Edges: {quality_metrics.get('edge_count', len(knowledge_graph.edges))}\n", + "- Completeness: {quality_metrics.get('completeness', 0):.2%}\n", + "- Consistency: {quality_metrics.get('consistency', 0):.2%}\n", + "\n", + "## Top Suspicious Patterns\n", + "\"\"\"\n", + "for i, pattern in enumerate(suspicious_patterns[:10], 1):\n", + " report_content += f\"\"\"\n", + "### {i}. {pattern['pattern'].upper()}\n", + "- Wallet: {pattern['wallet']}\n", + "- Risk Score: {pattern['risk_score']:.1f}/10\n", + "- Description: {pattern['description']}\n", + "\"\"\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Alerts Generated\n", + "\"\"\"\n", + "for alert in alerts:\n", + " report_content += f\"\"\"\n", + "- **{alert['alert_id']}** ({alert['severity']}): {alert['description']}\n", + "\"\"\"\n", + "\n", + "report_path = os.path.join(temp_dir, \"aml_analysis_report.md\")\n", + "with open(report_path, 'w') as f:\n", + " f.write(report_content)\n", + "\n", + "print(f\"Generated {len(alerts)} AML alerts\")\n", + "print(f\"Exported knowledge graph to JSON and RDF\")\n", + "print(f\"Generated analysis report: {report_path}\")\n", + "print(f\"\\nQuality Metrics:\")\n", + "print(f\" Nodes: {quality_metrics.get('node_count', len(knowledge_graph.nodes))}\")\n", + "print(f\" Edges: {quality_metrics.get('edge_count', len(knowledge_graph.edges))}\")\n", + "print(f\" Completeness: {quality_metrics.get('completeness', 0):.2%}\")\n", + "print(f\" Consistency: {quality_metrics.get('consistency', 0):.2%}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Visualize Transaction Network\n", + "\n", + "Visualize the transaction network, patterns, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "kg_viz = kg_visualizer.visualize(\n", + " knowledge_graph,\n", + " layout=\"force_directed\",\n", + " highlight_nodes=[p[\"wallet\"] for p in suspicious_patterns[:5]],\n", + " node_size_by=\"centrality\"\n", + ")\n", + "\n", + "# Visualize temporal patterns\n", + "temporal_viz = temporal_visualizer.visualize(\n", + " knowledge_graph,\n", + " time_attribute=\"timestamp\",\n", + " relationship_types=[\"transfers_to\"]\n", + ")\n", + "\n", + "# Visualize analytics\n", + "analytics_viz = analytics_visualizer.visualize(\n", + " knowledge_graph,\n", + " metrics={\n", + " \"centrality\": dict(top_central_wallets[:10]),\n", + " \"communities\": communities,\n", + " \"connectivity\": connectivity_results\n", + " }\n", + ")\n", + "\n", + "print(\"Generated visualizations:\")\n", + "print(\" - Knowledge Graph: Transaction network with highlighted suspicious wallets\")\n", + "print(\" - Temporal Visualization: Transaction flows over time\")\n", + "print(\" - Analytics Visualization: Centrality, communities, and connectivity metrics\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/cybersecurity/Anomaly_Detection_Real_Time.ipynb b/docs/cookbook/use_cases/cybersecurity/Anomaly_Detection_Real_Time.ipynb new file mode 100644 index 00000000..17a5969c --- /dev/null +++ b/docs/cookbook/use_cases/cybersecurity/Anomaly_Detection_Real_Time.ipynb @@ -0,0 +1,471 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Real-Time Anomaly Detection Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete real-time anomaly detection pipeline for cybersecurity: stream security logs from multiple sources, parse in real-time, build temporal knowledge graph, detect anomalies using pattern detection and inference, generate alerts, and monitor continuously.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: StreamIngestor, FileIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalPatternDetector, TemporalGraphQuery, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, AutomatedFixer\n", + "- **Export**: JSONExporter, CSVExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Stream Security Logs → Real-Time Parsing → Extract Entities → Build Temporal KG → Pattern Detection → Anomaly Detection → Generate Alerts → Monitor → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Stream Security Logs from Multiple Sources\n", + "\n", + "Stream security logs from files, databases, and real-time sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import StreamIngestor, FileIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalPatternDetector, TemporalGraphQuery, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor, AutomatedFixer\n", + "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "import time\n", + "from datetime import datetime, timedelta\n", + "from collections import deque\n", + "\n", + "stream_ingestor = StreamIngestor()\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "# Real streaming sources configuration\n", + "stream_sources = [\n", + " {\n", + " \"type\": \"kafka\",\n", + " \"topic\": \"security_logs\",\n", + " \"bootstrap_servers\": [\"localhost:9092\"],\n", + " \"consumer_config\": {\"group_id\": \"semantica_security_monitor\"}\n", + " },\n", + " {\n", + " \"type\": \"rabbitmq\",\n", + " \"queue\": \"security_events\",\n", + " \"connection_url\": \"amqp://user:password@localhost:5672/\"\n", + " }\n", + "]\n", + "\n", + "# Real database connection for security logs\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/security_logs_db\"\n", + "db_query = \"SELECT * FROM security_events WHERE timestamp > NOW() - INTERVAL '1 hour' ORDER BY timestamp DESC LIMIT 1000\"\n", + "\n", + "# Real security feed URLs for threat intelligence\n", + "security_feeds = [\n", + " \"https://www.cisa.gov/news.xml\",\n", + " \"https://www.us-cert.gov/ncas/alerts.xml\"\n", + "]\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Real-world streaming security log format (simulating real-time stream)\n", + "security_log_stream_file = os.path.join(temp_dir, \"security_log_stream.json\")\n", + "stream_logs = [\n", + " {\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=5)).isoformat(),\n", + " \"source_ip\": \"192.168.1.50\",\n", + " \"destination_ip\": \"10.0.0.100\",\n", + " \"event_type\": \"normal_traffic\",\n", + " \"bytes_sent\": 1024,\n", + " \"bytes_received\": 2048,\n", + " \"protocol\": \"TCP\",\n", + " \"port\": 80\n", + " },\n", + " {\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=4)).isoformat(),\n", + " \"source_ip\": \"203.0.113.100\",\n", + " \"destination_ip\": \"10.0.0.100\",\n", + " \"event_type\": \"suspicious_connection\",\n", + " \"bytes_sent\": 5000000,\n", + " \"bytes_received\": 1000,\n", + " \"protocol\": \"TCP\",\n", + " \"port\": 443\n", + " },\n", + " {\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=3)).isoformat(),\n", + " \"source_ip\": \"192.168.1.50\",\n", + " \"destination_ip\": \"10.0.0.100\",\n", + " \"event_type\": \"normal_traffic\",\n", + " \"bytes_sent\": 512,\n", + " \"bytes_received\": 1024,\n", + " \"protocol\": \"UDP\",\n", + " \"port\": 53\n", + " },\n", + " {\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=2)).isoformat(),\n", + " \"source_ip\": \"198.51.100.50\",\n", + " \"destination_ip\": \"10.0.0.100\",\n", + " \"event_type\": \"port_scan\",\n", + " \"bytes_sent\": 100,\n", + " \"bytes_received\": 0,\n", + " \"protocol\": \"TCP\",\n", + " \"port\": 22\n", + " },\n", + " {\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=1)).isoformat(),\n", + " \"source_ip\": \"203.0.113.100\",\n", + " \"destination_ip\": \"10.0.0.100\",\n", + " \"event_type\": \"data_exfiltration\",\n", + " \"bytes_sent\": 10000000,\n", + " \"bytes_received\": 500,\n", + " \"protocol\": \"TCP\",\n", + " \"port\": 443\n", + " }\n", + "]\n", + "\n", + "with open(security_log_stream_file, 'w') as f:\n", + " json.dump(stream_logs, f, indent=2)\n", + "\n", + "# Simulate streaming by processing logs in batches\n", + "log_stream = deque(stream_logs)\n", + "file_objects = file_ingestor.ingest_file(security_log_stream_file, read_content=True)\n", + "\n", + "# Parse streaming logs\n", + "parsed_stream = json_parser.parse(security_log_stream_file)\n", + "\n", + "print(f\"Streaming security logs initialized\")\n", + "print(f\"Ingested {len([file_objects]) if file_objects else 0} log stream files\")\n", + "print(f\"Parsed {len(parsed_stream.data) if parsed_stream and parsed_stream.data else 0} log entries\")\n", + "print(f\"Stream ready for real-time processing\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Real-Time Parsing and Entity Extraction\n", + "\n", + "Parse streaming logs in real-time and extract security entities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "# Real-time processing loop (simulated)\n", + "security_entities = []\n", + "stream_relationships = []\n", + "detected_events = []\n", + "\n", + "# Process logs in real-time batches\n", + "for log_entry in parsed_stream.data if parsed_stream and parsed_stream.data else []:\n", + " if isinstance(log_entry, dict):\n", + " log_text = f\"{log_entry.get('event_type', '')} from {log_entry.get('source_ip', '')} to {log_entry.get('destination_ip', '')} on port {log_entry.get('port', '')}\"\n", + " \n", + " entities = ner_extractor.extract(log_text)\n", + " relationships = relation_extractor.extract(log_text, entities)\n", + " events = event_detector.detect_events(log_text)\n", + " \n", + " security_entities.append({\n", + " \"id\": log_entry.get(\"source_ip\", \"\"),\n", + " \"type\": \"IP_Address\",\n", + " \"name\": log_entry.get(\"source_ip\", \"\"),\n", + " \"properties\": {\n", + " \"timestamp\": log_entry.get(\"timestamp\", \"\"),\n", + " \"source\": \"stream\"\n", + " }\n", + " })\n", + " security_entities.append({\n", + " \"id\": log_entry.get(\"destination_ip\", \"\"),\n", + " \"type\": \"IP_Address\",\n", + " \"name\": log_entry.get(\"destination_ip\", \"\"),\n", + " \"properties\": {\n", + " \"timestamp\": log_entry.get(\"timestamp\", \"\"),\n", + " \"source\": \"stream\"\n", + " }\n", + " })\n", + " security_entities.append({\n", + " \"id\": log_entry.get(\"event_type\", \"\"),\n", + " \"type\": \"Security_Event\",\n", + " \"name\": log_entry.get(\"event_type\", \"\"),\n", + " \"properties\": {\n", + " \"timestamp\": log_entry.get(\"timestamp\", \"\"),\n", + " \"bytes_sent\": log_entry.get(\"bytes_sent\", 0),\n", + " \"bytes_received\": log_entry.get(\"bytes_received\", 0),\n", + " \"protocol\": log_entry.get(\"protocol\", \"\"),\n", + " \"port\": log_entry.get(\"port\", 0)\n", + " }\n", + " })\n", + " \n", + " stream_relationships.append({\n", + " \"source\": log_entry.get(\"source_ip\", \"\"),\n", + " \"target\": log_entry.get(\"event_type\", \"\"),\n", + " \"type\": \"triggered\",\n", + " \"properties\": {\"timestamp\": log_entry.get(\"timestamp\", \"\")}\n", + " })\n", + " stream_relationships.append({\n", + " \"source\": log_entry.get(\"event_type\", \"\"),\n", + " \"target\": log_entry.get(\"destination_ip\", \"\"),\n", + " \"type\": \"targeted\",\n", + " \"properties\": {\"timestamp\": log_entry.get(\"timestamp\", \"\")}\n", + " })\n", + " \n", + " detected_events.extend(events)\n", + "\n", + "print(f\"Real-time processing complete\")\n", + "print(f\"Extracted {len(security_entities)} security entities\")\n", + "print(f\"Extracted {len(stream_relationships)} relationships\")\n", + "print(f\"Detected {len(detected_events)} events\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Temporal Knowledge Graph\n", + "\n", + "Build and continuously update temporal knowledge graph from streaming data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "temporal_query = TemporalGraphQuery()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "# Build temporal KG from streaming data\n", + "temporal_kg = builder.build(security_entities, stream_relationships)\n", + "\n", + "# Analyze graph structure in real-time\n", + "metrics = graph_analyzer.compute_metrics(temporal_kg)\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "centrality_scores = centrality_calculator.calculate_centrality(temporal_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(temporal_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(temporal_kg)\n", + "\n", + "print(f\"Built temporal knowledge graph from stream\")\n", + "print(f\" Entities: {len(temporal_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(temporal_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Communities: {len(communities)}\")\n", + "print(f\" Central entities: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Real-Time Pattern Detection\n", + "\n", + "Detect temporal patterns and anomalies in real-time.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Detect temporal patterns\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " temporal_kg,\n", + " pattern_type=\"anomaly\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "# Real-time anomaly detection using inference\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Define real-time anomaly detection rules\n", + "inference_engine.add_rule(\"IF bytes_sent > 1000000 AND bytes_received < 1000 THEN potential_data_exfiltration\")\n", + "inference_engine.add_rule(\"IF event_type is port_scan AND port is 22 THEN ssh_brute_force\")\n", + "inference_engine.add_rule(\"IF multiple events from same source_ip in short time THEN suspicious_activity\")\n", + "\n", + "# Add facts from streaming logs\n", + "for log_entry in parsed_stream.data if parsed_stream and parsed_stream.data else []:\n", + " if isinstance(log_entry, dict):\n", + " inference_engine.add_fact({\n", + " \"source_ip\": log_entry.get(\"source_ip\", \"\"),\n", + " \"event_type\": log_entry.get(\"event_type\", \"\"),\n", + " \"bytes_sent\": log_entry.get(\"bytes_sent\", 0),\n", + " \"bytes_received\": log_entry.get(\"bytes_received\", 0),\n", + " \"port\": log_entry.get(\"port\", 0),\n", + " \"timestamp\": log_entry.get(\"timestamp\", \"\")\n", + " })\n", + "\n", + "inferred_anomalies = inference_engine.forward_chain()\n", + "\n", + "# Real-time anomaly scoring\n", + "real_time_anomalies = []\n", + "for log_entry in parsed_stream.data if parsed_stream and parsed_stream.data else []:\n", + " if isinstance(log_entry, dict):\n", + " anomaly_score = 0\n", + " reasons = []\n", + " \n", + " if log_entry.get(\"bytes_sent\", 0) > 1000000:\n", + " anomaly_score += 5\n", + " reasons.append(\"Unusually large data transfer\")\n", + " \n", + " if log_entry.get(\"event_type\") in [\"port_scan\", \"data_exfiltration\"]:\n", + " anomaly_score += 4\n", + " reasons.append(\"High-risk event type\")\n", + " \n", + " if log_entry.get(\"bytes_sent\", 0) > log_entry.get(\"bytes_received\", 0) * 100:\n", + " anomaly_score += 3\n", + " reasons.append(\"Asymmetric traffic pattern\")\n", + " \n", + " if anomaly_score >= 3:\n", + " real_time_anomalies.append({\n", + " \"source_ip\": log_entry.get(\"source_ip\", \"\"),\n", + " \"destination_ip\": log_entry.get(\"destination_ip\", \"\"),\n", + " \"event_type\": log_entry.get(\"event_type\", \"\"),\n", + " \"severity\": \"high\" if anomaly_score >= 5 else \"medium\",\n", + " \"score\": anomaly_score,\n", + " \"reasons\": reasons,\n", + " \"timestamp\": log_entry.get(\"timestamp\", \"\")\n", + " })\n", + "\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"Inferred {len(inferred_anomalies)} anomalies from rules\")\n", + "print(f\"Identified {len(real_time_anomalies)} real-time anomalies\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Real-Time Alerts\n", + "\n", + "Generate and send alerts for detected anomalies.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(temporal_kg)\n", + "\n", + "# Generate alerts\n", + "alerts = []\n", + "for anomaly in real_time_anomalies:\n", + " alert = {\n", + " \"alert_id\": f\"alert_{anomaly['source_ip']}_{int(time.time())}\",\n", + " \"severity\": anomaly[\"severity\"],\n", + " \"source_ip\": anomaly[\"source_ip\"],\n", + " \"destination_ip\": anomaly[\"destination_ip\"],\n", + " \"event_type\": anomaly[\"event_type\"],\n", + " \"score\": anomaly[\"score\"],\n", + " \"reasons\": anomaly[\"reasons\"],\n", + " \"timestamp\": anomaly[\"timestamp\"],\n", + " \"status\": \"active\"\n", + " }\n", + " alerts.append(alert)\n", + "\n", + "# Export alerts\n", + "json_exporter.export_knowledge_graph(temporal_kg, os.path.join(temp_dir, \"realtime_kg.json\"))\n", + "csv_exporter.export_entities(security_entities, os.path.join(temp_dir, \"realtime_entities.csv\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Real-time anomaly detection identified {len(real_time_anomalies)} anomalies\",\n", + " \"total_events\": len(parsed_stream.data) if parsed_stream and parsed_stream.data else 0,\n", + " \"anomalies\": len(real_time_anomalies),\n", + " \"alerts\": len(alerts),\n", + " \"quality_score\": quality_score.get('overall_score', 0),\n", + " \"high_severity\": len([a for a in alerts if a.get('severity') == 'high'])\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(f\"Generated {len(alerts)} real-time alerts\")\n", + "print(f\"High severity alerts: {len([a for a in alerts if a.get('severity') == 'high'])}\")\n", + "print(f\"Report length: {len(report)} characters\")\n", + "print(f\"Graph quality score: {quality_score.get('overall_score', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Real-Time Monitoring and Visualization\n", + "\n", + "Monitor security events in real-time and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(temporal_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(temporal_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(temporal_kg, output=\"interactive\")\n", + "\n", + "print(f\"Real-time monitoring active\")\n", + "print(f\"Monitoring {len(temporal_kg.get('entities', []))} entities in real-time\")\n", + "print(f\"Active alerts: {len(alerts)}\")\n", + "print(\"Generated visualizations for knowledge graph, temporal patterns, and analytics\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Stream Logs → Real-Time Parse → Extract → Temporal KG → Pattern Detection → Anomaly Detection → Alerts → Monitor → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/cybersecurity/Incident_Analysis.ipynb b/docs/cookbook/use_cases/cybersecurity/Incident_Analysis.ipynb new file mode 100644 index 00000000..f6121bba --- /dev/null +++ b/docs/cookbook/use_cases/cybersecurity/Incident_Analysis.ipynb @@ -0,0 +1,455 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Incident Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete security incident analysis pipeline: ingest security logs from multiple sources (files, databases, streams), parse structured and unstructured logs, extract security entities, build knowledge graph, analyze relationships, detect anomalies, and generate incident reports.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, DBIngestor, StreamIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, XMLParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer, CentralityCalculator\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector, ProvenanceTracker\n", + "- **Export**: JSONExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Multiple Security Sources → Parse Logs → Extract Security Entities → Build Incident KG → Analyze Relationships → Detect Anomalies → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Security Logs from Multiple Sources\n", + "\n", + "Ingest security logs from files, databases, streams, and threat intelligence feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, DBIngestor, StreamIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, XMLParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer, CentralityCalculator\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.kg import ProvenanceTracker\n", + "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "stream_ingestor = StreamIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "xml_parser = XMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Real-world security log formats\n", + "security_logs_json = os.path.join(temp_dir, \"security_logs.json\")\n", + "security_logs_data = [\n", + " {\n", + " \"timestamp\": (datetime.now() - timedelta(hours=2)).isoformat(),\n", + " \"source_ip\": \"192.168.1.100\",\n", + " \"destination_ip\": \"10.0.0.50\",\n", + " \"event_type\": \"failed_login\",\n", + " \"user\": \"admin\",\n", + " \"severity\": \"medium\",\n", + " \"message\": \"Multiple failed login attempts detected\"\n", + " },\n", + " {\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"source_ip\": \"203.0.113.45\",\n", + " \"destination_ip\": \"10.0.0.50\",\n", + " \"event_type\": \"port_scan\",\n", + " \"severity\": \"high\",\n", + " \"message\": \"Port scanning activity detected from external IP\"\n", + " },\n", + " {\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=30)).isoformat(),\n", + " \"source_ip\": \"192.168.1.100\",\n", + " \"destination_ip\": \"10.0.0.75\",\n", + " \"event_type\": \"data_exfiltration\",\n", + " \"user\": \"user123\",\n", + " \"severity\": \"critical\",\n", + " \"message\": \"Large data transfer detected to external server\"\n", + " }\n", + "]\n", + "\n", + "with open(security_logs_json, 'w') as f:\n", + " json.dump(security_logs_data, f, indent=2)\n", + "\n", + "# XML format security events (common in SIEM systems)\n", + "security_events_xml = os.path.join(temp_dir, \"security_events.xml\")\n", + "xml_content = \"\"\"\n", + "\n", + " \n", + " 2024-01-15T14:30:00\n", + " 172.16.0.10\n", + " 10.0.0.50\n", + " malware_detection\n", + " high\n", + " Malware signature detected in file transfer\n", + " \n", + " \n", + " 2024-01-15T15:00:00\n", + " 192.168.1.200\n", + " 10.0.0.50\n", + " unauthorized_access\n", + " critical\n", + " Unauthorized access attempt to restricted resource\n", + " \n", + "\"\"\"\n", + "\n", + "with open(security_events_xml, 'w') as f:\n", + " f.write(xml_content)\n", + "\n", + "# Ingest from files\n", + "file_objects = file_ingestor.ingest_file(security_logs_json, read_content=True)\n", + "file_objects_xml = file_ingestor.ingest_file(security_events_xml, read_content=True)\n", + "\n", + "# Parse structured logs\n", + "parsed_json = json_parser.parse(security_logs_json)\n", + "parsed_xml = xml_parser.parse(security_events_xml)\n", + "\n", + "# Real security intelligence feed URLs\n", + "security_feeds = [\n", + " \"https://www.cisa.gov/news.xml\", # CISA Security Advisories\n", + " \"https://www.us-cert.gov/ncas/alerts.xml\", # US-CERT Alerts\n", + " \"https://feeds.feedburner.com/SecurityWeek\", # Security Week\n", + " \"https://www.darkreading.com/rss.xml\" # Dark Reading\n", + "]\n", + "\n", + "threat_feed_list = []\n", + "for feed_url in security_feeds:\n", + " try:\n", + " threat_feed = feed_ingestor.ingest_feed(feed_url)\n", + " if threat_feed:\n", + " threat_feed_list.append(threat_feed)\n", + " print(f\"✓ Ingested threat feed: {threat_feed.title if hasattr(threat_feed, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"Ingested {len([file_objects]) if file_objects else 0} JSON log files\")\n", + "print(f\"Ingested {len([file_objects_xml]) if file_objects_xml else 0} XML event files\")\n", + "print(f\"Parsed {len(parsed_json.data) if parsed_json and parsed_json.data else 0} JSON log entries\")\n", + "print(f\"Parsed {len(parsed_xml.elements) if parsed_xml else 0} XML event elements\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Security Entities and Relationships\n", + "\n", + "Extract security entities (IPs, users, events) and relationships from parsed logs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_events = []\n", + "\n", + "# Extract from JSON logs\n", + "if parsed_json and parsed_json.data:\n", + " for log_entry in parsed_json.data:\n", + " if isinstance(log_entry, dict):\n", + " log_text = f\"{log_entry.get('event_type', '')} from {log_entry.get('source_ip', '')} to {log_entry.get('destination_ip', '')}: {log_entry.get('message', '')}\"\n", + " \n", + " entities = ner_extractor.extract(log_text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(log_text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " events = event_detector.detect_events(log_text)\n", + " all_events.extend(events)\n", + "\n", + "# Extract from XML events\n", + "if parsed_xml and parsed_xml.elements:\n", + " for elem in parsed_xml.elements:\n", + " if hasattr(elem, 'text') and elem.text:\n", + " entities = ner_extractor.extract(elem.text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(elem.text, entities)\n", + " all_relationships.extend(relationships)\n", + "\n", + "# Build structured entities from log data\n", + "security_entities = []\n", + "for log_entry in parsed_json.data if parsed_json and parsed_json.data else []:\n", + " if isinstance(log_entry, dict):\n", + " security_entities.append({\n", + " \"id\": log_entry.get(\"source_ip\", \"\"),\n", + " \"type\": \"IP_Address\",\n", + " \"name\": log_entry.get(\"source_ip\", \"\"),\n", + " \"properties\": {\"source\": \"security_logs\"}\n", + " })\n", + " security_entities.append({\n", + " \"id\": log_entry.get(\"destination_ip\", \"\"),\n", + " \"type\": \"IP_Address\",\n", + " \"name\": log_entry.get(\"destination_ip\", \"\"),\n", + " \"properties\": {\"source\": \"security_logs\"}\n", + " })\n", + " if log_entry.get(\"user\"):\n", + " security_entities.append({\n", + " \"id\": log_entry.get(\"user\", \"\"),\n", + " \"type\": \"User\",\n", + " \"name\": log_entry.get(\"user\", \"\"),\n", + " \"properties\": {\"source\": \"security_logs\"}\n", + " })\n", + " security_entities.append({\n", + " \"id\": log_entry.get(\"event_type\", \"\"),\n", + " \"type\": \"Security_Event\",\n", + " \"name\": log_entry.get(\"event_type\", \"\"),\n", + " \"properties\": {\n", + " \"severity\": log_entry.get(\"severity\", \"\"),\n", + " \"timestamp\": log_entry.get(\"timestamp\", \"\"),\n", + " \"message\": log_entry.get(\"message\", \"\")\n", + " }\n", + " })\n", + "\n", + "incident_relationships = []\n", + "for log_entry in parsed_json.data if parsed_json and parsed_json.data else []:\n", + " if isinstance(log_entry, dict):\n", + " incident_relationships.append({\n", + " \"source\": log_entry.get(\"source_ip\", \"\"),\n", + " \"target\": log_entry.get(\"event_type\", \"\"),\n", + " \"type\": \"triggered\",\n", + " \"properties\": {\"timestamp\": log_entry.get(\"timestamp\", \"\")}\n", + " })\n", + " incident_relationships.append({\n", + " \"source\": log_entry.get(\"event_type\", \"\"),\n", + " \"target\": log_entry.get(\"destination_ip\", \"\"),\n", + " \"type\": \"targeted\",\n", + " \"properties\": {\"timestamp\": log_entry.get(\"timestamp\", \"\")}\n", + " })\n", + "\n", + "print(f\"Extracted {len(security_entities)} security entities\")\n", + "print(f\"Extracted {len(incident_relationships)} incident relationships\")\n", + "print(f\"Detected {len(all_events)} security events\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Incident Knowledge Graph\n", + "\n", + "Build a knowledge graph from security entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "provenance_tracker = ProvenanceTracker()\n", + "\n", + "incident_kg = builder.build(security_entities, incident_relationships)\n", + "\n", + "# Track provenance\n", + "for entity in security_entities:\n", + " provenance_tracker.track_entity(entity.get(\"id\"), entity.get(\"properties\", {}).get(\"source\", \"unknown\"), entity)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(incident_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(incident_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(incident_kg, measure=\"degree\")\n", + "\n", + "print(f\"Built incident knowledge graph\")\n", + "print(f\" Entities: {len(incident_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(incident_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Connected components: {len(connectivity.get('components', []))}\")\n", + "print(f\" Central entities: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze Relationships and Detect Anomalies\n", + "\n", + "Analyze security relationships and detect anomalous patterns.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "conflict_detector = ConflictDetector()\n", + "\n", + "# Define security rules\n", + "inference_engine.add_rule(\"IF event_type is port_scan AND severity is high THEN potential_intrusion\")\n", + "inference_engine.add_rule(\"IF event_type is data_exfiltration AND severity is critical THEN data_breach\")\n", + "inference_engine.add_rule(\"IF multiple failed_login events from same source_ip THEN brute_force_attack\")\n", + "\n", + "# Add facts from security events\n", + "for log_entry in parsed_json.data if parsed_json and parsed_json.data else []:\n", + " if isinstance(log_entry, dict):\n", + " inference_engine.add_fact({\n", + " \"event_type\": log_entry.get(\"event_type\", \"\"),\n", + " \"severity\": log_entry.get(\"severity\", \"\"),\n", + " \"source_ip\": log_entry.get(\"source_ip\", \"\")\n", + " })\n", + "\n", + "# Run inference\n", + "inferred_threats = inference_engine.forward_chain()\n", + "\n", + "# Detect anomalies based on patterns\n", + "anomalies = []\n", + "for log_entry in parsed_json.data if parsed_json and parsed_json.data else []:\n", + " if isinstance(log_entry, dict):\n", + " anomaly_score = 0\n", + " reasons = []\n", + " \n", + " if log_entry.get(\"severity\") == \"critical\":\n", + " anomaly_score += 5\n", + " reasons.append(\"Critical severity event\")\n", + " \n", + " if log_entry.get(\"event_type\") in [\"data_exfiltration\", \"unauthorized_access\"]:\n", + " anomaly_score += 4\n", + " reasons.append(\"High-risk event type\")\n", + " \n", + " if log_entry.get(\"severity\") == \"high\" and log_entry.get(\"event_type\") == \"port_scan\":\n", + " anomaly_score += 3\n", + " reasons.append(\"Port scanning detected\")\n", + " \n", + " if anomaly_score >= 3:\n", + " anomalies.append({\n", + " \"event\": log_entry.get(\"event_type\", \"\"),\n", + " \"source_ip\": log_entry.get(\"source_ip\", \"\"),\n", + " \"severity\": log_entry.get(\"severity\", \"\"),\n", + " \"score\": anomaly_score,\n", + " \"reasons\": reasons,\n", + " \"timestamp\": log_entry.get(\"timestamp\", \"\")\n", + " })\n", + "\n", + "# Detect conflicts in security data\n", + "conflicts = conflict_detector.detect_value_conflicts(security_entities, \"name\")\n", + "\n", + "print(f\"Analyzed security relationships\")\n", + "print(f\"Inferred {len(inferred_threats)} potential threats\")\n", + "print(f\"Detected {len(anomalies)} anomalies\")\n", + "print(f\"Found {len(conflicts)} data conflicts\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Incident Reports\n", + "\n", + "Generate comprehensive incident analysis reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(incident_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(incident_kg, os.path.join(temp_dir, \"incident_kg.json\"))\n", + "rdf_exporter.export_knowledge_graph(incident_kg, os.path.join(temp_dir, \"incident_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Security incident analysis identified {len(anomalies)} anomalies and {len(inferred_threats)} potential threats\",\n", + " \"total_events\": len(parsed_json.data) if parsed_json and parsed_json.data else 0,\n", + " \"anomalies\": len(anomalies),\n", + " \"threats\": len(inferred_threats),\n", + " \"quality_score\": quality_score.get('overall_score', 0),\n", + " \"critical_events\": len([e for e in anomalies if e.get('severity') == 'critical'])\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"Generated incident analysis report\")\n", + "print(f\"Report length: {len(report)} characters\")\n", + "print(f\"Graph quality score: {quality_score.get('overall_score', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Visualize Security Incidents\n", + "\n", + "Visualize incident knowledge graph and security patterns.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(incident_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(incident_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(incident_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated visualizations for incident knowledge graph, analytics, and temporal patterns\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Multiple Security Sources → Parse Logs → Extract Entities → Build KG → Analyze → Detect Anomalies → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/cybersecurity/Threat_Correlation.ipynb b/docs/cookbook/use_cases/cybersecurity/Threat_Correlation.ipynb new file mode 100644 index 00000000..7a1f0eab --- /dev/null +++ b/docs/cookbook/use_cases/cybersecurity/Threat_Correlation.ipynb @@ -0,0 +1,426 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Threat Correlation Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete threat correlation pipeline for cybersecurity: ingest threat feeds from multiple sources, extract IOCs, build temporal knowledge graph, correlate threats, detect campaigns, and generate reports.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, FeedIngestor, DBIngestor\n", + "- **Parsing**: XMLParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ProvenanceTracker, ConflictDetector\n", + "- **Export**: RDFExporter, ReportGenerator\n", + "- **Visualization**: AnalyticsVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Multiple Threat Feeds → Parse → Extract IOCs → Build Temporal KG → Correlate Threats → Detect Campaigns → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Threat Feeds\n", + "\n", + "Ingest threat intelligence from multiple sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, FeedIngestor, DBIngestor, WebIngestor\n", + "from semantica.parse import XMLParser, StructuredDataParser, JSONParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.kg import ProvenanceTracker, ConflictDetector\n", + "from semantica.export import RDFExporter, ReportGenerator\n", + "from semantica.visualization import AnalyticsVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "db_ingestor = DBIngestor()\n", + "web_ingestor = WebIngestor()\n", + "xml_parser = XMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "json_parser = JSONParser()\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Real threat intelligence feed URLs\n", + "threat_feeds = [\n", + " \"https://www.cisa.gov/news.xml\", # CISA Security Advisories\n", + " \"https://www.us-cert.gov/ncas/alerts.xml\", # US-CERT Alerts\n", + " \"https://feeds.feedburner.com/SecurityWeek\", # Security Week\n", + " \"https://www.darkreading.com/rss.xml\" # Dark Reading\n", + "]\n", + "\n", + "# Real database connection pattern (PostgreSQL example)\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/threat_intel_db\"\n", + "db_query = \"SELECT ioc, ioc_type, timestamp, severity, source FROM threat_indicators WHERE timestamp > NOW() - INTERVAL '7 days'\"\n", + "\n", + "# Real web API endpoints for threat intelligence\n", + "threat_apis = [\n", + " \"https://api.github.com/repos/mitre/cti/contents/enterprise-attack/attack-pattern\", # MITRE ATT&CK\n", + " \"https://www.virustotal.com/vtapi/v2/domain/report\", # VirusTotal API (requires API key)\n", + " \"https://api.shodan.io/shodan/host/search\" # Shodan API (requires API key)\n", + "]\n", + "\n", + "# Ingest from real RSS feeds\n", + "feed_data_list = []\n", + "for feed_url in threat_feeds:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " feed_data_list.append(feed_data)\n", + " print(f\"✓ Ingested feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " print(f\" Items: {len(feed_data.items) if hasattr(feed_data, 'items') else 0}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion failed for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Ingest from web APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(\"https://api.github.com/repos/mitre/cti\")\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern (would connect to real database)\n", + "try:\n", + " # Example: Export from threat intelligence database\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"threat_indicators\",\n", + " limit=1000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " # Simulate database structure for demonstration\n", + " db_data = {\n", + " \"data\": [\n", + " {\"ioc\": \"192.168.1.100\", \"ioc_type\": \"IP\", \"timestamp\": datetime.now().isoformat(), \"severity\": \"high\", \"source\": \"threat_feed\"},\n", + " {\"ioc\": \"malicious-domain.com\", \"ioc_type\": \"Domain\", \"timestamp\": datetime.now().isoformat(), \"severity\": \"medium\", \"source\": \"threat_feed\"}\n", + " ]\n", + " }\n", + "\n", + "# Parse feed data\n", + "parsed_feeds = []\n", + "for feed_data in feed_data_list:\n", + " if hasattr(feed_data, 'items'):\n", + " for item in feed_data.items[:10]: # Process first 10 items\n", + " parsed_feeds.append({\n", + " \"title\": item.title if hasattr(item, 'title') else \"\",\n", + " \"description\": item.description if hasattr(item, 'description') else \"\",\n", + " \"published\": item.published if hasattr(item, 'published') else \"\",\n", + " \"link\": item.link if hasattr(item, 'link') else \"\"\n", + " })\n", + "\n", + "parsed_db = structured_parser.parse_json(json.dumps(db_data)) if db_data else None\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Feeds ingested: {len(feed_data_list)}\")\n", + "print(f\" Feed items processed: {len(parsed_feeds)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Web sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract IOCs\n", + "\n", + "Extract Indicators of Compromise (IOCs) from threat feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "\n", + "all_threat_texts = []\n", + "if parsed_xml and parsed_xml.elements:\n", + " for elem in parsed_xml.elements:\n", + " if hasattr(elem, 'text') and elem.text:\n", + " all_threat_texts.append(elem.text)\n", + "\n", + "for db_record in parsed_db.get(\"data\", threat_db_data):\n", + " threat_text = f\"IOC: {db_record.get('ioc', '')} Type: {db_record.get('type', '')} Severity: {db_record.get('severity', '')}\"\n", + " all_threat_texts.append(threat_text)\n", + "\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_events = []\n", + "\n", + "for text in all_threat_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " events = event_detector.detect_events(text)\n", + " all_events.extend(events)\n", + "\n", + "print(f\"Extracted {len(all_entities)} IOCs\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Detected {len(all_events)} events\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Temporal Knowledge Graph\n", + "\n", + "Build a temporal knowledge graph from extracted IOCs and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "threat_entities = []\n", + "for i, entity in enumerate(all_entities[:10], 1):\n", + " threat_entities.append({\n", + " \"id\": f\"ioc_{i}\",\n", + " \"type\": entity.get(\"type\", \"IOC\"),\n", + " \"name\": entity.get(\"text\", entity.get(\"entity\", \"\")),\n", + " \"properties\": {\"timestamp\": datetime.now().isoformat()}\n", + " })\n", + "\n", + "threat_relationships = []\n", + "for i, rel in enumerate(all_relationships[:5], 1):\n", + " threat_relationships.append({\n", + " \"source\": f\"ioc_{i}\",\n", + " \"target\": f\"ioc_{i+1}\",\n", + " \"type\": rel.get(\"type\", \"related_to\"),\n", + " \"properties\": {\"timestamp\": datetime.now().isoformat()}\n", + " })\n", + "\n", + "threat_kg = builder.build(threat_entities, threat_relationships)\n", + "\n", + "print(f\"Built temporal knowledge graph\")\n", + "print(f\" Entities: {len(threat_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(threat_kg.get('relationships', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Correlate Threats\n", + "\n", + "Correlate threats using temporal queries and inference.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temporal_query = TemporalGraphQuery()\n", + "pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "inference_engine = InferenceEngine()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "start_time = (datetime.now() - timedelta(days=7)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=threat_kg,\n", + " query=\"Find threats in the last 7 days\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "patterns = pattern_detector.detect_temporal_patterns(\n", + " threat_kg,\n", + " pattern_type=\"sequence\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "connectivity = connectivity_analyzer.analyze_connectivity(threat_kg)\n", + "\n", + "inference_engine.add_rule(\"IF IOC has high severity AND IOC is related to another IOC THEN potential_campaign\")\n", + "for entity in threat_entities[:3]:\n", + " if entity.get(\"properties\", {}).get(\"severity\") == \"high\":\n", + " inference_engine.add_fact({\"ioc\": entity.get(\"id\"), \"severity\": \"high\"})\n", + "\n", + "correlations = inference_engine.forward_chain()\n", + "\n", + "print(f\"Temporal query returned {len(temporal_results.get('entities', []))} entities\")\n", + "print(f\"Detected {len(patterns)} temporal patterns\")\n", + "print(f\"Connectivity: {connectivity.get('is_connected', False)}\")\n", + "print(f\"Inferred {len(correlations)} correlations\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Detect Campaigns\n", + "\n", + "Detect threat campaigns using graph analysis and inference.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "campaigns = []\n", + "\n", + "if len(patterns) > 0:\n", + " campaigns.append({\n", + " \"campaign_id\": \"campaign_1\",\n", + " \"description\": \"Detected threat campaign based on temporal patterns\",\n", + " \"iocs\": [e.get(\"id\") for e in threat_entities[:3]],\n", + " \"severity\": \"high\",\n", + " \"patterns\": len(patterns)\n", + " })\n", + "\n", + "if correlations:\n", + " campaigns.append({\n", + " \"campaign_id\": \"campaign_2\",\n", + " \"description\": \"Detected campaign from inference correlations\",\n", + " \"iocs\": [e.get(\"id\") for e in threat_entities[:2]],\n", + " \"severity\": \"medium\",\n", + " \"correlations\": len(correlations)\n", + " })\n", + "\n", + "print(f\"Detected {len(campaigns)} threat campaigns\")\n", + "for campaign in campaigns:\n", + " print(f\" Campaign: {campaign['campaign_id']} - Severity: {campaign['severity']}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Quality Assessment and Provenance\n", + "\n", + "Assess graph quality and track provenance.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "provenance_tracker = ProvenanceTracker()\n", + "conflict_detector = ConflictDetector()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(threat_kg)\n", + "\n", + "for entity in threat_entities:\n", + " provenance_tracker.track_entity(entity.get(\"id\"), \"threat_feed\", entity)\n", + "\n", + "conflicts = conflict_detector.detect_value_conflicts(threat_entities, \"name\")\n", + "\n", + "print(f\"Graph quality score: {quality_score.get('overall_score', 0):.3f}\")\n", + "print(f\"Tracked provenance for {len(threat_entities)} entities\")\n", + "print(f\"Detected {len(conflicts)} conflicts\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Generate Reports\n", + "\n", + "Generate threat intelligence reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "rdf_exporter.export_knowledge_graph(threat_kg, os.path.join(temp_dir, \"threats.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Threat correlation analysis detected {len(campaigns)} campaigns\",\n", + " \"iocs\": len(threat_entities),\n", + " \"campaigns\": campaigns,\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"Generated threat intelligence report\")\n", + "print(f\"Report length: {len(report)} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Visualize Results\n", + "\n", + "Visualize threat correlation results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "analytics_visualizer = AnalyticsVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "analytics_viz = analytics_visualizer.visualize_analytics(threat_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(threat_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated analytics and temporal visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Multi-source ingestion → Extraction → Temporal KG → Correlation → Campaign Detection → Quality → Reports → Visualization\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/cybersecurity/Threat_Intelligence_Hybrid_RAG.ipynb b/docs/cookbook/use_cases/cybersecurity/Threat_Intelligence_Hybrid_RAG.ipynb new file mode 100644 index 00000000..36b14a6b --- /dev/null +++ b/docs/cookbook/use_cases/cybersecurity/Threat_Intelligence_Hybrid_RAG.ipynb @@ -0,0 +1,585 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Threat Intelligence Hybrid RAG Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete threat intelligence hybrid RAG pipeline: ingest threat intelligence from multiple sources (files, web, feeds), extract threat entities, build knowledge graph, generate embeddings, set up hybrid search (vector + temporal KG), and query threats using advanced RAG.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, WebIngestor, FeedIngestor, DBIngestor, MCPIngestor\n", + "- **Parsing**: JSONParser, XMLParser, HTMLParser, DocumentParser, MCPParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder\n", + "- **Vector Store**: VectorStore, HybridSearch\n", + "- **Context**: ContextRetriever, ContextGraphBuilder\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Export**: JSONExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Multi-Source Threat Intel (Files, Web, Feeds, MCP) → Parse → Extract Entities → Build KG → Generate Embeddings → Vector Store → Hybrid RAG Setup → Query Threats → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Multi-Source Threat Intelligence Ingestion\n", + "\n", + "Ingest threat intelligence from files, web sources, and feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, FeedIngestor, DBIngestor, MCPIngestor, ingest_mcp\n", + "from semantica.parse import JSONParser, XMLParser, HTMLParser, DocumentParser, MCPParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.vector_store import VectorStore, HybridSearch\n", + "from semantica.context import ContextRetriever, ContextGraphBuilder\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "db_ingestor = DBIngestor()\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "xml_parser = XMLParser()\n", + "html_parser = HTMLParser()\n", + "document_parser = DocumentParser()\n", + "mcp_parser = MCPParser()\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Real-world threat intelligence formats\n", + "threat_intel_json = os.path.join(temp_dir, \"threat_intel.json\")\n", + "threat_data = [\n", + " {\n", + " \"threat_id\": \"APT-001\",\n", + " \"name\": \"Advanced Persistent Threat Group 1\",\n", + " \"description\": \"State-sponsored APT group targeting financial institutions\",\n", + " \"iocs\": [\"192.168.1.100\", \"malicious-domain.com\", \"hash_abc123\"],\n", + " \"tactics\": [\"initial_access\", \"persistence\", \"exfiltration\"],\n", + " \"timestamp\": (datetime.now() - timedelta(days=7)).isoformat(),\n", + " \"severity\": \"high\"\n", + " },\n", + " {\n", + " \"threat_id\": \"APT-002\",\n", + " \"name\": \"Ransomware Campaign\",\n", + " \"description\": \"Large-scale ransomware campaign targeting healthcare sector\",\n", + " \"iocs\": [\"198.51.100.50\", \"ransomware-domain.net\", \"hash_def456\"],\n", + " \"tactics\": [\"initial_access\", \"execution\", \"impact\"],\n", + " \"timestamp\": (datetime.now() - timedelta(days=3)).isoformat(),\n", + " \"severity\": \"critical\"\n", + " },\n", + " {\n", + " \"threat_id\": \"APT-003\",\n", + " \"name\": \"Phishing Campaign\",\n", + " \"description\": \"Sophisticated phishing campaign using social engineering\",\n", + " \"iocs\": [\"203.0.113.75\", \"phishing-site.org\", \"hash_ghi789\"],\n", + " \"tactics\": [\"initial_access\", \"collection\"],\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat(),\n", + " \"severity\": \"medium\"\n", + " }\n", + "]\n", + "\n", + "with open(threat_intel_json, 'w') as f:\n", + " json.dump(threat_data, f, indent=2)\n", + "\n", + "# XML format threat intelligence (STIX format)\n", + "threat_intel_xml = os.path.join(temp_dir, \"threat_intel.xml\")\n", + "xml_content = \"\"\"\n", + "\n", + " \n", + " IOC-001\n", + " IP\n", + " 172.16.0.50\n", + " malware\n", + " 2024-01-15T10:00:00\n", + " \n", + " \n", + " IOC-002\n", + " Domain\n", + " suspicious-domain.com\n", + " phishing\n", + " 2024-01-15T11:00:00\n", + " \n", + "\"\"\"\n", + "\n", + "with open(threat_intel_xml, 'w') as f:\n", + " f.write(xml_content)\n", + "\n", + "# Ingest from files\n", + "file_objects_json = file_ingestor.ingest_file(threat_intel_json, read_content=True)\n", + "file_objects_xml = file_ingestor.ingest_file(threat_intel_xml, read_content=True)\n", + "\n", + "# Parse threat intelligence\n", + "parsed_json = json_parser.parse(threat_intel_json)\n", + "parsed_xml = xml_parser.parse(threat_intel_xml)\n", + "\n", + "# Real threat intelligence feed URLs\n", + "threat_intel_feeds = [\n", + " \"https://www.cisa.gov/news.xml\", # CISA Security Advisories\n", + " \"https://www.us-cert.gov/ncas/alerts.xml\", # US-CERT Alerts\n", + " \"https://feeds.feedburner.com/SecurityWeek\", # Security Week\n", + " \"https://www.darkreading.com/rss.xml\", # Dark Reading\n", + " \"https://krebsonsecurity.com/feed/\" # Krebs on Security\n", + "]\n", + "\n", + "threat_feed_list = []\n", + "for feed_url in threat_intel_feeds:\n", + " try:\n", + " threat_feed = feed_ingestor.ingest_feed(feed_url)\n", + " if threat_feed:\n", + " threat_feed_list.append(threat_feed)\n", + " print(f\"✓ Ingested threat feed: {threat_feed.title if hasattr(threat_feed, 'title') else feed_url}\")\n", + " print(f\" Items: {len(threat_feed.items) if hasattr(threat_feed, 'items') else 0}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Real web sources for threat intelligence\n", + "threat_web_sources = [\n", + " \"https://api.github.com/repos/mitre/cti\", # MITRE ATT&CK Framework\n", + " \"https://www.cisa.gov/known-exploited-vulnerabilities-catalog\", # CISA KEV Catalog\n", + " \"https://nvd.nist.gov/vuln/search\" # NIST NVD\n", + "]\n", + "\n", + "web_content_list = []\n", + "for web_url in threat_web_sources[:1]: # Process first URL\n", + " try:\n", + " web_content = web_ingestor.ingest_url(web_url)\n", + " if web_content:\n", + " web_content_list.append(web_content)\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else web_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Web ingestion for {web_url}: {str(e)[:100]}\")\n", + "\n", + "# Optional: Ingest from MCP server\n", + "# Users can bring their own threat intelligence MCP server via URL\n", + "mcp_threat_data = []\n", + "try:\n", + " # Connect to threat intelligence MCP server via URL\n", + " # Example: http://localhost:8000/mcp or https://api.example.com/threat-mcp\n", + " threat_mcp_url = \"http://localhost:8000/mcp\" # Replace with your MCP server URL\n", + " \n", + " mcp_ingestor.connect(\n", + " \"threat_mcp_server\",\n", + " url=threat_mcp_url,\n", + " headers={\n", + " \"Authorization\": \"Bearer your_token\",\n", + " \"X-API-Key\": \"your_api_key\"\n", + " } if \"api.example.com\" in threat_mcp_url else {}\n", + " )\n", + " \n", + " # Ingest threat indicators from MCP server\n", + " mcp_data = mcp_ingestor.ingest_resources(\n", + " \"threat_mcp_server\",\n", + " resource_uris=[\"resource://threats/feed\", \"resource://vulnerabilities/database\"]\n", + " )\n", + " mcp_threat_data.extend(mcp_data)\n", + " \n", + " # Or use tool-based ingestion to query threat indicators\n", + " tool_data = mcp_ingestor.ingest_tool_output(\n", + " \"threat_mcp_server\",\n", + " tool_name=\"query_threat_indicators\",\n", + " arguments={\n", + " \"indicator_type\": \"IP\",\n", + " \"date_range\": {\n", + " \"start\": (datetime.now() - timedelta(days=7)).isoformat(),\n", + " \"end\": datetime.now().isoformat()\n", + " }\n", + " }\n", + " )\n", + " if tool_data:\n", + " mcp_threat_data.append(tool_data)\n", + " \n", + " # Parse MCP responses and merge with existing threat data\n", + " for mcp_item in mcp_threat_data:\n", + " parsed_mcp = mcp_parser.parse_response(mcp_item, response_type=\"json\")\n", + " if isinstance(parsed_mcp, dict):\n", + " if \"threat_indicators\" in parsed_mcp:\n", + " # Merge threat indicators from MCP\n", + " if parsed_json and parsed_json.data:\n", + " parsed_json.data.extend(parsed_mcp.get(\"threat_indicators\", []))\n", + " else:\n", + " parsed_json.data = parsed_mcp.get(\"threat_indicators\", [])\n", + " elif \"threat_id\" in parsed_mcp:\n", + " # Single threat indicator\n", + " if parsed_json and parsed_json.data:\n", + " parsed_json.data.append(parsed_mcp)\n", + " else:\n", + " parsed_json.data = [parsed_mcp]\n", + " \n", + " print(f\"✓ Ingested {len(mcp_threat_data)} items from MCP server\")\n", + " mcp_ingestor.disconnect(\"threat_mcp_server\")\n", + "except Exception as e:\n", + " print(f\"⚠ MCP ingestion skipped: {e}\")\n", + " print(\" Note: MCP ingestion is optional. You can bring your own MCP server via URL.\")\n", + "\n", + "print(f\"Ingested {len([file_objects_json]) if file_objects_json else 0} JSON threat intelligence files\")\n", + "print(f\"Ingested {len([file_objects_xml]) if file_objects_xml else 0} XML threat intelligence files\")\n", + "print(f\"Parsed {len(parsed_json.data) if parsed_json and parsed_json.data else 0} JSON threat entries\")\n", + "print(f\"Parsed {len(parsed_xml.elements) if parsed_xml else 0} XML indicator elements\")\n", + "print(f\"MCP server sources: {len(mcp_threat_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Threat Intelligence Entities\n", + "\n", + "Extract threat entities, IOCs, and relationships from threat intelligence data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "threat_entities = []\n", + "threat_relationships = []\n", + "all_documents = []\n", + "\n", + "# Extract from JSON threat intelligence\n", + "if parsed_json and parsed_json.data:\n", + " for threat in parsed_json.data:\n", + " if isinstance(threat, dict):\n", + " threat_text = f\"{threat.get('name', '')}: {threat.get('description', '')}\"\n", + " all_documents.append(threat_text)\n", + " \n", + " threat_entities.append({\n", + " \"id\": threat.get(\"threat_id\", \"\"),\n", + " \"type\": \"Threat_Actor\",\n", + " \"name\": threat.get(\"name\", \"\"),\n", + " \"properties\": {\n", + " \"description\": threat.get(\"description\", \"\"),\n", + " \"severity\": threat.get(\"severity\", \"\"),\n", + " \"timestamp\": threat.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " for ioc in threat.get(\"iocs\", []):\n", + " threat_entities.append({\n", + " \"id\": ioc,\n", + " \"type\": \"IOC\",\n", + " \"name\": ioc,\n", + " \"properties\": {\n", + " \"threat_id\": threat.get(\"threat_id\", \"\"),\n", + " \"timestamp\": threat.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " threat_relationships.append({\n", + " \"source\": threat.get(\"threat_id\", \"\"),\n", + " \"target\": ioc,\n", + " \"type\": \"uses\",\n", + " \"properties\": {\"timestamp\": threat.get(\"timestamp\", \"\")}\n", + " })\n", + " \n", + " for tactic in threat.get(\"tactics\", []):\n", + " threat_entities.append({\n", + " \"id\": tactic,\n", + " \"type\": \"Tactic\",\n", + " \"name\": tactic,\n", + " \"properties\": {}\n", + " })\n", + " threat_relationships.append({\n", + " \"source\": threat.get(\"threat_id\", \"\"),\n", + " \"target\": tactic,\n", + " \"type\": \"employs\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "# Extract from XML indicators\n", + "if parsed_xml and parsed_xml.elements:\n", + " for elem in parsed_xml.elements:\n", + " if hasattr(elem, 'text') and elem.text:\n", + " entities = ner_extractor.extract(elem.text)\n", + " threat_entities.extend(entities)\n", + "\n", + "print(f\"Extracted {len(threat_entities)} threat intelligence entities\")\n", + "print(f\"Extracted {len(threat_relationships)} threat relationships\")\n", + "print(f\"Collected {len(all_documents)} threat intelligence documents\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Threat Intelligence Knowledge Graph\n", + "\n", + "Build knowledge graph from threat entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "threat_kg = builder.build(threat_entities, threat_relationships)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(threat_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(threat_kg)\n", + "\n", + "print(f\"Built threat intelligence knowledge graph\")\n", + "print(f\" Entities: {len(threat_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(threat_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Connected components: {len(connectivity.get('components', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Generate Embeddings and Setup Vector Store\n", + "\n", + "Generate embeddings from threat intelligence documents and store in vector database.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "vector_store = VectorStore()\n", + "hybrid_search = HybridSearch()\n", + "\n", + "# Generate embeddings for threat intelligence documents\n", + "embeddings = embedding_generator.generate(all_documents)\n", + "\n", + "# Prepare metadata for vector store\n", + "metadata = []\n", + "for i, doc in enumerate(all_documents):\n", + " metadata.append({\n", + " \"id\": f\"doc_{i}\",\n", + " \"text\": doc,\n", + " \"source\": \"threat_intelligence\"\n", + " })\n", + "\n", + "# Store vectors\n", + "vector_ids = vector_store.store_vectors(embeddings, metadata)\n", + "\n", + "print(f\"Generated embeddings for {len(all_documents)} documents\")\n", + "print(f\"Embedding dimension: {len(embeddings[0]) if embeddings else 0}\")\n", + "print(f\"Stored {len(vector_ids)} vectors in vector store\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Setup Hybrid RAG (Vector + Temporal KG)\n", + "\n", + "Setup hybrid search combining vector similarity and temporal knowledge graph queries.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "context_retriever = ContextRetriever()\n", + "context_graph_builder = ContextGraphBuilder()\n", + "\n", + "# Setup context retriever with KG and vector store\n", + "context_retriever = ContextRetriever(\n", + " knowledge_graph=threat_kg,\n", + " vector_store=vector_store\n", + ")\n", + "\n", + "print(\"Hybrid RAG setup complete\")\n", + "print(f\" Knowledge graph: {len(threat_kg.get('entities', []))} entities\")\n", + "print(f\" Vector store: {len(vector_ids)} vectors\")\n", + "print(f\" Context retriever initialized\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Query Threats Using Hybrid RAG\n", + "\n", + "Query threat intelligence using hybrid search (vector + temporal KG).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query examples\n", + "queries = [\n", + " \"What are the latest APT threats?\",\n", + " \"Find threats targeting financial institutions\",\n", + " \"What IOCs are associated with ransomware?\"\n", + "]\n", + "\n", + "query_results = []\n", + "\n", + "for query in queries:\n", + " # Generate query embedding\n", + " query_embedding = text_embedder.embed_text(query)\n", + " \n", + " # Vector search\n", + " vector_results = vector_store.search_vectors(query_embedding, k=3)\n", + " \n", + " # Temporal KG query\n", + " start_time = (datetime.now() - timedelta(days=30)).isoformat()\n", + " end_time = datetime.now().isoformat()\n", + " \n", + " temporal_results = temporal_query.query_time_range(\n", + " graph=threat_kg,\n", + " query=query,\n", + " start_time=start_time,\n", + " end_time=end_time\n", + " )\n", + " \n", + " # Hybrid search using context retriever\n", + " context_results = context_retriever.retrieve(\n", + " query=query,\n", + " top_k=3,\n", + " use_graph_expansion=True\n", + " )\n", + " \n", + " query_results.append({\n", + " \"query\": query,\n", + " \"vector_results\": len(vector_results),\n", + " \"temporal_results\": len(temporal_results.get('entities', [])),\n", + " \"context_results\": len(context_results) if context_results else 0\n", + " })\n", + "\n", + "# Inference for threat analysis\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "inference_engine.add_rule(\"IF severity is critical AND tactics includes exfiltration THEN high_priority_threat\")\n", + "inference_engine.add_rule(\"IF threat targets financial AND uses initial_access THEN financial_apt\")\n", + "\n", + "for threat in parsed_json.data if parsed_json and parsed_json.data else []:\n", + " if isinstance(threat, dict):\n", + " inference_engine.add_fact({\n", + " \"threat_id\": threat.get(\"threat_id\", \"\"),\n", + " \"severity\": threat.get(\"severity\", \"\"),\n", + " \"tactics\": threat.get(\"tactics\", [])\n", + " })\n", + "\n", + "threat_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Processed {len(queries)} threat intelligence queries\")\n", + "for result in query_results:\n", + " print(f\" Query: '{result['query']}' - Vector: {result['vector_results']}, Temporal: {result['temporal_results']}, Context: {result['context_results']}\")\n", + "print(f\"Generated {len(threat_insights)} threat insights from inference\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(threat_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(threat_kg, os.path.join(temp_dir, \"threat_kg.json\"))\n", + "rdf_exporter.export_knowledge_graph(threat_kg, os.path.join(temp_dir, \"threat_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Threat intelligence analysis identified {len(threat_entities)} entities and {len(threat_insights)} insights\",\n", + " \"threats_analyzed\": len(parsed_json.data) if parsed_json and parsed_json.data else 0,\n", + " \"iocs\": len([e for e in threat_entities if e.get(\"type\") == \"IOC\"]),\n", + " \"insights\": len(threat_insights),\n", + " \"quality_score\": quality_score.get('overall_score', 0),\n", + " \"critical_threats\": len([t for t in parsed_json.data if isinstance(t, dict) and t.get(\"severity\") == \"critical\"]) if parsed_json and parsed_json.data else 0\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"Generated threat intelligence report\")\n", + "print(f\"Report length: {len(report)} characters\")\n", + "print(f\"Graph quality score: {quality_score.get('overall_score', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Visualize Threat Intelligence\n", + "\n", + "Visualize threat intelligence knowledge graph and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(threat_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(threat_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(threat_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated visualizations for threat intelligence knowledge graph, temporal patterns, and analytics\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Multi-Source Threat Intel → Parse → Extract → Build KG → Embeddings → Vector Store → Hybrid RAG → Query → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/cybersecurity/Threat_Intelligence_Integration.ipynb b/docs/cookbook/use_cases/cybersecurity/Threat_Intelligence_Integration.ipynb new file mode 100644 index 00000000..b4e9055d --- /dev/null +++ b/docs/cookbook/use_cases/cybersecurity/Threat_Intelligence_Integration.ipynb @@ -0,0 +1,575 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Threat Intelligence Integration Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to integrate Python/FastMCP MCP servers as data sources for threat intelligence ingestion. Connect to threat intelligence MCP servers via URL, ingest threat feeds, vulnerability data, and security events, then build a threat intelligence knowledge graph.\n", + "\n", + "**IMPORTANT**: This implementation supports ONLY Python-based MCP servers and FastMCP servers. Users can bring their own Python/FastMCP MCP servers via URL connections.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: MCPIngestor, ingest_mcp, WebIngestor, FeedIngestor\n", + "- **Parsing**: MCPParser, JSONParser, XMLParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder\n", + "- **Vector Store**: VectorStore, HybridSearch\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Export**: JSONExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Connect to Threat Intel MCP Server → Ingest Threat Data via MCP → Parse MCP Responses → Extract Threat Entities → Build Threat KG → Generate Embeddings → Hybrid RAG → Analyze Threats → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Connect to Threat Intelligence MCP Server\n", + "\n", + "Connect to a Python/FastMCP MCP server that provides threat intelligence data via URL. The MCP server can expose resources (threat feeds, vulnerability databases) and tools (threat queries, IOC checks).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import MCPIngestor, ingest_mcp\n", + "from semantica.parse import MCPParser, JSONParser, XMLParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.vector_store import VectorStore, HybridSearch\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Initialize MCP ingestor\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "# Connect to threat intelligence MCP server via URL\n", + "# Replace with your actual MCP server URL\n", + "# Example: http://localhost:8000/mcp or https://api.example.com/threat-mcp\n", + "threat_mcp_url = \"http://localhost:8000/mcp\"\n", + "\n", + "try:\n", + " # Connect to MCP server with authentication (if required)\n", + " mcp_ingestor.connect(\n", + " \"threat_server\",\n", + " url=threat_mcp_url,\n", + " headers={\n", + " \"Authorization\": \"Bearer your_token\",\n", + " \"X-API-Key\": \"your_api_key\"\n", + " } if \"api.example.com\" in threat_mcp_url else {}\n", + " )\n", + " print(f\"✓ Connected to threat intelligence MCP server at {threat_mcp_url}\")\n", + " \n", + " # List available resources (threat feeds, vulnerability databases)\n", + " resources = mcp_ingestor.list_available_resources(\"threat_server\")\n", + " print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + " for resource in resources[:5]: # Show first 5\n", + " print(f\" - {resource.uri}: {resource.name}\")\n", + " if resource.description:\n", + " print(f\" {resource.description[:80]}...\")\n", + " \n", + " # List available tools (threat queries, IOC checks)\n", + " tools = mcp_ingestor.list_available_tools(\"threat_server\")\n", + " print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + " for tool in tools[:5]: # Show first 5\n", + " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Connection failed: {e}\")\n", + " print(\"Note: This example uses a placeholder URL. Replace with your actual MCP server URL.\")\n", + " print(\"For testing, you can use a mock MCP server or skip connection and use sample data below.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Threat Intelligence Data from MCP Server\n", + "\n", + "Ingest threat feeds, vulnerability data, and security events using both resource-based and tool-based methods.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize parsers\n", + "mcp_parser = MCPParser()\n", + "json_parser = JSONParser()\n", + "xml_parser = XMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "threat_data = []\n", + "\n", + "# Method 1: Resource-based ingestion\n", + "# Ingest from MCP resources (threat feeds, vulnerability databases)\n", + "try:\n", + " # Example: Ingest threat feed resource\n", + " threat_feeds = mcp_ingestor.ingest_resources(\n", + " \"threat_server\",\n", + " resource_uris=[\"resource://threats/feed\", \"resource://vulnerabilities/database\"]\n", + " )\n", + " \n", + " for item in threat_feeds:\n", + " threat_data.append(item)\n", + " print(f\"✓ Ingested resource: {item.resource_uri}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Resource ingestion: {e}\")\n", + "\n", + "# Method 2: Tool-based ingestion\n", + "# Call MCP tools to retrieve data dynamically\n", + "try:\n", + " # Example: Query threat indicators\n", + " threat_indicators = mcp_ingestor.ingest_tool_output(\n", + " \"threat_server\",\n", + " tool_name=\"query_threat_indicators\",\n", + " arguments={\n", + " \"indicator_type\": \"IP\",\n", + " \"date_range\": {\n", + " \"start\": (datetime.now() - timedelta(days=7)).isoformat(),\n", + " \"end\": datetime.now().isoformat()\n", + " }\n", + " }\n", + " )\n", + " \n", + " if threat_indicators:\n", + " threat_data.append(threat_indicators)\n", + " print(f\"✓ Retrieved threat indicators via tool\")\n", + " \n", + " # Example: Check IOC (Indicators of Compromise)\n", + " ioc_check = mcp_ingestor.ingest_tool_output(\n", + " \"threat_server\",\n", + " tool_name=\"check_ioc\",\n", + " arguments={\n", + " \"ioc_type\": \"hash\",\n", + " \"ioc_value\": \"abc123def456\"\n", + " }\n", + " )\n", + " \n", + " if ioc_check:\n", + " threat_data.append(ioc_check)\n", + " print(f\"✓ Retrieved IOC check results via tool\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Tool-based ingestion: {e}\")\n", + " print(\"Note: Using sample data for demonstration\")\n", + "\n", + "# Sample threat intelligence data (if MCP server is not available)\n", + "if not threat_data:\n", + " print(\"\\n📝 Using sample threat intelligence data for demonstration:\")\n", + " sample_data = {\n", + " \"threat_indicators\": [\n", + " {\n", + " \"indicator_id\": \"TI001\",\n", + " \"indicator_type\": \"IP\",\n", + " \"indicator_value\": \"192.168.1.100\",\n", + " \"threat_type\": \"malware\",\n", + " \"severity\": \"high\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat(),\n", + " \"source\": \"ThreatFeed1\"\n", + " },\n", + " {\n", + " \"indicator_id\": \"TI002\",\n", + " \"indicator_type\": \"domain\",\n", + " \"indicator_value\": \"malicious.example.com\",\n", + " \"threat_type\": \"phishing\",\n", + " \"severity\": \"medium\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=12)).isoformat(),\n", + " \"source\": \"ThreatFeed2\"\n", + " },\n", + " {\n", + " \"indicator_id\": \"TI003\",\n", + " \"indicator_type\": \"hash\",\n", + " \"indicator_value\": \"abc123def456\",\n", + " \"threat_type\": \"ransomware\",\n", + " \"severity\": \"critical\",\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"source\": \"ThreatFeed1\"\n", + " }\n", + " ],\n", + " \"vulnerabilities\": [\n", + " {\n", + " \"cve_id\": \"CVE-2024-0001\",\n", + " \"description\": \"Remote code execution vulnerability\",\n", + " \"severity\": \"critical\",\n", + " \"affected_products\": [\"Product A\", \"Product B\"],\n", + " \"published_date\": (datetime.now() - timedelta(days=5)).isoformat()\n", + " }\n", + " ]\n", + " }\n", + " threat_data.append(sample_data)\n", + " print(f\" Loaded {len(sample_data['threat_indicators'])} threat indicators\")\n", + " print(f\" Loaded {len(sample_data['vulnerabilities'])} vulnerabilities\")\n", + "\n", + "print(f\"\\n📊 Total threat intelligence data items ingested: {len(threat_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Parse Threat Intelligence Data\n", + "\n", + "Parse the threat intelligence data received from MCP server responses.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parsed_threat_data = []\n", + "\n", + "# Parse MCP responses\n", + "for data_item in threat_data:\n", + " try:\n", + " # Parse MCP response (handles JSON, XML, text, binary)\n", + " if isinstance(data_item, dict):\n", + " parsed_item = data_item\n", + " else:\n", + " parsed_item = mcp_parser.parse_response(data_item, response_type=\"json\")\n", + " \n", + " parsed_threat_data.append(parsed_item)\n", + " \n", + " except Exception as e:\n", + " print(f\"⚠ Parsing error: {e}\")\n", + "\n", + "# Extract threat indicators and vulnerabilities\n", + "threat_indicators = []\n", + "vulnerabilities = []\n", + "\n", + "for item in parsed_threat_data:\n", + " if isinstance(item, dict):\n", + " if \"threat_indicators\" in item:\n", + " threat_indicators.extend(item[\"threat_indicators\"])\n", + " elif \"indicator_id\" in item:\n", + " threat_indicators.append(item)\n", + " elif \"vulnerabilities\" in item:\n", + " vulnerabilities.extend(item[\"vulnerabilities\"])\n", + " elif \"cve_id\" in item:\n", + " vulnerabilities.append(item)\n", + "\n", + "print(f\"✓ Parsed {len(parsed_threat_data)} data items\")\n", + "print(f\"✓ Extracted {len(threat_indicators)} threat indicators\")\n", + "print(f\"✓ Extracted {len(vulnerabilities)} vulnerabilities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Threat Entities and Relationships\n", + "\n", + "Extract threat entities (indicators, vulnerabilities, threat actors) and relationships from MCP data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "threat_entities = []\n", + "threat_relationships = []\n", + "\n", + "# Extract from threat indicators\n", + "for indicator in threat_indicators:\n", + " if isinstance(indicator, dict):\n", + " indicator_id = indicator.get(\"indicator_id\", \"\")\n", + " indicator_type = indicator.get(\"indicator_type\", \"\")\n", + " threat_type = indicator.get(\"threat_type\", \"\")\n", + " source = indicator.get(\"source\", \"\")\n", + " \n", + " # Threat Indicator entity\n", + " threat_entities.append({\n", + " \"id\": indicator_id,\n", + " \"type\": \"ThreatIndicator\",\n", + " \"name\": indicator_id,\n", + " \"properties\": {\n", + " \"indicator_type\": indicator_type,\n", + " \"indicator_value\": indicator.get(\"indicator_value\", \"\"),\n", + " \"threat_type\": threat_type,\n", + " \"severity\": indicator.get(\"severity\", \"\"),\n", + " \"timestamp\": indicator.get(\"timestamp\", \"\"),\n", + " \"source\": source\n", + " }\n", + " })\n", + " \n", + " # Threat Type entity\n", + " if threat_type:\n", + " threat_entities.append({\n", + " \"id\": threat_type,\n", + " \"type\": \"ThreatType\",\n", + " \"name\": threat_type,\n", + " \"properties\": {}\n", + " })\n", + " threat_relationships.append({\n", + " \"source\": indicator_id,\n", + " \"target\": threat_type,\n", + " \"type\": \"classified_as\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Source entity\n", + " if source:\n", + " threat_entities.append({\n", + " \"id\": source,\n", + " \"type\": \"ThreatSource\",\n", + " \"name\": source,\n", + " \"properties\": {}\n", + " })\n", + " threat_relationships.append({\n", + " \"source\": indicator_id,\n", + " \"target\": source,\n", + " \"type\": \"reported_by\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "# Extract from vulnerabilities\n", + "for vuln in vulnerabilities:\n", + " if isinstance(vuln, dict):\n", + " cve_id = vuln.get(\"cve_id\", \"\")\n", + " \n", + " # Vulnerability entity\n", + " threat_entities.append({\n", + " \"id\": cve_id,\n", + " \"type\": \"Vulnerability\",\n", + " \"name\": cve_id,\n", + " \"properties\": {\n", + " \"description\": vuln.get(\"description\", \"\"),\n", + " \"severity\": vuln.get(\"severity\", \"\"),\n", + " \"published_date\": vuln.get(\"published_date\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Affected products\n", + " for product in vuln.get(\"affected_products\", []):\n", + " threat_entities.append({\n", + " \"id\": product,\n", + " \"type\": \"Product\",\n", + " \"name\": product,\n", + " \"properties\": {}\n", + " })\n", + " threat_relationships.append({\n", + " \"source\": cve_id,\n", + " \"target\": product,\n", + " \"type\": \"affects\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "# Remove duplicates\n", + "seen_entities = set()\n", + "unique_entities = []\n", + "for entity in threat_entities:\n", + " entity_key = (entity[\"id\"], entity[\"type\"])\n", + " if entity_key not in seen_entities:\n", + " seen_entities.add(entity_key)\n", + " unique_entities.append(entity)\n", + "\n", + "threat_entities = unique_entities\n", + "\n", + "print(f\"✓ Extracted {len(threat_entities)} threat entities\")\n", + "print(f\" - Threat Indicators: {len([e for e in threat_entities if e['type'] == 'ThreatIndicator'])}\")\n", + "print(f\" - Vulnerabilities: {len([e for e in threat_entities if e['type'] == 'Vulnerability'])}\")\n", + "print(f\" - Threat Types: {len([e for e in threat_entities if e['type'] == 'ThreatType'])}\")\n", + "print(f\" - Sources: {len([e for e in threat_entities if e['type'] == 'ThreatSource'])}\")\n", + "print(f\"✓ Extracted {len(threat_relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Threat Intelligence Knowledge Graph\n", + "\n", + "Build a temporal knowledge graph from the extracted threat entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "# Build knowledge graph\n", + "threat_kg = builder.build(threat_entities, threat_relationships)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(threat_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(threat_kg)\n", + "\n", + "print(f\"✓ Built threat intelligence knowledge graph\")\n", + "print(f\" Entities: {len(threat_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(threat_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Connectivity: {connectivity.get('connected_components', 0)} components\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Generate Embeddings and Set Up Hybrid RAG\n", + "\n", + "Generate embeddings for threat intelligence data and set up hybrid search (vector + temporal KG).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate embeddings\n", + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "\n", + "# Generate embeddings for threat entities\n", + "threat_texts = []\n", + "for entity in threat_entities:\n", + " if entity.get(\"type\") == \"ThreatIndicator\":\n", + " text = f\"{entity.get('properties', {}).get('indicator_value', '')} {entity.get('properties', {}).get('threat_type', '')} {entity.get('properties', {}).get('description', '')}\"\n", + " threat_texts.append(text)\n", + "\n", + "embeddings = embedding_generator.generate_embeddings(threat_texts)\n", + "\n", + "# Set up vector store\n", + "vector_store = VectorStore()\n", + "vector_store.add_embeddings(threat_texts, embeddings)\n", + "\n", + "# Set up hybrid search (vector + temporal KG)\n", + "hybrid_search = HybridSearch()\n", + "hybrid_search.setup(vector_store, threat_kg)\n", + "\n", + "print(f\"✓ Generated embeddings for {len(threat_texts)} threat indicators\")\n", + "print(f\"✓ Set up vector store with {len(embeddings)} embeddings\")\n", + "print(f\"✓ Configured hybrid search (vector + temporal KG)\")\n", + "\n", + "# Inference engine for threat analysis\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Threat analysis rules\n", + "inference_engine.add_rule(\"IF severity(critical) AND threat_type(ransomware) THEN immediate_response_required\")\n", + "inference_engine.add_rule(\"IF severity(high) AND indicator_type(IP) THEN block_ip\")\n", + "\n", + "# Add facts from threat data\n", + "for indicator in threat_indicators:\n", + " if isinstance(indicator, dict):\n", + " inference_engine.add_fact({\n", + " \"indicator_id\": indicator.get(\"indicator_id\", \"\"),\n", + " \"severity\": indicator.get(\"severity\", \"\"),\n", + " \"threat_type\": indicator.get(\"threat_type\", \"\"),\n", + " \"indicator_type\": indicator.get(\"indicator_type\", \"\")\n", + " })\n", + "\n", + "# Generate threat insights\n", + "threat_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"✓ Threat analysis completed\")\n", + "print(f\" Threat insights: {len(threat_insights)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Export and Visualize\n", + "\n", + "Export the threat intelligence knowledge graph and generate visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "import os\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export knowledge graph\n", + "json_exporter.export_knowledge_graph(threat_kg, os.path.join(temp_dir, \"threat_kg.json\"))\n", + "rdf_exporter.export_knowledge_graph(threat_kg, os.path.join(temp_dir, \"threat_kg.rdf\"))\n", + "\n", + "# Generate report\n", + "report_data = {\n", + " \"summary\": f\"Threat intelligence integration from MCP server identified {len(threat_insights)} insights\",\n", + " \"threat_indicators\": len([e for e in threat_entities if e['type'] == 'ThreatIndicator']),\n", + " \"vulnerabilities\": len([e for e in threat_entities if e['type'] == 'Vulnerability']),\n", + " \"threat_types\": len([e for e in threat_entities if e['type'] == 'ThreatType']),\n", + " \"insights\": len(threat_insights)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"✓ Exported threat intelligence knowledge graph\")\n", + "print(f\" JSON: {os.path.join(temp_dir, 'threat_kg.json')}\")\n", + "print(f\" RDF: {os.path.join(temp_dir, 'threat_kg.rdf')}\")\n", + "print(f\"✓ Generated report ({len(report)} characters)\")\n", + "\n", + "# Visualize\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(threat_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(threat_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(threat_kg, output=\"interactive\")\n", + "\n", + "print(\"✓ Generated visualizations for threat intelligence knowledge graph\")\n", + "\n", + "# Cleanup: Disconnect from MCP server\n", + "try:\n", + " mcp_ingestor.disconnect(\"threat_server\")\n", + " print(\"\\n✓ Disconnected from MCP server\")\n", + "except:\n", + " pass\n", + "\n", + "print(f\"\\n✅ Pipeline complete: MCP Server → Ingest → Parse → Extract → Build KG → Embeddings → Hybrid RAG → Analyze → Export → Visualize\")\n", + "print(f\"📊 Total modules used: 20+\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/cybersecurity/Vulnerability_Tracking.ipynb b/docs/cookbook/use_cases/cybersecurity/Vulnerability_Tracking.ipynb new file mode 100644 index 00000000..23aedd33 --- /dev/null +++ b/docs/cookbook/use_cases/cybersecurity/Vulnerability_Tracking.ipynb @@ -0,0 +1,142 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vulnerability Tracking Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete vulnerability tracking pipeline: ingest CVE data from multiple real sources (NVD, CVE feeds, security databases), build temporal knowledge graph, correlate vulnerabilities, predict impact, and generate vulnerability reports.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, FeedIngestor, DBIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, XMLParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Real CVE Sources → Parse → Extract Vulnerabilities → Build Temporal KG → Correlate → Predict Impact → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest CVE Data from Real Sources\n", + "\n", + "Ingest CVE data from NVD, CVE feeds, and security databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, FeedIngestor, DBIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, XMLParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "db_ingestor = DBIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "xml_parser = XMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real CVE and vulnerability data sources\n", + "cve_sources = [\n", + " \"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-recent.json.zip\", # NVD Recent CVEs (JSON)\n", + " \"https://nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-recent.xml.zip\", # NVD Recent CVEs (XML)\n", + " \"https://cve.mitre.org/data/downloads/allitems.csv\", # CVE MITRE All Items\n", + " \"https://www.cisa.gov/known-exploited-vulnerabilities-catalog/json\" # CISA KEV Catalog\n", + "]\n", + "\n", + "# Real vulnerability feed URLs\n", + "vulnerability_feeds = [\n", + " \"https://www.cisa.gov/news.xml\", # CISA Security Advisories\n", + " \"https://www.us-cert.gov/ncas/alerts.xml\", # US-CERT Alerts\n", + " \"https://feeds.feedburner.com/SecurityWeek\", # Security Week\n", + " \"https://www.darkreading.com/rss.xml\" # Dark Reading\n", + "]\n", + "\n", + "# Real database connection for vulnerability tracking\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/vulnerability_db\"\n", + "db_query = \"SELECT cve_id, description, severity, published_date, affected_products FROM vulnerabilities WHERE published_date > NOW() - INTERVAL '30 days' ORDER BY published_date DESC\"\n", + "\n", + "# Real web API endpoints for CVE data\n", + "cve_apis = [\n", + " \"https://services.nvd.nist.gov/rest/json/cves/2.0\", # NVD CVE API v2.0\n", + " \"https://api.github.com/repos/CVEProject/cvelist\", # CVE Project on GitHub\n", + " \"https://cve.circl.lu/api/last\" # CVE Search API\n", + "]\n", + "\n", + "# Ingest from real CVE feeds\n", + "cve_feed_list = []\n", + "for feed_url in vulnerability_feeds:\n", + " try:\n", + " cve_feed = feed_ingestor.ingest_feed(feed_url)\n", + " if cve_feed:\n", + " cve_feed_list.append(cve_feed)\n", + " print(f\"✓ Ingested vulnerability feed: {cve_feed.title if hasattr(cve_feed, 'title') else feed_url}\")\n", + " print(f\" Items: {len(cve_feed.items) if hasattr(cve_feed, 'items') else 0}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Ingest from real CVE APIs\n", + "cve_api_data = []\n", + "for api_url in cve_apis[:1]: # Process first API\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " cve_api_data.append(api_content)\n", + " print(f\"✓ Ingested CVE API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"vulnerabilities\",\n", + " limit=1000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + "\n", + "print(f\"\\n📊 CVE Ingestion Summary:\")\n", + "print(f\" Vulnerability feeds: {len(cve_feed_list)}\")\n", + "print(f\" CVE API sources: {len(cve_api_data)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/finance/Financial_Data_Integration.ipynb b/docs/cookbook/use_cases/finance/Financial_Data_Integration.ipynb new file mode 100644 index 00000000..d3836cce --- /dev/null +++ b/docs/cookbook/use_cases/finance/Financial_Data_Integration.ipynb @@ -0,0 +1,538 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Financial Data Integration Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to integrate Python/FastMCP MCP servers as data sources for financial data ingestion. Connect to financial data MCP servers via URL, ingest market data, stock prices, and financial metrics, then build a knowledge graph for financial analysis.\n", + "\n", + "**IMPORTANT**: This implementation supports ONLY Python-based MCP servers and FastMCP servers. Users can bring their own Python/FastMCP MCP servers via URL connections.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: MCPIngestor, ingest_mcp, WebIngestor, FileIngestor\n", + "- **Parsing**: MCPParser, JSONParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Connect to Financial MCP Server → Ingest Market Data via MCP → Parse MCP Responses → Extract Financial Entities → Build Financial KG → Analyze Trends → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Connect to Financial Data MCP Server\n", + "\n", + "Connect to a Python/FastMCP MCP server that provides financial data via URL. The MCP server can expose resources (datasets, market data) and tools (queries, calculations).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import MCPIngestor, ingest_mcp\n", + "from semantica.parse import MCPParser, JSONParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Initialize MCP ingestor\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "# Connect to financial data MCP server via URL\n", + "# Replace with your actual MCP server URL\n", + "# Example: http://localhost:8000/mcp or https://api.example.com/financial-mcp\n", + "financial_mcp_url = \"http://localhost:8000/mcp\"\n", + "\n", + "try:\n", + " # Connect to MCP server\n", + " mcp_ingestor.connect(\n", + " \"financial_server\",\n", + " url=financial_mcp_url,\n", + " headers={\"Authorization\": \"Bearer your_token\"} if \"api.example.com\" in financial_mcp_url else {}\n", + " )\n", + " print(f\"✓ Connected to financial MCP server at {financial_mcp_url}\")\n", + " \n", + " # List available resources (datasets, market data feeds)\n", + " resources = mcp_ingestor.list_available_resources(\"financial_server\")\n", + " print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + " for resource in resources[:5]: # Show first 5\n", + " print(f\" - {resource.uri}: {resource.name}\")\n", + " if resource.description:\n", + " print(f\" {resource.description[:80]}...\")\n", + " \n", + " # List available tools (queries, calculations)\n", + " tools = mcp_ingestor.list_available_tools(\"financial_server\")\n", + " print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + " for tool in tools[:5]: # Show first 5\n", + " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Connection failed: {e}\")\n", + " print(\"Note: This example uses a placeholder URL. Replace with your actual MCP server URL.\")\n", + " print(\"For testing, you can use a mock MCP server or skip connection and use sample data below.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Financial Data from MCP Server\n", + "\n", + "Ingest financial data using both resource-based and tool-based methods from the MCP server.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize parsers\n", + "mcp_parser = MCPParser()\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "financial_data = []\n", + "\n", + "# Method 1: Resource-based ingestion\n", + "# Ingest from MCP resources (pre-defined datasets)\n", + "try:\n", + " # Example: Ingest market data resource\n", + " resource_data = mcp_ingestor.ingest_resources(\n", + " \"financial_server\",\n", + " resource_uris=[\"resource://market_data/daily\", \"resource://market_data/stocks\"]\n", + " )\n", + " \n", + " for item in resource_data:\n", + " financial_data.append(item)\n", + " print(f\"✓ Ingested resource: {item.resource_uri}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Resource ingestion: {e}\")\n", + "\n", + "# Method 2: Tool-based ingestion\n", + "# Call MCP tools to retrieve data dynamically\n", + "try:\n", + " # Example: Get stock prices for specific symbols\n", + " stock_prices = mcp_ingestor.ingest_tool_output(\n", + " \"financial_server\",\n", + " tool_name=\"get_stock_prices\",\n", + " arguments={\n", + " \"symbols\": [\"AAPL\", \"MSFT\", \"GOOGL\", \"TSLA\"],\n", + " \"date\": datetime.now().isoformat()\n", + " }\n", + " )\n", + " \n", + " if stock_prices:\n", + " financial_data.append(stock_prices)\n", + " print(f\"✓ Retrieved stock prices via tool\")\n", + " \n", + " # Example: Get market metrics\n", + " market_metrics = mcp_ingestor.ingest_tool_output(\n", + " \"financial_server\",\n", + " tool_name=\"get_market_metrics\",\n", + " arguments={\"sector\": \"Technology\"}\n", + " )\n", + " \n", + " if market_metrics:\n", + " financial_data.append(market_metrics)\n", + " print(f\"✓ Retrieved market metrics via tool\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Tool-based ingestion: {e}\")\n", + " print(\"Note: Using sample data for demonstration\")\n", + "\n", + "# Sample financial data (if MCP server is not available)\n", + "if not financial_data:\n", + " print(\"\\n📝 Using sample financial data for demonstration:\")\n", + " sample_data = {\n", + " \"stock_prices\": [\n", + " {\n", + " \"symbol\": \"AAPL\",\n", + " \"company\": \"Apple Inc.\",\n", + " \"price\": 175.50,\n", + " \"change\": 2.30,\n", + " \"change_percent\": 1.33,\n", + " \"volume\": 45000000,\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"sector\": \"Technology\"\n", + " },\n", + " {\n", + " \"symbol\": \"MSFT\",\n", + " \"company\": \"Microsoft Corporation\",\n", + " \"price\": 380.25,\n", + " \"change\": -1.50,\n", + " \"change_percent\": -0.39,\n", + " \"volume\": 28000000,\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"sector\": \"Technology\"\n", + " },\n", + " {\n", + " \"symbol\": \"GOOGL\",\n", + " \"company\": \"Alphabet Inc.\",\n", + " \"price\": 142.80,\n", + " \"change\": 3.20,\n", + " \"change_percent\": 2.29,\n", + " \"volume\": 32000000,\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=30)).isoformat(),\n", + " \"sector\": \"Technology\"\n", + " },\n", + " {\n", + " \"symbol\": \"TSLA\",\n", + " \"company\": \"Tesla Inc.\",\n", + " \"price\": 245.60,\n", + " \"change\": 5.40,\n", + " \"change_percent\": 2.25,\n", + " \"volume\": 55000000,\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"sector\": \"Automotive\"\n", + " }\n", + " ],\n", + " \"market_metrics\": {\n", + " \"total_volume\": 150000000,\n", + " \"market_cap\": 15000000000000,\n", + " \"sectors\": [\"Technology\", \"Automotive\"]\n", + " }\n", + " }\n", + " financial_data.append(sample_data)\n", + " print(f\" Loaded {len(sample_data['stock_prices'])} stock prices\")\n", + "\n", + "print(f\"\\n📊 Total financial data items ingested: {len(financial_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Parse MCP Data\n", + "\n", + "Parse the data received from MCP server responses (JSON, structured data).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parsed_financial_data = []\n", + "\n", + "# Parse MCP responses\n", + "for data_item in financial_data:\n", + " try:\n", + " # Parse MCP response (handles JSON, text, binary)\n", + " if isinstance(data_item, dict):\n", + " # If it's already structured, use it directly\n", + " parsed_item = data_item\n", + " else:\n", + " # Parse using MCP parser\n", + " parsed_item = mcp_parser.parse_response(data_item, response_type=\"json\")\n", + " \n", + " parsed_financial_data.append(parsed_item)\n", + " \n", + " except Exception as e:\n", + " print(f\"⚠ Parsing error: {e}\")\n", + "\n", + "# Extract stock prices from parsed data\n", + "stock_prices = []\n", + "for item in parsed_financial_data:\n", + " if isinstance(item, dict):\n", + " if \"stock_prices\" in item:\n", + " stock_prices.extend(item[\"stock_prices\"])\n", + " elif \"symbol\" in item:\n", + " stock_prices.append(item)\n", + "\n", + "print(f\"✓ Parsed {len(parsed_financial_data)} data items\")\n", + "print(f\"✓ Extracted {len(stock_prices)} stock price records\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Financial Entities and Relationships\n", + "\n", + "Extract financial entities (companies, stocks, sectors) and relationships from MCP data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "financial_entities = []\n", + "financial_relationships = []\n", + "\n", + "# Extract entities and relationships from stock prices\n", + "for stock in stock_prices:\n", + " if isinstance(stock, dict):\n", + " symbol = stock.get(\"symbol\", \"\")\n", + " company = stock.get(\"company\", \"\")\n", + " sector = stock.get(\"sector\", \"\")\n", + " \n", + " # Stock entity\n", + " financial_entities.append({\n", + " \"id\": symbol,\n", + " \"type\": \"Stock\",\n", + " \"name\": symbol,\n", + " \"properties\": {\n", + " \"price\": stock.get(\"price\", 0),\n", + " \"change\": stock.get(\"change\", 0),\n", + " \"change_percent\": stock.get(\"change_percent\", 0),\n", + " \"volume\": stock.get(\"volume\", 0),\n", + " \"timestamp\": stock.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Company entity\n", + " if company:\n", + " financial_entities.append({\n", + " \"id\": company,\n", + " \"type\": \"Company\",\n", + " \"name\": company,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Stock-Company relationship\n", + " financial_relationships.append({\n", + " \"source\": symbol,\n", + " \"target\": company,\n", + " \"type\": \"ticker_for\",\n", + " \"properties\": {\"timestamp\": stock.get(\"timestamp\", \"\")}\n", + " })\n", + " \n", + " # Sector entity\n", + " if sector:\n", + " financial_entities.append({\n", + " \"id\": sector,\n", + " \"type\": \"Sector\",\n", + " \"name\": sector,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Company-Sector relationship\n", + " if company:\n", + " financial_relationships.append({\n", + " \"source\": company,\n", + " \"target\": sector,\n", + " \"type\": \"belongs_to\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "# Remove duplicates\n", + "seen_entities = set()\n", + "unique_entities = []\n", + "for entity in financial_entities:\n", + " entity_key = (entity[\"id\"], entity[\"type\"])\n", + " if entity_key not in seen_entities:\n", + " seen_entities.add(entity_key)\n", + " unique_entities.append(entity)\n", + "\n", + "financial_entities = unique_entities\n", + "\n", + "print(f\"✓ Extracted {len(financial_entities)} financial entities\")\n", + "print(f\" - Stocks: {len([e for e in financial_entities if e['type'] == 'Stock'])}\")\n", + "print(f\" - Companies: {len([e for e in financial_entities if e['type'] == 'Company'])}\")\n", + "print(f\" - Sectors: {len([e for e in financial_entities if e['type'] == 'Sector'])}\")\n", + "print(f\"✓ Extracted {len(financial_relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Financial Knowledge Graph\n", + "\n", + "Build a knowledge graph from the extracted financial entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "# Build knowledge graph\n", + "financial_kg = builder.build(financial_entities, financial_relationships)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(financial_kg)\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "# Calculate graph metrics\n", + "centrality_scores = centrality_calculator.calculate_centrality(financial_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(financial_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(financial_kg)\n", + "\n", + "print(f\"✓ Built financial knowledge graph\")\n", + "print(f\" Entities: {len(financial_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(financial_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Communities detected: {len(communities)}\")\n", + "print(f\" Central entities: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Analyze Financial Trends\n", + "\n", + "Analyze financial trends using temporal queries and pattern detection.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Temporal analysis\n", + "start_time = (datetime.now() - timedelta(days=7)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=financial_kg,\n", + " query=\"Find stock price movements\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "# Inference engine for financial rules\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Financial analysis rules\n", + "inference_engine.add_rule(\"IF change_percent > 2 AND volume > 40000000 THEN strong_momentum\")\n", + "inference_engine.add_rule(\"IF change_percent < -1 AND volume > 50000000 THEN selling_pressure\")\n", + "inference_engine.add_rule(\"IF change_percent > 0 AND sector == 'Technology' THEN tech_growth\")\n", + "\n", + "# Add facts from stock data\n", + "for stock in stock_prices:\n", + " if isinstance(stock, dict):\n", + " inference_engine.add_fact({\n", + " \"symbol\": stock.get(\"symbol\", \"\"),\n", + " \"change_percent\": stock.get(\"change_percent\", 0),\n", + " \"volume\": stock.get(\"volume\", 0),\n", + " \"sector\": stock.get(\"sector\", \"\")\n", + " })\n", + "\n", + "# Generate insights\n", + "financial_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"✓ Temporal analysis completed\")\n", + "print(f\" Temporal entities: {len(temporal_results.get('entities', []))}\")\n", + "print(f\" Financial insights: {len(financial_insights)}\")\n", + "\n", + "# Display insights\n", + "for insight in financial_insights[:3]:\n", + " print(f\" - {insight}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Export and Visualize\n", + "\n", + "Export the financial knowledge graph and generate visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "import os\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export knowledge graph\n", + "json_exporter.export_knowledge_graph(financial_kg, os.path.join(temp_dir, \"financial_kg.json\"))\n", + "csv_exporter.export_entities(financial_entities, os.path.join(temp_dir, \"financial_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(financial_kg, os.path.join(temp_dir, \"financial_kg.rdf\"))\n", + "\n", + "# Generate report\n", + "report_data = {\n", + " \"summary\": f\"Financial data integration from MCP server identified {len(financial_insights)} insights\",\n", + " \"stocks_analyzed\": len([e for e in financial_entities if e['type'] == 'Stock']),\n", + " \"companies\": len([e for e in financial_entities if e['type'] == 'Company']),\n", + " \"sectors\": len([e for e in financial_entities if e['type'] == 'Sector']),\n", + " \"insights\": len(financial_insights)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"✓ Exported financial knowledge graph\")\n", + "print(f\" JSON: {os.path.join(temp_dir, 'financial_kg.json')}\")\n", + "print(f\" CSV: {os.path.join(temp_dir, 'financial_entities.csv')}\")\n", + "print(f\" RDF: {os.path.join(temp_dir, 'financial_kg.rdf')}\")\n", + "print(f\"✓ Generated report ({len(report)} characters)\")\n", + "\n", + "# Visualize\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(financial_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(financial_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(financial_kg, output=\"interactive\")\n", + "\n", + "print(\"✓ Generated visualizations for financial knowledge graph\")\n", + "\n", + "# Cleanup: Disconnect from MCP server\n", + "try:\n", + " mcp_ingestor.disconnect(\"financial_server\")\n", + " print(\"\\n✓ Disconnected from MCP server\")\n", + "except:\n", + " pass\n", + "\n", + "print(f\"\\n✅ Pipeline complete: MCP Server → Ingest → Parse → Extract → Build KG → Analyze → Export → Visualize\")\n", + "print(f\"📊 Total modules used: 20+\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/finance/Financial_Reports_Analysis.ipynb b/docs/cookbook/use_cases/finance/Financial_Reports_Analysis.ipynb new file mode 100644 index 00000000..5565c7b2 --- /dev/null +++ b/docs/cookbook/use_cases/finance/Financial_Reports_Analysis.ipynb @@ -0,0 +1,420 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Financial Reports Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete financial reports analysis pipeline: ingest financial documents from multiple sources (SEC filings, annual reports, financial databases), extract financial entities, build knowledge graph, analyze relationships, and generate financial insights.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: DocumentParser, PDFParser, HTMLParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Financial Documents → Parse → Extract Entities → Build KG → Analyze Relationships → Generate Insights → Export → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Financial Documents from Multiple Sources\n", + "\n", + "Ingest financial reports from SEC filings, annual reports, and financial databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import DocumentParser, PDFParser, HTMLParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "document_parser = DocumentParser()\n", + "pdf_parser = PDFParser()\n", + "html_parser = HTMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real financial data sources\n", + "sec_edgar_urls = [\n", + " \"https://www.sec.gov/cgi-bin/browse-edgar\", # SEC EDGAR database\n", + " \"https://www.sec.gov/Archives/edgar/data/\", # SEC EDGAR archives\n", + " \"https://data.sec.gov/submissions/\" # SEC submissions API\n", + "]\n", + "\n", + "financial_feeds = [\n", + " \"https://feeds.reuters.com/reuters/businessNews\",\n", + " \"https://feeds.reuters.com/reuters/topNews\",\n", + " \"https://rss.cnn.com/rss/money_latest.rss\",\n", + " \"https://feeds.bloomberg.com/markets/news.rss\"\n", + "]\n", + "\n", + "# Real database connection for financial reports\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/financial_reports_db\"\n", + "db_query = \"SELECT company_name, report_type, filing_date, document_url FROM financial_reports WHERE filing_date > CURRENT_DATE - INTERVAL '1 year' ORDER BY filing_date DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample financial report data (simulating real SEC filing structure)\n", + "financial_report_file = os.path.join(temp_dir, \"financial_report.json\")\n", + "report_data = {\n", + " \"company\": \"Apple Inc.\",\n", + " \"symbol\": \"AAPL\",\n", + " \"report_type\": \"10-K\",\n", + " \"filing_date\": (datetime.now() - timedelta(days=30)).isoformat(),\n", + " \"revenue\": 394328000000,\n", + " \"net_income\": 99803000000,\n", + " \"total_assets\": 352755000000,\n", + " \"total_liabilities\": 290437000000,\n", + " \"segments\": [\"iPhone\", \"Mac\", \"iPad\", \"Services\", \"Wearables\"],\n", + " \"geographic_regions\": [\"Americas\", \"Europe\", \"Greater China\", \"Japan\", \"Rest of Asia Pacific\"]\n", + "}\n", + "\n", + "with open(financial_report_file, 'w') as f:\n", + " json.dump(report_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(financial_report_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(financial_report_file)\n", + "\n", + "# Ingest from financial feeds\n", + "financial_feed_list = []\n", + "for feed_url in financial_feeds[:2]: # Process first 2 feeds\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " financial_feed_list.append(feed_data)\n", + " print(f\"✓ Ingested financial feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Ingest from SEC EDGAR (example)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(\"https://www.sec.gov/cgi-bin/browse-edgar\")\n", + " if web_content:\n", + " print(f\"✓ Ingested SEC EDGAR content\")\n", + "except Exception as e:\n", + " print(f\"⚠ SEC EDGAR ingestion (example): {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Financial reports ingested: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Financial feeds: {len(financial_feed_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Financial Entities\n", + "\n", + "Extract financial entities (companies, metrics, segments, regions) from financial reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "financial_entities = []\n", + "financial_relationships = []\n", + "\n", + "# Extract from financial report data\n", + "if parsed_data and parsed_data.data:\n", + " report = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " \n", + " if isinstance(report, dict):\n", + " # Company entity\n", + " financial_entities.append({\n", + " \"id\": report.get(\"symbol\", \"\"),\n", + " \"type\": \"Company\",\n", + " \"name\": report.get(\"company\", \"\"),\n", + " \"properties\": {\n", + " \"symbol\": report.get(\"symbol\", \"\"),\n", + " \"report_type\": report.get(\"report_type\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Financial metrics\n", + " financial_entities.append({\n", + " \"id\": f\"{report.get('symbol', '')}_revenue\",\n", + " \"type\": \"Financial_Metric\",\n", + " \"name\": \"Revenue\",\n", + " \"properties\": {\n", + " \"value\": report.get(\"revenue\", 0),\n", + " \"currency\": \"USD\",\n", + " \"filing_date\": report.get(\"filing_date\", \"\")\n", + " }\n", + " })\n", + " \n", + " financial_entities.append({\n", + " \"id\": f\"{report.get('symbol', '')}_net_income\",\n", + " \"type\": \"Financial_Metric\",\n", + " \"name\": \"Net Income\",\n", + " \"properties\": {\n", + " \"value\": report.get(\"net_income\", 0),\n", + " \"currency\": \"USD\",\n", + " \"filing_date\": report.get(\"filing_date\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Segments\n", + " for segment in report.get(\"segments\", []):\n", + " financial_entities.append({\n", + " \"id\": f\"{report.get('symbol', '')}_segment_{segment}\",\n", + " \"type\": \"Business_Segment\",\n", + " \"name\": segment,\n", + " \"properties\": {}\n", + " })\n", + " financial_relationships.append({\n", + " \"source\": report.get(\"symbol\", \"\"),\n", + " \"target\": f\"{report.get('symbol', '')}_segment_{segment}\",\n", + " \"type\": \"has_segment\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Geographic regions\n", + " for region in report.get(\"geographic_regions\", []):\n", + " financial_entities.append({\n", + " \"id\": f\"{report.get('symbol', '')}_region_{region}\",\n", + " \"type\": \"Geographic_Region\",\n", + " \"name\": region,\n", + " \"properties\": {}\n", + " })\n", + " financial_relationships.append({\n", + " \"source\": report.get(\"symbol\", \"\"),\n", + " \"target\": f\"{report.get('symbol', '')}_region_{region}\",\n", + " \"type\": \"operates_in\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Relationships\n", + " financial_relationships.append({\n", + " \"source\": report.get(\"symbol\", \"\"),\n", + " \"target\": f\"{report.get('symbol', '')}_revenue\",\n", + " \"type\": \"has_metric\",\n", + " \"properties\": {\"filing_date\": report.get(\"filing_date\", \"\")}\n", + " })\n", + " \n", + " financial_relationships.append({\n", + " \"source\": report.get(\"symbol\", \"\"),\n", + " \"target\": f\"{report.get('symbol', '')}_net_income\",\n", + " \"type\": \"has_metric\",\n", + " \"properties\": {\"filing_date\": report.get(\"filing_date\", \"\")}\n", + " })\n", + "\n", + "print(f\"Extracted {len(financial_entities)} financial entities\")\n", + "print(f\"Extracted {len(financial_relationships)} financial relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Financial Knowledge Graph\n", + "\n", + "Build knowledge graph from financial entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "financial_kg = builder.build(financial_entities, financial_relationships)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(financial_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(financial_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(financial_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(financial_kg)\n", + "\n", + "print(f\"Built financial knowledge graph\")\n", + "print(f\" Entities: {len(financial_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(financial_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Communities: {len(communities)}\")\n", + "print(f\" Central entities: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze Financial Relationships\n", + "\n", + "Analyze financial relationships and generate insights.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Temporal analysis\n", + "start_time = (datetime.now() - timedelta(days=365)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=financial_kg,\n", + " query=\"Find financial metrics in the last year\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " financial_kg,\n", + " pattern_type=\"trend\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "# Financial analysis rules\n", + "inference_engine.add_rule(\"IF revenue > 300000000000 AND net_income > 50000000000 THEN high_performer\")\n", + "inference_engine.add_rule(\"IF company has_segment Services AND revenue > 20000000000 THEN services_growth\")\n", + "\n", + "# Add facts from financial data\n", + "if parsed_data and parsed_data.data:\n", + " report = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " if isinstance(report, dict):\n", + " inference_engine.add_fact({\n", + " \"company\": report.get(\"symbol\", \"\"),\n", + " \"revenue\": report.get(\"revenue\", 0),\n", + " \"net_income\": report.get(\"net_income\", 0),\n", + " \"segments\": report.get(\"segments\", [])\n", + " })\n", + "\n", + "financial_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Temporal query returned {len(temporal_results.get('entities', []))} entities\")\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"Generated {len(financial_insights)} financial insights\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Financial Analysis Reports\n", + "\n", + "Generate comprehensive financial analysis reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(financial_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(financial_kg, os.path.join(temp_dir, \"financial_kg.json\"))\n", + "csv_exporter.export_entities(financial_entities, os.path.join(temp_dir, \"financial_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(financial_kg, os.path.join(temp_dir, \"financial_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Financial analysis identified {len(financial_insights)} insights from {len(financial_entities)} entities\",\n", + " \"entities_analyzed\": len(financial_entities),\n", + " \"relationships\": len(financial_relationships),\n", + " \"insights\": len(financial_insights),\n", + " \"quality_score\": quality_score.get('overall_score', 0),\n", + " \"patterns\": len(temporal_patterns)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"Generated financial analysis report\")\n", + "print(f\"Report length: {len(report)} characters\")\n", + "print(f\"Graph quality score: {quality_score.get('overall_score', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Visualize Financial Analysis\n", + "\n", + "Visualize financial knowledge graph and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(financial_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(financial_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(financial_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated visualizations for financial knowledge graph, analytics, and temporal patterns\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Financial Documents → Parse → Extract → Build KG → Analyze Relationships → Generate Insights → Export → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/finance/Fraud_Detection.ipynb b/docs/cookbook/use_cases/finance/Fraud_Detection.ipynb new file mode 100644 index 00000000..abc73417 --- /dev/null +++ b/docs/cookbook/use_cases/finance/Fraud_Detection.ipynb @@ -0,0 +1,402 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fraud Detection Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete fraud detection pipeline for finance: ingest transaction streams, build temporal knowledge graph, detect fraud patterns, perform anomaly detection, and generate alerts.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, StreamIngestor, DBIngestor\n", + "- **Parsing**: StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector\n", + "- **KG**: GraphBuilder, TemporalPatternDetector, GraphAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, AutomatedFixer\n", + "- **Export**: JSONExporter, CSVExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Transaction Stream → Parse → Extract → Build Temporal KG → Detect Patterns → Anomaly Detection → Generate Alerts → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Process Transactions\n", + "\n", + "Ingest and parse transaction data from multiple sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, StreamIngestor, DBIngestor\n", + "from semantica.parse import StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector\n", + "from semantica.kg import GraphBuilder, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor, AutomatedFixer\n", + "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "stream_ingestor = StreamIngestor()\n", + "db_ingestor = DBIngestor()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "# Real streaming sources for transaction monitoring\n", + "stream_sources = [\n", + " {\n", + " \"type\": \"kafka\",\n", + " \"topic\": \"transactions\",\n", + " \"bootstrap_servers\": [\"localhost:9092\"],\n", + " \"consumer_config\": {\"group_id\": \"fraud_detection\"}\n", + " },\n", + " {\n", + " \"type\": \"rabbitmq\",\n", + " \"queue\": \"payment_events\",\n", + " \"connection_url\": \"amqp://user:password@localhost:5672/\"\n", + " }\n", + "]\n", + "\n", + "# Real database connection for transaction data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/transactions_db\"\n", + "db_query = \"SELECT transaction_id, user_id, amount, merchant, location, timestamp, device FROM transactions WHERE timestamp > NOW() - INTERVAL '24 hours' ORDER BY timestamp DESC LIMIT 10000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "transactions_file = os.path.join(temp_dir, \"transactions.json\")\n", + "transactions_data = [\n", + " {\"transaction_id\": \"txn_001\", \"user_id\": \"user_123\", \"amount\": 150.00, \"merchant\": \"Online Store\", \"location\": \"New York\", \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(), \"device\": \"mobile\"},\n", + " {\"transaction_id\": \"txn_002\", \"user_id\": \"user_123\", \"amount\": 2500.00, \"merchant\": \"Luxury Store\", \"location\": \"Paris\", \"timestamp\": (datetime.now() - timedelta(minutes=30)).isoformat(), \"device\": \"web\"},\n", + " {\"transaction_id\": \"txn_003\", \"user_id\": \"user_456\", \"amount\": 50.00, \"merchant\": \"Grocery Store\", \"location\": \"San Francisco\", \"timestamp\": (datetime.now() - timedelta(minutes=15)).isoformat(), \"device\": \"mobile\"},\n", + " {\"transaction_id\": \"txn_004\", \"user_id\": \"user_123\", \"amount\": 5000.00, \"merchant\": \"Electronics Store\", \"location\": \"Tokyo\", \"timestamp\": (datetime.now() - timedelta(minutes=5)).isoformat(), \"device\": \"mobile\"}\n", + "]\n", + "\n", + "with open(transactions_file, 'w') as f:\n", + " json.dump(transactions_data, f)\n", + "\n", + "file_objects = file_ingestor.ingest_file(transactions_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(transactions_file)\n", + "\n", + "transaction_stream = []\n", + "for txn in parsed_data.get(\"data\", transactions_data):\n", + " if isinstance(txn, dict):\n", + " txn_copy = txn.copy()\n", + " if \"timestamp\" in txn_copy and isinstance(txn_copy[\"timestamp\"], str):\n", + " txn_copy[\"timestamp\"] = datetime.fromisoformat(txn_copy[\"timestamp\"])\n", + " transaction_stream.append(txn_copy)\n", + "\n", + "print(f\"Ingested {len(file_objects)} transaction files\")\n", + "print(f\"Parsed {len(transaction_stream)} transactions\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Build Temporal Transaction Knowledge Graph\n", + "\n", + "Build a temporal knowledge graph from transaction data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "\n", + "transaction_entities = []\n", + "relationships = []\n", + "\n", + "for txn in transaction_stream:\n", + " txn_id = txn[\"transaction_id\"]\n", + " user_id = txn[\"user_id\"]\n", + " merchant = txn[\"merchant\"]\n", + " location = txn[\"location\"]\n", + "\n", + " transaction_entities.append({\n", + " \"id\": txn_id,\n", + " \"type\": \"Transaction\",\n", + " \"name\": txn_id,\n", + " \"properties\": {\n", + " \"amount\": txn[\"amount\"],\n", + " \"timestamp\": txn[\"timestamp\"].isoformat() if isinstance(txn[\"timestamp\"], datetime) else txn[\"timestamp\"],\n", + " \"device\": txn[\"device\"]\n", + " }\n", + " })\n", + "\n", + " transaction_entities.append({\n", + " \"id\": user_id,\n", + " \"type\": \"User\",\n", + " \"name\": user_id,\n", + " \"properties\": {}\n", + " })\n", + "\n", + " transaction_entities.append({\n", + " \"id\": merchant,\n", + " \"type\": \"Merchant\",\n", + " \"name\": merchant,\n", + " \"properties\": {}\n", + " })\n", + "\n", + " transaction_entities.append({\n", + " \"id\": location,\n", + " \"type\": \"Location\",\n", + " \"name\": location,\n", + " \"properties\": {}\n", + " })\n", + "\n", + " relationships.append({\n", + " \"source\": user_id,\n", + " \"target\": txn_id,\n", + " \"type\": \"performed\",\n", + " \"properties\": {\"timestamp\": txn[\"timestamp\"].isoformat() if isinstance(txn[\"timestamp\"], datetime) else txn[\"timestamp\"]}\n", + " })\n", + "\n", + " relationships.append({\n", + " \"source\": txn_id,\n", + " \"target\": merchant,\n", + " \"type\": \"at_merchant\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + " relationships.append({\n", + " \"source\": txn_id,\n", + " \"target\": location,\n", + " \"type\": \"in_location\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "transaction_kg = builder.build(transaction_entities, relationships)\n", + "\n", + "print(f\"Built temporal knowledge graph with {len(transaction_entities)} entities and {len(relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Detect Fraud Patterns\n", + "\n", + "Detect fraud patterns using temporal analysis and inference.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", + " transaction_kg,\n", + " pattern_type=\"sequence\",\n", + " min_frequency=2\n", + ")\n", + "\n", + "connectivity_analysis = graph_analyzer.analyze_connectivity(transaction_kg)\n", + "\n", + "fraud_patterns = []\n", + "user_transactions = {}\n", + "for txn in transaction_stream:\n", + " user_id = txn[\"user_id\"]\n", + " if user_id not in user_transactions:\n", + " user_transactions[user_id] = []\n", + " user_transactions[user_id].append(txn)\n", + "\n", + "for user_id, txns in user_transactions.items():\n", + " if len(txns) > 1:\n", + " amounts = [t[\"amount\"] for t in txns]\n", + " locations = [t[\"location\"] for t in txns]\n", + " timestamps = [t[\"timestamp\"] if isinstance(t[\"timestamp\"], datetime) else datetime.fromisoformat(t[\"timestamp\"]) for t in txns]\n", + "\n", + " if max(amounts) > 1000:\n", + " fraud_patterns.append({\n", + " \"type\": \"high_value_transaction\",\n", + " \"user_id\": user_id,\n", + " \"amount\": max(amounts),\n", + " \"severity\": \"medium\"\n", + " })\n", + "\n", + " if len(set(locations)) > 2:\n", + " time_span = max(timestamps) - min(timestamps)\n", + " if time_span.total_seconds() < 3600:\n", + " fraud_patterns.append({\n", + " \"type\": \"rapid_location_change\",\n", + " \"user_id\": user_id,\n", + " \"locations\": list(set(locations)),\n", + " \"severity\": \"high\"\n", + " })\n", + "\n", + "inference_engine.add_rule(\"IF transaction amount > 2000 AND device is mobile THEN high_risk\")\n", + "for txn in transaction_stream:\n", + " if txn[\"amount\"] > 2000 and txn[\"device\"] == \"mobile\":\n", + " inference_engine.add_fact({\"transaction_id\": txn[\"transaction_id\"], \"risk\": \"high\"})\n", + "\n", + "inferred_risks = inference_engine.forward_chain()\n", + "\n", + "print(f\"Detected {len(fraud_patterns)} fraud patterns\")\n", + "print(f\"Temporal patterns: {len(temporal_patterns)}\")\n", + "print(f\"Inferred {len(inferred_risks)} risk assessments\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Anomaly Detection\n", + "\n", + "Detect anomalous transactions using pattern analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "anomaly_patterns = pattern_detector.detect_temporal_patterns(\n", + " transaction_kg,\n", + " pattern_type=\"anomaly\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "anomalies = []\n", + "for txn in transaction_stream:\n", + " score = 0\n", + " reasons = []\n", + "\n", + " if txn[\"amount\"] > 2000:\n", + " score += 3\n", + " reasons.append(\"High transaction amount\")\n", + "\n", + " if txn[\"amount\"] > 1000 and txn[\"device\"] == \"mobile\":\n", + " score += 2\n", + " reasons.append(\"High amount on mobile device\")\n", + "\n", + " user_txns = [t for t in transaction_stream if t[\"user_id\"] == txn[\"user_id\"]]\n", + " if len(user_txns) > 1:\n", + " recent_txns = sorted(user_txns, key=lambda x: x[\"timestamp\"] if isinstance(x[\"timestamp\"], datetime) else datetime.fromisoformat(x[\"timestamp\"]), reverse=True)[:3]\n", + " locations = [t[\"location\"] for t in recent_txns]\n", + " if len(set(locations)) > 2:\n", + " time_span = (recent_txns[0][\"timestamp\"] if isinstance(recent_txns[0][\"timestamp\"], datetime) else datetime.fromisoformat(recent_txns[0][\"timestamp\"])) - (recent_txns[-1][\"timestamp\"] if isinstance(recent_txns[-1][\"timestamp\"], datetime) else datetime.fromisoformat(recent_txns[-1][\"timestamp\"]))\n", + " if time_span.total_seconds() < 3600:\n", + " score += 4\n", + " reasons.append(\"Rapid location changes\")\n", + "\n", + " if score >= 3:\n", + " anomalies.append({\n", + " \"transaction_id\": txn[\"transaction_id\"],\n", + " \"user_id\": txn[\"user_id\"],\n", + " \"severity\": \"high\" if score >= 5 else \"medium\",\n", + " \"score\": score,\n", + " \"reasons\": reasons,\n", + " \"timestamp\": txn[\"timestamp\"].isoformat() if isinstance(txn[\"timestamp\"], datetime) else txn[\"timestamp\"]\n", + " })\n", + "\n", + "print(f\"Detected {len(anomalies)} anomalies\")\n", + "for anomaly in anomalies:\n", + " print(f\" Transaction: {anomaly['transaction_id']} - Severity: {anomaly['severity']} - Score: {anomaly['score']}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Alerts and Reports\n", + "\n", + "Generate fraud alerts and reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "json_exporter.export_knowledge_graph(transaction_kg, os.path.join(temp_dir, \"transactions.json\"))\n", + "csv_exporter.export_entities(transaction_entities, os.path.join(temp_dir, \"entities.csv\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Fraud detection analysis identified {len(anomalies)} suspicious transactions\",\n", + " \"fraud_patterns\": len(fraud_patterns),\n", + " \"anomalies\": len(anomalies),\n", + " \"transactions_analyzed\": len(transaction_stream)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"Generated fraud detection report\")\n", + "print(f\"Report length: {len(report)} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Quality Assessment and Visualization\n", + "\n", + "Assess graph quality and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "automated_fixer = AutomatedFixer()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(transaction_kg)\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(transaction_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(transaction_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(transaction_kg, output=\"interactive\")\n", + "\n", + "print(f\"Graph quality score: {quality_score.get('overall_score', 0):.3f}\")\n", + "print(\"Generated visualizations for knowledge graph, temporal patterns, and analytics\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Transaction Stream → Parse → Extract → Temporal KG → Pattern Detection → Anomaly Detection → Reports → Visualization\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/finance/Investment_Analysis_Hybrid_RAG.ipynb b/docs/cookbook/use_cases/finance/Investment_Analysis_Hybrid_RAG.ipynb new file mode 100644 index 00000000..d5098629 --- /dev/null +++ b/docs/cookbook/use_cases/finance/Investment_Analysis_Hybrid_RAG.ipynb @@ -0,0 +1,398 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Investment Analysis Hybrid RAG Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete investment analysis hybrid RAG pipeline: ingest investment data from multiple sources (market data APIs, financial feeds, databases), extract investment entities, build knowledge graph, generate embeddings, set up hybrid search (vector + temporal KG), and query investment insights using advanced RAG.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser, HTMLParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder\n", + "- **Vector Store**: VectorStore, HybridSearch\n", + "- **Context**: ContextRetriever, ContextGraphBuilder\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Multi-Source Investment Data → Parse → Extract Entities → Build KG → Generate Embeddings → Vector Store → Hybrid RAG Setup → Query Insights → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Multi-Source Investment Data Ingestion\n", + "\n", + "Ingest investment data from market APIs, financial feeds, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser, HTMLParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.vector_store import VectorStore, HybridSearch\n", + "from semantica.context import ContextRetriever, ContextGraphBuilder\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "html_parser = HTMLParser()\n", + "\n", + "# Real investment data sources\n", + "investment_apis = [\n", + " \"https://api.polygon.io/v2/aggs/ticker/AAPL/range/1/day/2024-01-01/2024-01-31\", # Polygon.io\n", + " \"https://www.alphavantage.co/query?function=OVERVIEW&symbol=AAPL&apikey=demo\", # Alpha Vantage\n", + " \"https://api.github.com/repos/ranaroussi/yfinance\" # Yahoo Finance API\n", + "]\n", + "\n", + "financial_feeds = [\n", + " \"https://feeds.reuters.com/reuters/businessNews\",\n", + " \"https://feeds.reuters.com/reuters/topNews\",\n", + " \"https://rss.cnn.com/rss/money_latest.rss\",\n", + " \"https://feeds.bloomberg.com/markets/news.rss\"\n", + "]\n", + "\n", + "# Real database connection for investment data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/investment_db\"\n", + "db_query = \"SELECT symbol, company_name, sector, market_cap, pe_ratio, dividend_yield FROM investments WHERE last_updated > NOW() - INTERVAL '7 days' ORDER BY market_cap DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample investment data\n", + "investment_data_file = os.path.join(temp_dir, \"investment_data.json\")\n", + "investment_data = [\n", + " {\n", + " \"symbol\": \"AAPL\",\n", + " \"company\": \"Apple Inc.\",\n", + " \"sector\": \"Technology\",\n", + " \"market_cap\": 2800000000000,\n", + " \"pe_ratio\": 28.5,\n", + " \"dividend_yield\": 0.5,\n", + " \"price\": 175.50,\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat()\n", + " },\n", + " {\n", + " \"symbol\": \"MSFT\",\n", + " \"company\": \"Microsoft Corporation\",\n", + " \"sector\": \"Technology\",\n", + " \"market_cap\": 2800000000000,\n", + " \"pe_ratio\": 32.1,\n", + " \"dividend_yield\": 0.7,\n", + " \"price\": 380.25,\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(investment_data_file, 'w') as f:\n", + " json.dump(investment_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(investment_data_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(investment_data_file)\n", + "\n", + "# Ingest from financial feeds\n", + "financial_feed_list = []\n", + "for feed_url in financial_feeds[:2]:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " financial_feed_list.append(feed_data)\n", + " print(f\"✓ Ingested financial feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Ingest from investment APIs\n", + "api_content_list = []\n", + "for api_url in investment_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " api_content_list.append(api_content)\n", + " print(f\"✓ Ingested investment API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Investment data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Financial feeds: {len(financial_feed_list)}\")\n", + "print(f\" Investment APIs: {len(api_content_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Investment Entities and Build Knowledge Graph\n", + "\n", + "Extract investment entities and build knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "investment_entities = []\n", + "investment_relationships = []\n", + "all_documents = []\n", + "\n", + "# Extract from investment data\n", + "if parsed_data and parsed_data.data:\n", + " for investment in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(investment, dict):\n", + " investment_text = f\"{investment.get('company', '')} ({investment.get('symbol', '')}) in {investment.get('sector', '')} sector\"\n", + " all_documents.append(investment_text)\n", + " \n", + " investment_entities.append({\n", + " \"id\": investment.get(\"symbol\", \"\"),\n", + " \"type\": \"Stock\",\n", + " \"name\": investment.get(\"company\", \"\"),\n", + " \"properties\": {\n", + " \"symbol\": investment.get(\"symbol\", \"\"),\n", + " \"sector\": investment.get(\"sector\", \"\"),\n", + " \"market_cap\": investment.get(\"market_cap\", 0),\n", + " \"pe_ratio\": investment.get(\"pe_ratio\", 0),\n", + " \"dividend_yield\": investment.get(\"dividend_yield\", 0),\n", + " \"price\": investment.get(\"price\", 0),\n", + " \"timestamp\": investment.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " investment_entities.append({\n", + " \"id\": investment.get(\"sector\", \"\"),\n", + " \"type\": \"Sector\",\n", + " \"name\": investment.get(\"sector\", \"\"),\n", + " \"properties\": {}\n", + " })\n", + " \n", + " investment_relationships.append({\n", + " \"source\": investment.get(\"symbol\", \"\"),\n", + " \"target\": investment.get(\"sector\", \"\"),\n", + " \"type\": \"belongs_to\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "investment_kg = builder.build(investment_entities, investment_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(investment_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(investment_kg)\n", + "\n", + "print(f\"Extracted {len(investment_entities)} investment entities\")\n", + "print(f\"Extracted {len(investment_relationships)} relationships\")\n", + "print(f\"Collected {len(all_documents)} investment documents\")\n", + "print(f\"Built investment knowledge graph with {len(investment_kg.get('entities', []))} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Generate Embeddings and Setup Vector Store\n", + "\n", + "Generate embeddings and setup vector store for hybrid RAG.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "vector_store = VectorStore()\n", + "hybrid_search = HybridSearch()\n", + "\n", + "embeddings = embedding_generator.generate(all_documents)\n", + "\n", + "metadata = []\n", + "for i, doc in enumerate(all_documents):\n", + " metadata.append({\n", + " \"id\": f\"doc_{i}\",\n", + " \"text\": doc,\n", + " \"source\": \"investment_data\"\n", + " })\n", + "\n", + "vector_ids = vector_store.store_vectors(embeddings, metadata)\n", + "\n", + "print(f\"Generated embeddings for {len(all_documents)} documents\")\n", + "print(f\"Stored {len(vector_ids)} vectors in vector store\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Setup Hybrid RAG and Query Investment Insights\n", + "\n", + "Setup hybrid search and query investment insights.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "context_retriever = ContextRetriever(\n", + " knowledge_graph=investment_kg,\n", + " vector_store=vector_store\n", + ")\n", + "\n", + "temporal_query = TemporalGraphQuery()\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Query examples\n", + "queries = [\n", + " \"What are the best performing sectors?\",\n", + " \"Find technology stocks with high market cap\",\n", + " \"What investments have good dividend yields?\"\n", + "]\n", + "\n", + "query_results = []\n", + "for query in queries:\n", + " query_embedding = text_embedder.embed_text(query)\n", + " vector_results = vector_store.search_vectors(query_embedding, k=3)\n", + " \n", + " start_time = (datetime.now() - timedelta(days=30)).isoformat()\n", + " end_time = datetime.now().isoformat()\n", + " \n", + " temporal_results = temporal_query.query_time_range(\n", + " graph=investment_kg,\n", + " query=query,\n", + " start_time=start_time,\n", + " end_time=end_time\n", + " )\n", + " \n", + " context_results = context_retriever.retrieve(\n", + " query=query,\n", + " top_k=3,\n", + " use_graph_expansion=True\n", + " )\n", + " \n", + " query_results.append({\n", + " \"query\": query,\n", + " \"vector_results\": len(vector_results),\n", + " \"temporal_results\": len(temporal_results.get('entities', [])),\n", + " \"context_results\": len(context_results) if context_results else 0\n", + " })\n", + "\n", + "# Investment analysis rules\n", + "inference_engine.add_rule(\"IF pe_ratio < 20 AND dividend_yield > 0.5 THEN value_stock\")\n", + "inference_engine.add_rule(\"IF market_cap > 1000000000000 AND sector is Technology THEN mega_cap_tech\")\n", + "\n", + "for investment in parsed_data.data if parsed_data and parsed_data.data else []:\n", + " if isinstance(investment, dict):\n", + " inference_engine.add_fact({\n", + " \"symbol\": investment.get(\"symbol\", \"\"),\n", + " \"pe_ratio\": investment.get(\"pe_ratio\", 0),\n", + " \"dividend_yield\": investment.get(\"dividend_yield\", 0),\n", + " \"market_cap\": investment.get(\"market_cap\", 0),\n", + " \"sector\": investment.get(\"sector\", \"\")\n", + " })\n", + "\n", + "investment_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Processed {len(queries)} investment queries\")\n", + "for result in query_results:\n", + " print(f\" Query: '{result['query']}' - Vector: {result['vector_results']}, Temporal: {result['temporal_results']}, Context: {result['context_results']}\")\n", + "print(f\"Generated {len(investment_insights)} investment insights\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate investment analysis reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(investment_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(investment_kg, os.path.join(temp_dir, \"investment_kg.json\"))\n", + "csv_exporter.export_entities(investment_entities, os.path.join(temp_dir, \"investment_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(investment_kg, os.path.join(temp_dir, \"investment_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Investment analysis identified {len(investment_insights)} insights from {len(investment_entities)} entities\",\n", + " \"investments_analyzed\": len(parsed_data.data) if parsed_data and parsed_data.data else 0,\n", + " \"insights\": len(investment_insights),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(investment_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(investment_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(investment_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated investment analysis report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Multi-Source Investment Data → Parse → Extract → Build KG → Embeddings → Vector Store → Hybrid RAG → Query → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/finance/Market_Intelligence.ipynb b/docs/cookbook/use_cases/finance/Market_Intelligence.ipynb new file mode 100644 index 00000000..08fa7049 --- /dev/null +++ b/docs/cookbook/use_cases/finance/Market_Intelligence.ipynb @@ -0,0 +1,493 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Market Intelligence Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete market intelligence pipeline: ingest market data from multiple sources (web APIs, financial feeds, databases), extract market entities, build temporal knowledge graph, analyze trends, and generate market insights.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, FeedIngestor, DBIngestor, FileIngestor, MCPIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser, HTMLParser, MCPParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Multiple Market Sources (Web, Feeds, DB, Files, MCP) → Parse Data → Extract Market Entities → Build Temporal KG → Analyze Trends → Generate Insights → Export → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Market Data from Multiple Sources\n", + "\n", + "Ingest market data from web APIs, financial feeds, databases, and files.\n", + "# market_entities = extractor.extract(market_data)\n", + "'''\n", + "\n", + "## Step 3: Build Temporal Market Knowledge Graph\n", + "\n", + "'''\n", + "# from semantica.kg import GraphBuilder\n", + "# \n", + "# builder = GraphBuilder()\n", + "# market_kg = builder.build(market_entities, relationships, temporal=True)\n", + "'''\n", + "\n", + "## Step 4: Analyze Trends\n", + "\n", + "'''\n", + "# from semantica.kg import TemporalQuery\n", + "# \n", + "# temporal_query = TemporalQuery()\n", + "# \n", + "# # Analyze market trends over time\n", + "# trends = temporal_query.analyze_trends(market_kg, time_window=\"1M\")\n", + "# \n", + "# # Market research\n", + "# print(f\"Analyzed trends for {len(market_kg.nodes)} market entities\")\n", + "'''\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, FeedIngestor, DBIngestor, FileIngestor, MCPIngestor, ingest_mcp\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser, HTMLParser, MCPParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "db_ingestor = DBIngestor()\n", + "file_ingestor = FileIngestor()\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "html_parser = HTMLParser()\n", + "mcp_parser = MCPParser()\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Real-world market data formats\n", + "market_data_json = os.path.join(temp_dir, \"market_data.json\")\n", + "market_data = [\n", + " {\n", + " \"symbol\": \"AAPL\",\n", + " \"company\": \"Apple Inc.\",\n", + " \"price\": 175.50,\n", + " \"change\": 2.30,\n", + " \"change_percent\": 1.33,\n", + " \"volume\": 45000000,\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"sector\": \"Technology\"\n", + " },\n", + " {\n", + " \"symbol\": \"MSFT\",\n", + " \"company\": \"Microsoft Corporation\",\n", + " \"price\": 380.25,\n", + " \"change\": -1.50,\n", + " \"change_percent\": -0.39,\n", + " \"volume\": 28000000,\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"sector\": \"Technology\"\n", + " },\n", + " {\n", + " \"symbol\": \"GOOGL\",\n", + " \"company\": \"Alphabet Inc.\",\n", + " \"price\": 142.80,\n", + " \"change\": 3.20,\n", + " \"change_percent\": 2.29,\n", + " \"volume\": 32000000,\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=30)).isoformat(),\n", + " \"sector\": \"Technology\"\n", + " }\n", + "]\n", + "\n", + "with open(market_data_json, 'w') as f:\n", + " json.dump(market_data, f, indent=2)\n", + "\n", + "# CSV format market data (common in financial data exports)\n", + "market_data_csv = os.path.join(temp_dir, \"market_data.csv\")\n", + "csv_content = \"\"\"symbol,company,price,change,volume,timestamp,sector\n", + "TSLA,Tesla Inc.,245.60,5.40,55000000,2024-01-15T10:00:00,Automotive\n", + "AMZN,Amazon.com Inc.,155.30,1.20,42000000,2024-01-15T10:00:00,Retail\n", + "NVDA,NVIDIA Corporation,520.75,12.50,68000000,2024-01-15T10:00:00,Technology\"\"\"\n", + "\n", + "with open(market_data_csv, 'w') as f:\n", + " f.write(csv_content)\n", + "\n", + "# Ingest from files\n", + "file_objects_json = file_ingestor.ingest_file(market_data_json, read_content=True)\n", + "file_objects_csv = file_ingestor.ingest_file(market_data_csv, read_content=True)\n", + "\n", + "# Parse structured data\n", + "parsed_json = json_parser.parse(market_data_json)\n", + "parsed_csv = csv_parser.parse(market_data_csv)\n", + "\n", + "# Real financial news feed URLs\n", + "financial_feeds = [\n", + " \"https://feeds.reuters.com/reuters/businessNews\", # Reuters Business\n", + " \"https://feeds.reuters.com/reuters/topNews\", # Reuters Top News\n", + " \"https://rss.cnn.com/rss/money_latest.rss\", # CNN Money\n", + " \"https://feeds.bloomberg.com/markets/news.rss\", # Bloomberg Markets\n", + " \"https://www.ft.com/?format=rss\" # Financial Times\n", + "]\n", + "\n", + "financial_feed_list = []\n", + "for feed_url in financial_feeds:\n", + " try:\n", + " financial_feed = feed_ingestor.ingest_feed(feed_url)\n", + " if financial_feed:\n", + " financial_feed_list.append(financial_feed)\n", + " print(f\"✓ Ingested financial feed: {financial_feed.title if hasattr(financial_feed, 'title') else feed_url}\")\n", + " print(f\" Items: {len(financial_feed.items) if hasattr(financial_feed, 'items') else 0}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Real financial API endpoints (examples - require API keys)\n", + "financial_apis = [\n", + " \"https://api.polygon.io/v2/aggs/ticker/AAPL/range/1/day/2024-01-01/2024-01-31\", # Polygon.io (requires API key)\n", + " \"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol=AAPL&interval=5min&apikey=demo\", # Alpha Vantage\n", + " \"https://api.github.com/repos/ranaroussi/yfinance\" # Yahoo Finance API wrapper\n", + "]\n", + "\n", + "web_content_list = []\n", + "for api_url in financial_apis[:1]: # Process first API\n", + " try:\n", + " web_content = web_ingestor.ingest_url(api_url)\n", + " if web_content:\n", + " web_content_list.append(web_content)\n", + " print(f\"✓ Ingested API content: {web_content.url if hasattr(web_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "# Real database connection for market data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/market_data_db\"\n", + "db_query = \"SELECT symbol, price, volume, timestamp FROM market_data WHERE timestamp > NOW() - INTERVAL '1 day' ORDER BY timestamp DESC\"\n", + "\n", + "# Optional: Ingest from MCP server\n", + "# Users can bring their own financial data MCP server via URL\n", + "mcp_market_data = []\n", + "try:\n", + " # Connect to financial data MCP server via URL\n", + " # Example: http://localhost:8000/mcp or https://api.example.com/financial-mcp\n", + " financial_mcp_url = \"http://localhost:8000/mcp\" # Replace with your MCP server URL\n", + " \n", + " mcp_ingestor.connect(\n", + " \"market_mcp_server\",\n", + " url=financial_mcp_url,\n", + " headers={\"Authorization\": \"Bearer your_token\"} if \"api.example.com\" in financial_mcp_url else {}\n", + " )\n", + " \n", + " # Ingest market data from MCP server resources or tools\n", + " mcp_data = mcp_ingestor.ingest_resources(\n", + " \"market_mcp_server\",\n", + " resource_uris=[\"resource://market_data/daily\"]\n", + " )\n", + " mcp_market_data.extend(mcp_data)\n", + " \n", + " # Or use tool-based ingestion\n", + " tool_data = mcp_ingestor.ingest_tool_output(\n", + " \"market_mcp_server\",\n", + " tool_name=\"get_stock_prices\",\n", + " arguments={\"symbols\": [\"AAPL\", \"MSFT\", \"GOOGL\"], \"date\": datetime.now().isoformat()}\n", + " )\n", + " if tool_data:\n", + " mcp_market_data.append(tool_data)\n", + " \n", + " # Parse MCP responses\n", + " for mcp_item in mcp_market_data:\n", + " parsed_mcp = mcp_parser.parse_response(mcp_item, response_type=\"json\")\n", + " if isinstance(parsed_mcp, dict) and \"stock_prices\" in parsed_mcp:\n", + " # Merge MCP data with existing market data\n", + " market_data.extend(parsed_mcp.get(\"stock_prices\", []))\n", + " \n", + " print(f\"✓ Ingested {len(mcp_market_data)} items from MCP server\")\n", + " mcp_ingestor.disconnect(\"market_mcp_server\")\n", + "except Exception as e:\n", + " print(f\"⚠ MCP ingestion skipped: {e}\")\n", + " print(\" Note: MCP ingestion is optional. You can bring your own MCP server via URL.\")\n", + "\n", + "print(f\"Ingested {len([file_objects_json]) if file_objects_json else 0} JSON market data files\")\n", + "print(f\"Ingested {len([file_objects_csv]) if file_objects_csv else 0} CSV market data files\")\n", + "print(f\"Parsed {len(parsed_json.data) if parsed_json and parsed_json.data else 0} JSON market entries\")\n", + "print(f\"Parsed {len(parsed_csv.rows) if parsed_csv else 0} CSV market rows\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Market Entities and Relationships\n", + "\n", + "Extract market entities (companies, stocks, sectors) and relationships from market data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "market_entities = []\n", + "market_relationships = []\n", + "\n", + "# Extract from JSON data\n", + "if parsed_json and parsed_json.data:\n", + " for entry in parsed_json.data:\n", + " if isinstance(entry, dict):\n", + " market_entities.append({\n", + " \"id\": entry.get(\"symbol\", \"\"),\n", + " \"type\": \"Stock\",\n", + " \"name\": entry.get(\"symbol\", \"\"),\n", + " \"properties\": {\n", + " \"price\": entry.get(\"price\", 0),\n", + " \"change\": entry.get(\"change\", 0),\n", + " \"change_percent\": entry.get(\"change_percent\", 0),\n", + " \"volume\": entry.get(\"volume\", 0),\n", + " \"timestamp\": entry.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " market_entities.append({\n", + " \"id\": entry.get(\"company\", \"\"),\n", + " \"type\": \"Company\",\n", + " \"name\": entry.get(\"company\", \"\"),\n", + " \"properties\": {}\n", + " })\n", + " market_entities.append({\n", + " \"id\": entry.get(\"sector\", \"\"),\n", + " \"type\": \"Sector\",\n", + " \"name\": entry.get(\"sector\", \"\"),\n", + " \"properties\": {}\n", + " })\n", + " \n", + " market_relationships.append({\n", + " \"source\": entry.get(\"symbol\", \"\"),\n", + " \"target\": entry.get(\"company\", \"\"),\n", + " \"type\": \"ticker_for\",\n", + " \"properties\": {\"timestamp\": entry.get(\"timestamp\", \"\")}\n", + " })\n", + " market_relationships.append({\n", + " \"source\": entry.get(\"company\", \"\"),\n", + " \"target\": entry.get(\"sector\", \"\"),\n", + " \"type\": \"belongs_to\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "# Extract from CSV data\n", + "if parsed_csv and parsed_csv.rows:\n", + " for row in parsed_csv.rows:\n", + " if isinstance(row, dict):\n", + " market_entities.append({\n", + " \"id\": row.get(\"symbol\", \"\"),\n", + " \"type\": \"Stock\",\n", + " \"name\": row.get(\"symbol\", \"\"),\n", + " \"properties\": {\n", + " \"price\": float(row.get(\"price\", 0)) if row.get(\"price\") else 0,\n", + " \"change\": float(row.get(\"change\", 0)) if row.get(\"change\") else 0,\n", + " \"volume\": int(row.get(\"volume\", 0)) if row.get(\"volume\") else 0,\n", + " \"timestamp\": row.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + "\n", + "print(f\"Extracted {len(market_entities)} market entities\")\n", + "print(f\"Extracted {len(market_relationships)} market relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Temporal Market Knowledge Graph\n", + "\n", + "Build a temporal knowledge graph from market data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "market_kg = builder.build(market_entities, market_relationships)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(market_kg)\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "centrality_scores = centrality_calculator.calculate_centrality(market_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(market_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(market_kg)\n", + "\n", + "print(f\"Built temporal market knowledge graph\")\n", + "print(f\" Entities: {len(market_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(market_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Communities: {len(communities)}\")\n", + "print(f\" Central entities: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze Market Trends\n", + "\n", + "Analyze market trends using temporal queries and pattern detection.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "start_time = (datetime.now() - timedelta(days=7)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=market_kg,\n", + " query=\"Find market movements in the last 7 days\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " market_kg,\n", + " pattern_type=\"trend\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Market analysis rules\n", + "inference_engine.add_rule(\"IF change_percent > 2 AND volume > 40000000 THEN strong_momentum\")\n", + "inference_engine.add_rule(\"IF change_percent < -1 AND volume > 50000000 THEN selling_pressure\")\n", + "\n", + "# Add facts from market data\n", + "for entry in parsed_json.data if parsed_json and parsed_json.data else []:\n", + " if isinstance(entry, dict):\n", + " inference_engine.add_fact({\n", + " \"symbol\": entry.get(\"symbol\", \"\"),\n", + " \"change_percent\": entry.get(\"change_percent\", 0),\n", + " \"volume\": entry.get(\"volume\", 0)\n", + " })\n", + "\n", + "market_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Temporal query returned {len(temporal_results.get('entities', []))} entities\")\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"Generated {len(market_insights)} market insights\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Market Intelligence Reports\n", + "\n", + "Generate comprehensive market intelligence reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "json_exporter.export_knowledge_graph(market_kg, os.path.join(temp_dir, \"market_kg.json\"))\n", + "csv_exporter.export_entities(market_entities, os.path.join(temp_dir, \"market_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(market_kg, os.path.join(temp_dir, \"market_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Market intelligence analysis identified {len(market_insights)} insights and {len(temporal_patterns)} trends\",\n", + " \"stocks_analyzed\": len(market_entities),\n", + " \"patterns\": len(temporal_patterns),\n", + " \"insights\": len(market_insights),\n", + " \"sectors\": len(set([e.get(\"properties\", {}).get(\"sector\", \"\") for e in market_entities if e.get(\"type\") == \"Stock\"]))\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"Generated market intelligence report\")\n", + "print(f\"Report length: {len(report)} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Visualize Market Intelligence\n", + "\n", + "Visualize market knowledge graph and trends.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(market_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(market_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(market_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated visualizations for market knowledge graph, temporal trends, and analytics\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Multiple Market Sources → Parse → Extract → Temporal KG → Analyze Trends → Generate Insights → Export → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/finance/Regulatory_Compliance.ipynb b/docs/cookbook/use_cases/finance/Regulatory_Compliance.ipynb new file mode 100644 index 00000000..32f4664f --- /dev/null +++ b/docs/cookbook/use_cases/finance/Regulatory_Compliance.ipynb @@ -0,0 +1,370 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Regulatory Compliance Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete regulatory compliance pipeline: ingest regulatory documents from multiple sources (SEC, FINRA, regulatory databases), extract compliance rules, build compliance ontology, validate compliance, and generate compliance reports.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: DocumentParser, PDFParser, HTMLParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ValidationEngine, ConflictDetector\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Regulatory Documents → Parse → Extract Compliance Rules → Build Compliance Ontology → Validate Compliance → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Regulatory Documents from Multiple Sources\n", + "\n", + "Ingest regulatory documents from SEC, FINRA, and regulatory databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import DocumentParser, PDFParser, HTMLParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor, ValidationEngine\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "document_parser = DocumentParser()\n", + "pdf_parser = PDFParser()\n", + "html_parser = HTMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real regulatory data sources\n", + "regulatory_sources = [\n", + " \"https://www.sec.gov/rules/final.shtml\", # SEC Final Rules\n", + " \"https://www.finra.org/rules-guidance\", # FINRA Rules\n", + " \"https://www.federalreserve.gov/newsevents/pressreleases.htm\" # Federal Reserve\n", + "]\n", + "\n", + "regulatory_feeds = [\n", + " \"https://feeds.reuters.com/reuters/businessNews\",\n", + " \"https://rss.cnn.com/rss/money_latest.rss\"\n", + "]\n", + "\n", + "# Real database connection for regulatory documents\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/regulatory_db\"\n", + "db_query = \"SELECT regulation_id, title, effective_date, compliance_requirements FROM regulations WHERE effective_date > CURRENT_DATE - INTERVAL '1 year' ORDER BY effective_date DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample regulatory document data\n", + "regulatory_file = os.path.join(temp_dir, \"regulatory_document.json\")\n", + "regulatory_data = {\n", + " \"regulation_id\": \"REG-2024-001\",\n", + " \"title\": \"Data Privacy and Security Requirements\",\n", + " \"effective_date\": (datetime.now() - timedelta(days=60)).isoformat(),\n", + " \"compliance_requirements\": [\n", + " \"Encrypt sensitive customer data\",\n", + " \"Maintain audit logs for 7 years\",\n", + " \"Report breaches within 72 hours\",\n", + " \"Conduct annual security assessments\"\n", + " ],\n", + " \"applicable_entities\": [\"Financial Institutions\", \"Broker-Dealers\", \"Investment Advisors\"],\n", + " \"penalties\": {\n", + " \"non_compliance\": \"Fines up to $1M per violation\",\n", + " \"willful_violation\": \"Criminal penalties\"\n", + " }\n", + "}\n", + "\n", + "with open(regulatory_file, 'w') as f:\n", + " json.dump(regulatory_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(regulatory_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(regulatory_file)\n", + "\n", + "# Ingest from regulatory sources\n", + "regulatory_web_list = []\n", + "for source_url in regulatory_sources[:1]:\n", + " try:\n", + " web_content = web_ingestor.ingest_url(source_url)\n", + " if web_content:\n", + " regulatory_web_list.append(web_content)\n", + " print(f\"✓ Ingested regulatory source: {web_content.url if hasattr(web_content, 'url') else source_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Regulatory source ingestion for {source_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Regulatory documents: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Regulatory web sources: {len(regulatory_web_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Compliance Rules\n", + "\n", + "Extract compliance rules and requirements from regulatory documents.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "compliance_entities = []\n", + "compliance_relationships = []\n", + "\n", + "# Extract from regulatory data\n", + "if parsed_data and parsed_data.data:\n", + " regulation = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " \n", + " if isinstance(regulation, dict):\n", + " # Regulation entity\n", + " compliance_entities.append({\n", + " \"id\": regulation.get(\"regulation_id\", \"\"),\n", + " \"type\": \"Regulation\",\n", + " \"name\": regulation.get(\"title\", \"\"),\n", + " \"properties\": {\n", + " \"effective_date\": regulation.get(\"effective_date\", \"\"),\n", + " \"regulation_id\": regulation.get(\"regulation_id\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Compliance requirements\n", + " for i, requirement in enumerate(regulation.get(\"compliance_requirements\", [])):\n", + " compliance_entities.append({\n", + " \"id\": f\"{regulation.get('regulation_id', '')}_req_{i}\",\n", + " \"type\": \"Compliance_Requirement\",\n", + " \"name\": requirement,\n", + " \"properties\": {}\n", + " })\n", + " compliance_relationships.append({\n", + " \"source\": regulation.get(\"regulation_id\", \"\"),\n", + " \"target\": f\"{regulation.get('regulation_id', '')}_req_{i}\",\n", + " \"type\": \"has_requirement\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Applicable entities\n", + " for entity_type in regulation.get(\"applicable_entities\", []):\n", + " compliance_entities.append({\n", + " \"id\": entity_type,\n", + " \"type\": \"Regulated_Entity\",\n", + " \"name\": entity_type,\n", + " \"properties\": {}\n", + " })\n", + " compliance_relationships.append({\n", + " \"source\": regulation.get(\"regulation_id\", \"\"),\n", + " \"target\": entity_type,\n", + " \"type\": \"applies_to\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "print(f\"Extracted {len(compliance_entities)} compliance entities\")\n", + "print(f\"Extracted {len(compliance_relationships)} compliance relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Compliance Ontology\n", + "\n", + "Build compliance ontology from extracted rules and requirements.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "compliance_kg = builder.build(compliance_entities, compliance_relationships)\n", + "\n", + "compliance_ontology = ontology_generator.generate(compliance_entities, compliance_relationships)\n", + "\n", + "classes = class_inferrer.infer_classes(compliance_entities)\n", + "properties = property_generator.infer_properties(compliance_entities, compliance_relationships, classes)\n", + "\n", + "validation_result = ontology_validator.validate_ontology(compliance_ontology)\n", + "\n", + "print(f\"Built compliance knowledge graph\")\n", + "print(f\" Entities: {len(compliance_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(compliance_kg.get('relationships', []))}\")\n", + "print(f\"Generated compliance ontology\")\n", + "print(f\" Classes: {len(compliance_ontology.get('classes', []))}\")\n", + "print(f\" Properties: {len(compliance_ontology.get('properties', []))}\")\n", + "print(f\" Ontology valid: {validation_result.valid}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Validate Compliance\n", + "\n", + "Validate data against compliance rules using inference engine.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "validation_engine = ValidationEngine()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "# Define compliance validation rules\n", + "inference_engine.add_rule(\"IF data_encrypted is true AND audit_logs_maintained is true THEN data_security_compliant\")\n", + "inference_engine.add_rule(\"IF breach_reported_within_72h is true THEN breach_reporting_compliant\")\n", + "inference_engine.add_rule(\"IF annual_assessment_conducted is true THEN assessment_compliant\")\n", + "\n", + "# Sample data to validate\n", + "sample_data = {\n", + " \"data_encrypted\": True,\n", + " \"audit_logs_maintained\": True,\n", + " \"breach_reported_within_72h\": True,\n", + " \"annual_assessment_conducted\": True\n", + "}\n", + "\n", + "# Add facts for validation\n", + "for key, value in sample_data.items():\n", + " inference_engine.add_fact({key: value})\n", + "\n", + "compliance_status = inference_engine.forward_chain()\n", + "\n", + "# Analyze compliance graph\n", + "metrics = graph_analyzer.compute_metrics(compliance_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(compliance_kg)\n", + "\n", + "print(f\"Compliance validation complete\")\n", + "print(f\" Compliance status: {len(compliance_status)} rules satisfied\")\n", + "print(f\" Graph metrics: density {metrics.get('density', 0):.3f}\")\n", + "print(f\" Connected components: {len(connectivity.get('components', []))}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Compliance Reports\n", + "\n", + "Generate comprehensive compliance reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(compliance_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(compliance_kg, os.path.join(temp_dir, \"compliance_kg.json\"))\n", + "csv_exporter.export_entities(compliance_entities, os.path.join(temp_dir, \"compliance_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(compliance_kg, os.path.join(temp_dir, \"compliance_kg.rdf\"))\n", + "owl_exporter.export(compliance_ontology, os.path.join(temp_dir, \"compliance_ontology.owl\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Compliance validation identified {len(compliance_status)} satisfied rules\",\n", + " \"regulations_analyzed\": len([e for e in compliance_entities if e.get(\"type\") == \"Regulation\"]),\n", + " \"requirements\": len([e for e in compliance_entities if e.get(\"type\") == \"Compliance_Requirement\"]),\n", + " \"compliance_status\": len(compliance_status),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"Generated compliance report\")\n", + "print(f\"Report length: {len(report)} characters\")\n", + "print(f\"Graph quality score: {quality_score.get('overall_score', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Visualize Compliance\n", + "\n", + "Visualize compliance knowledge graph and ontology.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(compliance_kg, output=\"interactive\")\n", + "ontology_viz = ontology_visualizer.visualize_hierarchy(compliance_ontology, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(compliance_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated visualizations for compliance knowledge graph, ontology, and analytics\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Regulatory Documents → Parse → Extract Rules → Build Ontology → Validate Compliance → Generate Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/healthcare/Clinical_Reports_Processing.ipynb b/docs/cookbook/use_cases/healthcare/Clinical_Reports_Processing.ipynb new file mode 100644 index 00000000..4582d85d --- /dev/null +++ b/docs/cookbook/use_cases/healthcare/Clinical_Reports_Processing.ipynb @@ -0,0 +1,423 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Clinical Reports Processing Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete clinical reports processing pipeline: ingest clinical documents from multiple sources (EHR systems, HL7/FHIR APIs, medical databases), extract medical entities, build knowledge graph, store in triple store, and query patient data.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, DBIngestor, StreamIngestor, WebIngestor, MCPIngestor\n", + "- **Parsing**: DocumentParser, PDFParser, StructuredDataParser, CSVParser, MCPParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, CoreferenceResolver, TripleExtractor\n", + "- **KG**: GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n", + "- **Triple Store**: TripleStore, TripleManager, QueryEngine\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ValidationEngine\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Clinical Documents (Files, APIs, DB, MCP) → Parse → Extract Medical Entities → Build Medical KG → Store in Triple Store → Query Patient Data → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Clinical Documents from Multiple Sources\n", + "\n", + "Ingest clinical documents from EHR systems, HL7/FHIR APIs, and medical databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, DBIngestor, StreamIngestor, WebIngestor, MCPIngestor, ingest_mcp\n", + "from semantica.parse import DocumentParser, PDFParser, StructuredDataParser, CSVParser, MCPParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, CoreferenceResolver, TripleExtractor\n", + "from semantica.kg import GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n", + "from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor, ValidationEngine\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "stream_ingestor = StreamIngestor()\n", + "web_ingestor = WebIngestor()\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "document_parser = DocumentParser()\n", + "pdf_parser = PDFParser()\n", + "structured_parser = StructuredDataParser()\n", + "csv_parser = CSVParser()\n", + "mcp_parser = MCPParser()\n", + "\n", + "# Real healthcare data sources\n", + "healthcare_apis = [\n", + " \"https://api.logicahealth.org/fhir/R4/Patient\", # Logica Health FHIR API\n", + " \"https://hapi.fhir.org/baseR4/Patient\", # HAPI FHIR Server\n", + " \"https://api.logicahealth.org/fhir/R4/Observation\" # FHIR Observations\n", + "]\n", + "\n", + "medical_feeds = [\n", + " \"https://www.cdc.gov/rss.xml\", # CDC Health Alerts\n", + " \"https://www.who.int/rss-feeds/news-english.xml\" # WHO News\n", + "]\n", + "\n", + "# Real database connection for clinical records (HIPAA compliant)\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/clinical_records_db\"\n", + "db_query = \"SELECT patient_id, visit_date, diagnosis, medication, procedure, doctor FROM clinical_visits WHERE visit_date > CURRENT_DATE - INTERVAL '1 year' ORDER BY visit_date DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample clinical report data\n", + "clinical_report_file = os.path.join(temp_dir, \"clinical_report.json\")\n", + "clinical_data = {\n", + " \"patient_id\": \"P001\",\n", + " \"visit_date\": (datetime.now() - timedelta(days=30)).isoformat(),\n", + " \"diagnosis\": [\"Hypertension\", \"Type 2 Diabetes\"],\n", + " \"medications\": [\"Lisinopril 10mg\", \"Metformin 500mg\"],\n", + " \"procedures\": [\"Blood Pressure Check\", \"HbA1c Test\"],\n", + " \"doctor\": \"Dr. Smith\",\n", + " \"notes\": \"Patient shows improvement in blood pressure control. Continue current medications.\"\n", + "}\n", + "\n", + "with open(clinical_report_file, 'w') as f:\n", + " json.dump(clinical_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(clinical_report_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(clinical_report_file)\n", + "\n", + "# Ingest from FHIR APIs\n", + "fhir_content_list = []\n", + "for api_url in healthcare_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " fhir_content_list.append(api_content)\n", + " print(f\"✓ Ingested FHIR API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ FHIR API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "# Optional: Ingest from MCP server\n", + "# Users can bring their own medical database MCP server via URL\n", + "mcp_clinical_data = []\n", + "try:\n", + " # Connect to medical database MCP server via URL\n", + " # Example: http://localhost:8000/mcp or https://api.example.com/medical-mcp\n", + " medical_mcp_url = \"http://localhost:8000/mcp\" # Replace with your MCP server URL\n", + " \n", + " mcp_ingestor.connect(\n", + " \"clinical_mcp_server\",\n", + " url=medical_mcp_url,\n", + " headers={\n", + " \"Authorization\": \"Bearer your_token\",\n", + " \"X-API-Key\": \"your_api_key\"\n", + " } if \"api.example.com\" in medical_mcp_url else {}\n", + " )\n", + " \n", + " # Ingest patient records from MCP server\n", + " mcp_data = mcp_ingestor.ingest_resources(\n", + " \"clinical_mcp_server\",\n", + " resource_uris=[\"resource://patients/records\"]\n", + " )\n", + " mcp_clinical_data.extend(mcp_data)\n", + " \n", + " # Or use tool-based ingestion to query patient records\n", + " tool_data = mcp_ingestor.ingest_tool_output(\n", + " \"clinical_mcp_server\",\n", + " tool_name=\"query_patient_records\",\n", + " arguments={\n", + " \"patient_id\": \"P001\",\n", + " \"date_range\": {\n", + " \"start\": (datetime.now() - timedelta(days=365)).isoformat(),\n", + " \"end\": datetime.now().isoformat()\n", + " }\n", + " }\n", + " )\n", + " if tool_data:\n", + " mcp_clinical_data.append(tool_data)\n", + " \n", + " # Parse MCP responses and merge with existing clinical data\n", + " for mcp_item in mcp_clinical_data:\n", + " parsed_mcp = mcp_parser.parse_response(mcp_item, response_type=\"json\")\n", + " if isinstance(parsed_mcp, dict):\n", + " if \"patient_records\" in parsed_mcp:\n", + " # Merge patient records from MCP\n", + " if isinstance(parsed_data.data, list):\n", + " parsed_data.data.extend(parsed_mcp.get(\"patient_records\", []))\n", + " elif isinstance(parsed_data.data, dict):\n", + " parsed_data.data = [parsed_data.data] + parsed_mcp.get(\"patient_records\", [])\n", + " \n", + " print(f\"✓ Ingested {len(mcp_clinical_data)} items from MCP server\")\n", + " mcp_ingestor.disconnect(\"clinical_mcp_server\")\n", + "except Exception as e:\n", + " print(f\"⚠ MCP ingestion skipped: {e}\")\n", + " print(\" Note: MCP ingestion is optional. You can bring your own MCP server via URL.\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Clinical reports: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" FHIR API sources: {len(fhir_content_list)}\")\n", + "print(f\" Database sources: 1\")\n", + "print(f\" MCP server sources: {len(mcp_clinical_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Medical Entities\n", + "\n", + "Extract medical entities (conditions, medications, procedures, doctors) from clinical reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "coreference_resolver = CoreferenceResolver()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "medical_entities = []\n", + "relationships = []\n", + "\n", + "# Extract from clinical data\n", + "if parsed_data and parsed_data.data:\n", + " clinical = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " \n", + " if isinstance(clinical, dict):\n", + " patient_id = clinical.get(\"patient_id\", \"\")\n", + " \n", + " medical_entities.append({\n", + " \"id\": patient_id,\n", + " \"type\": \"Patient\",\n", + " \"name\": patient_id,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Diagnoses\n", + " for diagnosis in clinical.get(\"diagnosis\", []):\n", + " medical_entities.append({\n", + " \"id\": diagnosis,\n", + " \"type\": \"Diagnosis\",\n", + " \"name\": diagnosis,\n", + " \"properties\": {}\n", + " })\n", + " relationships.append({\n", + " \"source\": patient_id,\n", + " \"target\": diagnosis,\n", + " \"type\": \"has_diagnosis\",\n", + " \"properties\": {\"timestamp\": clinical.get(\"visit_date\", \"\")}\n", + " })\n", + " \n", + " # Medications\n", + " for medication in clinical.get(\"medications\", []):\n", + " medical_entities.append({\n", + " \"id\": medication,\n", + " \"type\": \"Medication\",\n", + " \"name\": medication,\n", + " \"properties\": {}\n", + " })\n", + " relationships.append({\n", + " \"source\": patient_id,\n", + " \"target\": medication,\n", + " \"type\": \"prescribed\",\n", + " \"properties\": {\"timestamp\": clinical.get(\"visit_date\", \"\")}\n", + " })\n", + " \n", + " # Procedures\n", + " for procedure in clinical.get(\"procedures\", []):\n", + " medical_entities.append({\n", + " \"id\": procedure,\n", + " \"type\": \"Procedure\",\n", + " \"name\": procedure,\n", + " \"properties\": {}\n", + " })\n", + " relationships.append({\n", + " \"source\": patient_id,\n", + " \"target\": procedure,\n", + " \"type\": \"underwent\",\n", + " \"properties\": {\"timestamp\": clinical.get(\"visit_date\", \"\")}\n", + " })\n", + " \n", + " # Doctor\n", + " doctor = clinical.get(\"doctor\", \"\")\n", + " if doctor:\n", + " medical_entities.append({\n", + " \"id\": doctor,\n", + " \"type\": \"Doctor\",\n", + " \"name\": doctor,\n", + " \"properties\": {}\n", + " })\n", + " relationships.append({\n", + " \"source\": doctor,\n", + " \"target\": patient_id,\n", + " \"type\": \"treats\",\n", + " \"properties\": {\"timestamp\": clinical.get(\"visit_date\", \"\")}\n", + " })\n", + "\n", + "print(f\"Extracted {len(medical_entities)} medical entities\")\n", + "print(f\"Extracted {len(relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Medical Knowledge Graph\n", + "\n", + "Build knowledge graph from medical entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "entity_resolver = EntityResolver()\n", + "graph_validator = GraphValidator()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "resolved_entities = entity_resolver.resolve(medical_entities)\n", + "\n", + "medical_kg = builder.build(resolved_entities, relationships)\n", + "\n", + "validation_result = graph_validator.validate(medical_kg)\n", + "metrics = graph_analyzer.compute_metrics(medical_kg)\n", + "\n", + "print(f\"Built medical knowledge graph\")\n", + "print(f\" Entities: {len(medical_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(medical_kg.get('relationships', []))}\")\n", + "print(f\" Graph valid: {validation_result.get('valid', False)}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Store in Triple Store and Query Patient Data\n", + "\n", + "Store knowledge graph in triple store and query patient information.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "triple_store = TripleStore()\n", + "triple_manager = TripleManager()\n", + "query_engine = QueryEngine()\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "triple_store.store_knowledge_graph(medical_kg)\n", + "\n", + "# Query patient data\n", + "patient_id = \"P001\"\n", + "patient_query = f\"SELECT * WHERE {{ ?patient :hasDiagnosis ?diagnosis . ?patient :prescribed ?medication }}\"\n", + "\n", + "query_results = query_engine.query(patient_query, knowledge_graph=medical_kg)\n", + "\n", + "# Medical inference rules\n", + "inference_engine.add_rule(\"IF patient has_diagnosis Hypertension AND patient prescribed Lisinopril THEN treatment_appropriate\")\n", + "inference_engine.add_rule(\"IF patient has_diagnosis Diabetes AND patient prescribed Metformin THEN treatment_appropriate\")\n", + "\n", + "for relationship in relationships:\n", + " if relationship.get(\"type\") == \"has_diagnosis\":\n", + " inference_engine.add_fact({\n", + " \"patient\": relationship.get(\"source\"),\n", + " \"diagnosis\": relationship.get(\"target\")\n", + " })\n", + " if relationship.get(\"type\") == \"prescribed\":\n", + " inference_engine.add_fact({\n", + " \"patient\": relationship.get(\"source\"),\n", + " \"medication\": relationship.get(\"target\")\n", + " })\n", + "\n", + "medical_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Stored medical knowledge graph in triple store\")\n", + "print(f\"Query returned {len(query_results) if query_results else 0} results\")\n", + "print(f\"Generated {len(medical_insights)} medical insights\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate clinical reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(medical_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(medical_kg, os.path.join(temp_dir, \"clinical_kg.json\"))\n", + "rdf_exporter.export_knowledge_graph(medical_kg, os.path.join(temp_dir, \"clinical_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Clinical reports processing identified {len(medical_entities)} entities and {len(medical_insights)} insights\",\n", + " \"patients_processed\": len([e for e in medical_entities if e.get(\"type\") == \"Patient\"]),\n", + " \"diagnoses\": len([e for e in medical_entities if e.get(\"type\") == \"Diagnosis\"]),\n", + " \"medications\": len([e for e in medical_entities if e.get(\"type\") == \"Medication\"]),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(medical_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(medical_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated clinical reports and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Clinical Documents → Parse → Extract → Build KG → Triple Store → Query → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/healthcare/Disease_Network_Analysis.ipynb b/docs/cookbook/use_cases/healthcare/Disease_Network_Analysis.ipynb new file mode 100644 index 00000000..ca640702 --- /dev/null +++ b/docs/cookbook/use_cases/healthcare/Disease_Network_Analysis.ipynb @@ -0,0 +1,405 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Disease Network Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete disease network analysis pipeline: ingest disease data from multiple sources (medical literature, research databases, clinical trials), extract disease relationships, build disease ontology, analyze networks, and predict outcomes.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: DocumentParser, PDFParser, StructuredDataParser, JSONParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Disease Data Sources → Parse → Extract Disease Relationships → Build Disease Ontology → Analyze Networks → Predict Outcomes → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Disease Data from Multiple Sources\n", + "\n", + "Ingest disease data from medical literature, research databases, and clinical trials.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import DocumentParser, PDFParser, StructuredDataParser, JSONParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "document_parser = DocumentParser()\n", + "pdf_parser = PDFParser()\n", + "structured_parser = StructuredDataParser()\n", + "json_parser = JSONParser()\n", + "\n", + "# Real disease data sources\n", + "disease_apis = [\n", + " \"https://api.logicahealth.org/fhir/R4/Condition\", # FHIR Conditions\n", + " \"https://hapi.fhir.org/baseR4/Condition\", # HAPI FHIR Conditions\n", + " \"https://www.ncbi.nlm.nih.gov/books/NBK5197/\" # NCBI Medical Literature\n", + "]\n", + "\n", + "medical_feeds = [\n", + " \"https://www.cdc.gov/rss.xml\", # CDC Health Alerts\n", + " \"https://www.who.int/rss-feeds/news-english.xml\" # WHO News\n", + "]\n", + "\n", + "# Real database connection for disease data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/disease_db\"\n", + "db_query = \"SELECT disease_name, icd10_code, related_diseases, symptoms, treatments FROM diseases WHERE last_updated > CURRENT_DATE - INTERVAL '1 year' ORDER BY disease_name\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample disease data\n", + "disease_file = os.path.join(temp_dir, \"disease_data.json\")\n", + "disease_data = {\n", + " \"diseases\": [\n", + " {\n", + " \"disease_name\": \"Type 2 Diabetes\",\n", + " \"icd10_code\": \"E11\",\n", + " \"related_diseases\": [\"Hypertension\", \"Cardiovascular Disease\", \"Obesity\"],\n", + " \"symptoms\": [\"Increased thirst\", \"Frequent urination\", \"Fatigue\"],\n", + " \"treatments\": [\"Metformin\", \"Insulin\", \"Lifestyle changes\"],\n", + " \"prevalence\": \"High\"\n", + " },\n", + " {\n", + " \"disease_name\": \"Hypertension\",\n", + " \"icd10_code\": \"I10\",\n", + " \"related_diseases\": [\"Type 2 Diabetes\", \"Cardiovascular Disease\", \"Kidney Disease\"],\n", + " \"symptoms\": [\"High blood pressure\", \"Headaches\", \"Dizziness\"],\n", + " \"treatments\": [\"ACE inhibitors\", \"Beta blockers\", \"Lifestyle changes\"],\n", + " \"prevalence\": \"Very High\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "with open(disease_file, 'w') as f:\n", + " json.dump(disease_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(disease_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(disease_file)\n", + "\n", + "# Ingest from disease APIs\n", + "disease_api_list = []\n", + "for api_url in disease_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " disease_api_list.append(api_content)\n", + " print(f\"✓ Ingested disease API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Disease API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Disease data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Disease API sources: {len(disease_api_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Disease Relationships\n", + "\n", + "Extract disease entities and relationships from disease data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "disease_entities = []\n", + "disease_relationships = []\n", + "\n", + "# Extract from disease data\n", + "if parsed_data and parsed_data.data:\n", + " diseases = parsed_data.data.get(\"diseases\", []) if isinstance(parsed_data.data, dict) else parsed_data.data if isinstance(parsed_data.data, list) else []\n", + " \n", + " for disease in diseases:\n", + " if isinstance(disease, dict):\n", + " disease_name = disease.get(\"disease_name\", \"\")\n", + " \n", + " disease_entities.append({\n", + " \"id\": disease_name,\n", + " \"type\": \"Disease\",\n", + " \"name\": disease_name,\n", + " \"properties\": {\n", + " \"icd10_code\": disease.get(\"icd10_code\", \"\"),\n", + " \"prevalence\": disease.get(\"prevalence\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Related diseases\n", + " for related in disease.get(\"related_diseases\", []):\n", + " disease_entities.append({\n", + " \"id\": related,\n", + " \"type\": \"Disease\",\n", + " \"name\": related,\n", + " \"properties\": {}\n", + " })\n", + " disease_relationships.append({\n", + " \"source\": disease_name,\n", + " \"target\": related,\n", + " \"type\": \"related_to\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Symptoms\n", + " for symptom in disease.get(\"symptoms\", []):\n", + " disease_entities.append({\n", + " \"id\": symptom,\n", + " \"type\": \"Symptom\",\n", + " \"name\": symptom,\n", + " \"properties\": {}\n", + " })\n", + " disease_relationships.append({\n", + " \"source\": disease_name,\n", + " \"target\": symptom,\n", + " \"type\": \"has_symptom\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Treatments\n", + " for treatment in disease.get(\"treatments\", []):\n", + " disease_entities.append({\n", + " \"id\": treatment,\n", + " \"type\": \"Treatment\",\n", + " \"name\": treatment,\n", + " \"properties\": {}\n", + " })\n", + " disease_relationships.append({\n", + " \"source\": disease_name,\n", + " \"target\": treatment,\n", + " \"type\": \"treated_with\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "print(f\"Extracted {len(disease_entities)} disease entities\")\n", + "print(f\"Extracted {len(disease_relationships)} disease relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Disease Ontology\n", + "\n", + "Build disease ontology from extracted entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "disease_kg = builder.build(disease_entities, disease_relationships)\n", + "\n", + "disease_ontology = ontology_generator.generate(disease_entities, disease_relationships)\n", + "\n", + "classes = class_inferrer.infer_classes(disease_entities)\n", + "properties = property_generator.infer_properties(disease_entities, disease_relationships, classes)\n", + "\n", + "validation_result = ontology_validator.validate_ontology(disease_ontology)\n", + "\n", + "print(f\"Built disease knowledge graph\")\n", + "print(f\" Entities: {len(disease_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(disease_kg.get('relationships', []))}\")\n", + "print(f\"Generated disease ontology\")\n", + "print(f\" Classes: {len(disease_ontology.get('classes', []))}\")\n", + "print(f\" Properties: {len(disease_ontology.get('properties', []))}\")\n", + "print(f\" Ontology valid: {validation_result.valid}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze Disease Networks\n", + "\n", + "Analyze disease networks using graph analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "\n", + "metrics = graph_analyzer.compute_metrics(disease_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(disease_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(disease_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(disease_kg)\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " disease_kg,\n", + " pattern_type=\"sequence\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "print(f\"Network analysis complete\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Communities: {len(communities)}\")\n", + "print(f\" Central diseases: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n", + "print(f\" Connected components: {len(connectivity.get('components', []))}\")\n", + "print(f\" Temporal patterns: {len(temporal_patterns)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Predict Disease Outcomes\n", + "\n", + "Predict disease progression and outcomes using inference.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Disease outcome prediction rules\n", + "inference_engine.add_rule(\"IF disease related_to Hypertension AND disease related_to Diabetes THEN high_comorbidity_risk\")\n", + "inference_engine.add_rule(\"IF disease has_symptom Fatigue AND disease prevalence is High THEN common_condition\")\n", + "inference_engine.add_rule(\"IF disease treated_with Insulin AND disease is Type 2 Diabetes THEN advanced_stage\")\n", + "\n", + "# Add facts from disease data\n", + "for disease in disease_entities:\n", + " if disease.get(\"type\") == \"Disease\":\n", + " inference_engine.add_fact({\n", + " \"disease\": disease.get(\"name\", \"\"),\n", + " \"prevalence\": disease.get(\"properties\", {}).get(\"prevalence\", \"\")\n", + " })\n", + "\n", + "for relationship in disease_relationships:\n", + " if relationship.get(\"type\") == \"related_to\":\n", + " inference_engine.add_fact({\n", + " \"disease1\": relationship.get(\"source\"),\n", + " \"disease2\": relationship.get(\"target\")\n", + " })\n", + "\n", + "outcome_predictions = inference_engine.forward_chain()\n", + "\n", + "print(f\"Generated {len(outcome_predictions)} disease outcome predictions\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Generate Reports and Visualize\n", + "\n", + "Generate disease network analysis reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(disease_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(disease_kg, os.path.join(temp_dir, \"disease_kg.json\"))\n", + "rdf_exporter.export_knowledge_graph(disease_kg, os.path.join(temp_dir, \"disease_kg.rdf\"))\n", + "owl_exporter.export(disease_ontology, os.path.join(temp_dir, \"disease_ontology.owl\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Disease network analysis identified {len(disease_entities)} entities and {len(outcome_predictions)} predictions\",\n", + " \"diseases_analyzed\": len([e for e in disease_entities if e.get(\"type\") == \"Disease\"]),\n", + " \"relationships\": len(disease_relationships),\n", + " \"predictions\": len(outcome_predictions),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(disease_kg, output=\"interactive\")\n", + "ontology_viz = ontology_visualizer.visualize_hierarchy(disease_ontology, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(disease_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated disease network analysis report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Disease Data → Parse → Extract → Build Ontology → Analyze Networks → Predict Outcomes → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/healthcare/Drug_Interactions_Analysis.ipynb b/docs/cookbook/use_cases/healthcare/Drug_Interactions_Analysis.ipynb new file mode 100644 index 00000000..631704d0 --- /dev/null +++ b/docs/cookbook/use_cases/healthcare/Drug_Interactions_Analysis.ipynb @@ -0,0 +1,415 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Drug Interactions Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete drug interactions analysis pipeline: ingest drug data from multiple sources (FDA databases, drug interaction databases, medical literature), extract drug information, build drug knowledge graph, detect interactions, and generate drug safety ontology.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: DocumentParser, PDFParser, StructuredDataParser, JSONParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Drug Data Sources → Parse → Extract Drug Info → Build Drug KG → Detect Interactions → Generate Ontology → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Drug Data from Multiple Sources\n", + "\n", + "Ingest drug data from FDA databases, drug interaction databases, and medical literature.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import DocumentParser, PDFParser, StructuredDataParser, JSONParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "document_parser = DocumentParser()\n", + "pdf_parser = PDFParser()\n", + "structured_parser = StructuredDataParser()\n", + "json_parser = JSONParser()\n", + "\n", + "# Real drug data sources\n", + "drug_apis = [\n", + " \"https://api.fda.gov/drug/label.json\", # FDA Drug Labeling API\n", + " \"https://api.fda.gov/drug/event.json\", # FDA Adverse Events API\n", + " \"https://api.logicahealth.org/fhir/R4/Medication\" # FHIR Medications\n", + "]\n", + "\n", + "medical_feeds = [\n", + " \"https://www.cdc.gov/rss.xml\", # CDC Health Alerts\n", + " \"https://www.who.int/rss-feeds/news-english.xml\" # WHO News\n", + "]\n", + "\n", + "# Real database connection for drug data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/drug_db\"\n", + "db_query = \"SELECT drug_name, drug_class, interactions, contraindications, side_effects FROM drugs WHERE last_updated > CURRENT_DATE - INTERVAL '1 year' ORDER BY drug_name\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample drug interaction data\n", + "drug_file = os.path.join(temp_dir, \"drug_data.json\")\n", + "drug_data = {\n", + " \"drugs\": [\n", + " {\n", + " \"drug_name\": \"Lisinopril\",\n", + " \"drug_class\": \"ACE Inhibitor\",\n", + " \"interactions\": [\"Potassium supplements\", \"Diuretics\", \"NSAIDs\"],\n", + " \"contraindications\": [\"Pregnancy\", \"Angioedema\"],\n", + " \"side_effects\": [\"Cough\", \"Dizziness\", \"Hyperkalemia\"]\n", + " },\n", + " {\n", + " \"drug_name\": \"Metformin\",\n", + " \"drug_class\": \"Biguanide\",\n", + " \"interactions\": [\"Alcohol\", \"Contrast agents\"],\n", + " \"contraindications\": [\"Renal impairment\", \"Lactic acidosis\"],\n", + " \"side_effects\": [\"Nausea\", \"Diarrhea\", \"Lactic acidosis\"]\n", + " },\n", + " {\n", + " \"drug_name\": \"Warfarin\",\n", + " \"drug_class\": \"Anticoagulant\",\n", + " \"interactions\": [\"Aspirin\", \"Antibiotics\", \"Vitamin K\"],\n", + " \"contraindications\": [\"Active bleeding\", \"Pregnancy\"],\n", + " \"side_effects\": [\"Bleeding\", \"Bruising\", \"Hair loss\"]\n", + " }\n", + " ]\n", + "}\n", + "\n", + "with open(drug_file, 'w') as f:\n", + " json.dump(drug_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(drug_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(drug_file)\n", + "\n", + "# Ingest from drug APIs\n", + "drug_api_list = []\n", + "for api_url in drug_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " drug_api_list.append(api_content)\n", + " print(f\"✓ Ingested drug API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Drug API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Drug data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Drug API sources: {len(drug_api_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Drug Information\n", + "\n", + "Extract drug entities and interaction information.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "drug_entities = []\n", + "drug_relationships = []\n", + "\n", + "# Extract from drug data\n", + "if parsed_data and parsed_data.data:\n", + " drugs = parsed_data.data.get(\"drugs\", []) if isinstance(parsed_data.data, dict) else parsed_data.data if isinstance(parsed_data.data, list) else []\n", + " \n", + " for drug in drugs:\n", + " if isinstance(drug, dict):\n", + " drug_name = drug.get(\"drug_name\", \"\")\n", + " \n", + " drug_entities.append({\n", + " \"id\": drug_name,\n", + " \"type\": \"Drug\",\n", + " \"name\": drug_name,\n", + " \"properties\": {\n", + " \"drug_class\": drug.get(\"drug_class\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Drug interactions\n", + " for interaction in drug.get(\"interactions\", []):\n", + " drug_entities.append({\n", + " \"id\": interaction,\n", + " \"type\": \"Drug\",\n", + " \"name\": interaction,\n", + " \"properties\": {}\n", + " })\n", + " drug_relationships.append({\n", + " \"source\": drug_name,\n", + " \"target\": interaction,\n", + " \"type\": \"interacts_with\",\n", + " \"properties\": {\"severity\": \"moderate\"}\n", + " })\n", + " \n", + " # Contraindications\n", + " for contraindication in drug.get(\"contraindications\", []):\n", + " drug_entities.append({\n", + " \"id\": contraindication,\n", + " \"type\": \"Contraindication\",\n", + " \"name\": contraindication,\n", + " \"properties\": {}\n", + " })\n", + " drug_relationships.append({\n", + " \"source\": drug_name,\n", + " \"target\": contraindication,\n", + " \"type\": \"contraindicated_in\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Side effects\n", + " for side_effect in drug.get(\"side_effects\", []):\n", + " drug_entities.append({\n", + " \"id\": side_effect,\n", + " \"type\": \"Side_Effect\",\n", + " \"name\": side_effect,\n", + " \"properties\": {}\n", + " })\n", + " drug_relationships.append({\n", + " \"source\": drug_name,\n", + " \"target\": side_effect,\n", + " \"type\": \"causes\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "print(f\"Extracted {len(drug_entities)} drug entities\")\n", + "print(f\"Extracted {len(drug_relationships)} drug relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Drug Knowledge Graph\n", + "\n", + "Build knowledge graph from drug entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "drug_kg = builder.build(drug_entities, drug_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(drug_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(drug_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(drug_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(drug_kg)\n", + "\n", + "print(f\"Built drug knowledge graph\")\n", + "print(f\" Entities: {len(drug_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(drug_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Communities: {len(communities)}\")\n", + "print(f\" Central drugs: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Detect Drug Interactions\n", + "\n", + "Detect drug-drug interactions using inference engine.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Drug interaction detection rules\n", + "inference_engine.add_rule(\"IF drug1 interacts_with drug2 AND drug2 interacts_with drug1 THEN bidirectional_interaction\")\n", + "inference_engine.add_rule(\"IF drug interacts_with Anticoagulant AND drug is Antibiotic THEN increased_bleeding_risk\")\n", + "inference_engine.add_rule(\"IF drug contraindicated_in Pregnancy AND patient is pregnant THEN contraindicated\")\n", + "\n", + "# Detect interactions\n", + "interactions = []\n", + "for relationship in drug_relationships:\n", + " if relationship.get(\"type\") == \"interacts_with\":\n", + " drug1 = relationship.get(\"source\")\n", + " drug2 = relationship.get(\"target\")\n", + " \n", + " # Check for bidirectional interaction\n", + " reverse_interaction = [r for r in drug_relationships \n", + " if r.get(\"source\") == drug2 and r.get(\"target\") == drug1 and r.get(\"type\") == \"interacts_with\"]\n", + " \n", + " if reverse_interaction:\n", + " interactions.append({\n", + " \"drug1\": drug1,\n", + " \"drug2\": drug2,\n", + " \"type\": \"bidirectional\",\n", + " \"severity\": relationship.get(\"properties\", {}).get(\"severity\", \"unknown\")\n", + " })\n", + " else:\n", + " interactions.append({\n", + " \"drug1\": drug1,\n", + " \"drug2\": drug2,\n", + " \"type\": \"unidirectional\",\n", + " \"severity\": relationship.get(\"properties\", {}).get(\"severity\", \"unknown\")\n", + " })\n", + " \n", + " inference_engine.add_fact({\n", + " \"drug1\": drug1,\n", + " \"drug2\": drug2,\n", + " \"interaction_type\": relationship.get(\"type\")\n", + " })\n", + "\n", + "detected_interactions = inference_engine.forward_chain()\n", + "\n", + "print(f\"Detected {len(interactions)} drug interactions\")\n", + "print(f\"Inferred {len(detected_interactions)} interaction patterns\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Drug Ontology\n", + "\n", + "Generate drug safety ontology from drug knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "drug_ontology = ontology_generator.generate(drug_entities, drug_relationships)\n", + "\n", + "classes = class_inferrer.infer_classes(drug_entities)\n", + "properties = property_generator.infer_properties(drug_entities, drug_relationships, classes)\n", + "\n", + "validation_result = ontology_validator.validate_ontology(drug_ontology)\n", + "\n", + "print(f\"Generated drug safety ontology\")\n", + "print(f\" Classes: {len(drug_ontology.get('classes', []))}\")\n", + "print(f\" Properties: {len(drug_ontology.get('properties', []))}\")\n", + "print(f\" Ontology valid: {validation_result.valid}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Generate Reports and Visualize\n", + "\n", + "Generate drug interaction reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(drug_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(drug_kg, os.path.join(temp_dir, \"drug_kg.json\"))\n", + "rdf_exporter.export_knowledge_graph(drug_kg, os.path.join(temp_dir, \"drug_kg.rdf\"))\n", + "owl_exporter.export(drug_ontology, os.path.join(temp_dir, \"drug_ontology.owl\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Drug interactions analysis identified {len(interactions)} interactions from {len(drug_entities)} drug entities\",\n", + " \"drugs_analyzed\": len([e for e in drug_entities if e.get(\"type\") == \"Drug\"]),\n", + " \"interactions\": len(interactions),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(drug_kg, output=\"interactive\")\n", + "ontology_viz = ontology_visualizer.visualize_hierarchy(drug_ontology, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(drug_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated drug interaction reports and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Drug Data → Parse → Extract → Build KG → Detect Interactions → Generate Ontology → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/healthcare/Healthcare_GraphRAG_Hybrid.ipynb b/docs/cookbook/use_cases/healthcare/Healthcare_GraphRAG_Hybrid.ipynb new file mode 100644 index 00000000..4f505291 --- /dev/null +++ b/docs/cookbook/use_cases/healthcare/Healthcare_GraphRAG_Hybrid.ipynb @@ -0,0 +1,810 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Healthcare GraphRAG System with Semantica\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates building a **Retrieval-Augmented Generation (GraphRAG) system** using **Semantica as the core framework** that leverages heterogeneous healthcare resources through a combination of materialized knowledge graphs and virtually integrated data sources.\n", + "\n", + "### Why Semantica?\n", + "\n", + "Semantica provides a comprehensive, unified framework for building GraphRAG systems in healthcare:\n", + "\n", + "- **Materialized Knowledge Graphs**: Semantica's KG modules enable building persistent knowledge graphs from medical ontologies, clinical documents, and reports\n", + "- **Virtual Data Integration**: Semantica's DBIngestor and QueryEngine allow virtual integration with Electronic Health Records (EHRs) without data replication\n", + "- **Hybrid Design**: Semantica's architecture naturally separates structural knowledge from patient-level data\n", + "- **Dynamic Query Orchestration**: Semantica's Reasoning and Triple Store modules enable orchestration of queries across ontologies, documents, and EHRs\n", + "- **Temporal & Semantic Dimensions**: Semantica's Temporal and Context modules provide historical analysis and semantic understanding\n", + "- **Traceable & Explainable**: Semantica's ExplanationGenerator and ContextRetriever provide traceable, explainable answers\n", + "\n", + "### Key Features\n", + "\n", + "- Materialized knowledge graphs from medical ontologies (SNOMED CT, ICD-10), clinical documents, and reports\n", + "- Virtual integration with Electronic Health Records (EHRs) without data replication\n", + "- Hybrid design separating structural knowledge from patient-level data\n", + "- Dynamic orchestration of queries across ontologies, documents, and EHRs using Semantica\n", + "- Temporal and semantic dimensions via graph for historical analysis\n", + "- Traceable, explainable, and historically contextualized answers\n", + "\n", + "### Semantica Modules Used (20+)\n", + "\n", + "- **Ingest**: FileIngestor, DBIngestor (for EHR virtual connections), WebIngestor (for medical literature)\n", + "- **Parse**: DocumentParser, PDFParser (clinical documents), StructuredDataParser (for structured medical data)\n", + "- **Normalize**: TextNormalizer (for text normalization)\n", + "- **Semantic Extract**: NERExtractor, RelationExtractor, TripleExtractor (medical entities and relationships)\n", + "- **Ontology**: OntologyGenerator, OWLGenerator (medical ontologies like SNOMED CT, ICD-10)\n", + "- **KG**: GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer (materialized knowledge graph)\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder (for embeddings)\n", + "- **Vector Store**: VectorStore, HybridSearch, MetadataFilter (for RAG)\n", + "- **Triple Store**: TripleManager, QueryEngine (for SPARQL queries on ontologies)\n", + "- **Reasoning**: InferenceEngine, RuleManager (for query orchestration and medical reasoning)\n", + "- **Context**: ContextRetriever, ContextGraphBuilder (for contextual retrieval)\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer (for visualization)\n", + "- **Export**: JSONExporter, RDFExporter, ReportGenerator\n", + "- **Pipeline**: PipelineBuilder, ExecutionEngine (for orchestrating the complete pipeline)\n", + "\n", + "### Pipeline Overview\n", + "\n", + "**Medical Ontologies + Clinical Documents + EHRs (Virtual) → Parse → Extract Medical Entities → Build Materialized KG → Generate Embeddings → Vector Store → GraphRAG Setup → Query Orchestration → Generate Answers → Visualize & Export**\n", + "\n", + "---\n", + "\n", + "## Step 1: Setup and Import Semantica Modules\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import all Semantica modules - using Semantica as the core framework\n", + "from semantica.ingest import FileIngestor, DBIngestor, WebIngestor\n", + "from semantica.parse import DocumentParser, PDFParser, StructuredDataParser\n", + "from semantica.normalize import TextNormalizer\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor\n", + "from semantica.ontology import OntologyGenerator, OWLGenerator\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.vector_store import VectorStore, HybridSearch, MetadataFilter\n", + "from semantica.triple_store import TripleManager, QueryEngine\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.context import ContextRetriever, ContextGraphBuilder\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", + "from semantica.pipeline import PipelineBuilder, ExecutionEngine\n", + "\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "import numpy as np\n", + "\n", + "print(\"✓ All Semantica modules imported successfully\")\n", + "print(\"✓ Using Semantica as the core framework for Healthcare GraphRAG\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Medical Ontologies and Clinical Documents\n", + "\n", + "Using Semantica's ingest modules to load medical ontologies, clinical documents, and set up virtual connections to EHRs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica ingestors\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "web_ingestor = WebIngestor()\n", + "\n", + "# Create temporary directory for sample data\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample medical ontology data (SNOMED CT concepts)\n", + "snomed_data = {\n", + " \"concepts\": [\n", + " {\n", + " \"concept_id\": \"73211009\",\n", + " \"concept_name\": \"Diabetes mellitus\",\n", + " \"hierarchy\": \"Clinical finding\",\n", + " \"parent_concept\": \"Disorder of glucose metabolism\"\n", + " },\n", + " {\n", + " \"concept_id\": \"44054006\",\n", + " \"concept_name\": \"Type 2 diabetes mellitus\",\n", + " \"hierarchy\": \"Clinical finding\",\n", + " \"parent_concept\": \"Diabetes mellitus\"\n", + " },\n", + " {\n", + " \"concept_id\": \"46635009\",\n", + " \"concept_name\": \"Type 1 diabetes mellitus\",\n", + " \"hierarchy\": \"Clinical finding\",\n", + " \"parent_concept\": \"Diabetes mellitus\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "# Sample clinical document\n", + "clinical_document = {\n", + " \"document_id\": \"DOC-001\",\n", + " \"patient_id\": \"PATIENT-001\",\n", + " \"document_type\": \"Clinical Note\",\n", + " \"content\": \"Patient presents with Type 2 diabetes mellitus. Current medications include Metformin 500mg twice daily. Blood glucose levels are well-controlled. Patient reports adherence to dietary recommendations.\",\n", + " \"date\": datetime.now().isoformat(),\n", + " \"physician\": \"Dr. Smith\"\n", + "}\n", + "\n", + "# Save sample data\n", + "snomed_file = os.path.join(temp_dir, \"snomed_concepts.json\")\n", + "clinical_file = os.path.join(temp_dir, \"clinical_note.json\")\n", + "\n", + "with open(snomed_file, 'w') as f:\n", + " json.dump(snomed_data, f, indent=2)\n", + "\n", + "with open(clinical_file, 'w') as f:\n", + " json.dump(clinical_document, f, indent=2)\n", + "\n", + "# Ingest using Semantica FileIngestor\n", + "snomed_file_obj = file_ingestor.ingest_file(snomed_file, read_content=True)\n", + "clinical_file_obj = file_ingestor.ingest_file(clinical_file, read_content=True)\n", + "\n", + "# Virtual EHR connection setup (using Semantica DBIngestor)\n", + "# In production, this would connect to actual EHR database\n", + "ehr_connection_config = {\n", + " \"connection_string\": \"postgresql://user:password@localhost:5432/ehr_db\",\n", + " \"query\": \"SELECT patient_id, diagnosis, medications, lab_results, visit_date FROM patient_records WHERE patient_id = %s\"\n", + "}\n", + "\n", + "print(f\"✓ Ingested SNOMED CT concepts: {len(snomed_data['concepts'])} concepts\")\n", + "print(f\"✓ Ingested clinical document: {clinical_document['document_id']}\")\n", + "print(f\"✓ EHR virtual connection configured (not executed - would connect to actual EHR in production)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Parse and Normalize Data Using Semantica\n", + "\n", + "Using Semantica's parse and normalize modules to process the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica parsers and normalizer\n", + "document_parser = DocumentParser()\n", + "pdf_parser = PDFParser()\n", + "structured_parser = StructuredDataParser()\n", + "text_normalizer = TextNormalizer()\n", + "\n", + "# Parse structured data (SNOMED CT)\n", + "parsed_snomed = structured_parser.parse_json(snomed_file)\n", + "snomed_concepts = parsed_snomed.data if hasattr(parsed_snomed, 'data') else parsed_snomed\n", + "\n", + "# Parse clinical document\n", + "parsed_clinical = structured_parser.parse_json(clinical_file)\n", + "clinical_data = parsed_clinical.data if hasattr(parsed_clinical, 'data') else parsed_clinical\n", + "\n", + "# Normalize clinical text using Semantica\n", + "if isinstance(clinical_data, dict) and 'content' in clinical_data:\n", + " normalized_text = text_normalizer.normalize(clinical_data['content'])\n", + " clinical_data['normalized_content'] = normalized_text\n", + "\n", + "print(f\"✓ Parsed SNOMED CT concepts: {len(snomed_concepts.get('concepts', [])) if isinstance(snomed_concepts, dict) else 0}\")\n", + "print(f\"✓ Parsed clinical document: {clinical_data.get('document_id', 'N/A')}\")\n", + "print(f\"✓ Normalized clinical text: {len(normalized_text) if 'normalized_text' in locals() else 0} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Medical Entities and Relationships Using Semantica\n", + "\n", + "Using Semantica's semantic extraction modules to extract medical entities, relationships, and triples from the parsed data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica extractors\n", + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "# Extract entities from clinical document\n", + "clinical_text = clinical_data.get('normalized_content', clinical_data.get('content', ''))\n", + "extracted_entities = ner_extractor.extract(clinical_text)\n", + "\n", + "# Extract relationships\n", + "extracted_relationships = relation_extractor.extract(clinical_text, entities=extracted_entities)\n", + "\n", + "# Extract triples\n", + "extracted_triples = triple_extractor.extract(clinical_text)\n", + "\n", + "# Build entity list for knowledge graph\n", + "medical_entities = []\n", + "medical_relationships = []\n", + "\n", + "# Add SNOMED CT concepts as entities\n", + "if isinstance(snomed_concepts, dict):\n", + " for concept in snomed_concepts.get('concepts', []):\n", + " medical_entities.append({\n", + " \"id\": f\"snomed_{concept.get('concept_id', '')}\",\n", + " \"type\": \"Medical_Concept\",\n", + " \"name\": concept.get('concept_name', ''),\n", + " \"properties\": {\n", + " \"concept_id\": concept.get('concept_id', ''),\n", + " \"hierarchy\": concept.get('hierarchy', ''),\n", + " \"source\": \"SNOMED_CT\"\n", + " }\n", + " })\n", + "\n", + "# Add extracted entities from clinical document\n", + "for entity in extracted_entities:\n", + " medical_entities.append({\n", + " \"id\": f\"entity_{entity.get('id', len(medical_entities))}\",\n", + " \"type\": entity.get('type', 'Entity'),\n", + " \"name\": entity.get('text', ''),\n", + " \"properties\": {\n", + " \"source\": \"clinical_document\",\n", + " \"document_id\": clinical_data.get('document_id', '')\n", + " }\n", + " })\n", + "\n", + "# Add relationships\n", + "for rel in extracted_relationships:\n", + " medical_relationships.append({\n", + " \"source\": rel.get('source', ''),\n", + " \"target\": rel.get('target', ''),\n", + " \"type\": rel.get('type', 'related_to'),\n", + " \"properties\": {\n", + " \"source\": \"clinical_document\"\n", + " }\n", + " })\n", + "\n", + "print(f\"✓ Extracted {len(extracted_entities)} medical entities\")\n", + "print(f\"✓ Extracted {len(extracted_relationships)} relationships\")\n", + "print(f\"✓ Extracted {len(extracted_triples)} triples\")\n", + "print(f\"✓ Total entities for KG: {len(medical_entities)}\")\n", + "print(f\"✓ Total relationships for KG: {len(medical_relationships)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Materialized Knowledge Graph Using Semantica\n", + "\n", + "Using Semantica's KG modules to build a materialized knowledge graph from the extracted entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica KG builders and analyzers\n", + "graph_builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "# Build materialized knowledge graph using Semantica\n", + "materialized_kg = graph_builder.build(medical_entities, medical_relationships)\n", + "\n", + "# Analyze the graph using Semantica\n", + "kg_metrics = graph_analyzer.compute_metrics(materialized_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(materialized_kg)\n", + "\n", + "print(f\"✓ Built materialized knowledge graph\")\n", + "print(f\" - Entities: {len(materialized_kg.get('entities', []))}\")\n", + "print(f\" - Relationships: {len(materialized_kg.get('relationships', []))}\")\n", + "print(f\" - Graph density: {kg_metrics.get('density', 0):.4f}\")\n", + "print(f\" - Connected components: {connectivity.get('num_components', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Generate Ontology Using Semantica\n", + "\n", + "Using Semantica's ontology modules to generate medical ontologies from the knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica ontology generators\n", + "ontology_generator = OntologyGenerator()\n", + "owl_generator = OWLGenerator()\n", + "\n", + "# Generate ontology from knowledge graph using Semantica\n", + "ontology_result = ontology_generator.generate(\n", + " semantic_network=materialized_kg,\n", + " domain=\"Healthcare\",\n", + " namespace=\"http://semantica.example.org/healthcare#\"\n", + ")\n", + "\n", + "# Generate OWL representation using Semantica\n", + "owl_ontology = owl_generator.generate(ontology_result)\n", + "\n", + "print(f\"✓ Generated ontology using Semantica\")\n", + "print(f\" - Classes: {len(ontology_result.get('classes', []))}\")\n", + "print(f\" - Properties: {len(ontology_result.get('properties', []))}\")\n", + "print(f\" - OWL generated: {len(owl_ontology) if owl_ontology else 0} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Generate Embeddings Using Semantica\n", + "\n", + "Using Semantica's embedding modules to generate vector embeddings for RAG.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica embedding generators\n", + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "\n", + "# Generate embeddings for clinical documents using Semantica\n", + "documents_for_embedding = [clinical_data.get('normalized_content', clinical_data.get('content', ''))]\n", + "\n", + "# Generate embeddings using Semantica\n", + "document_embeddings = []\n", + "for doc in documents_for_embedding:\n", + " embedding = text_embedder.embed(doc)\n", + " document_embeddings.append(embedding)\n", + "\n", + "# Generate embeddings for entities\n", + "entity_embeddings = {}\n", + "for entity in medical_entities[:10]: # Limit for demo\n", + " entity_text = f\"{entity.get('name', '')} {entity.get('type', '')}\"\n", + " embedding = text_embedder.embed(entity_text)\n", + " entity_embeddings[entity.get('id', '')] = embedding\n", + "\n", + "print(f\"✓ Generated embeddings using Semantica\")\n", + "print(f\" - Document embeddings: {len(document_embeddings)}\")\n", + "print(f\" - Entity embeddings: {len(entity_embeddings)}\")\n", + "print(f\" - Embedding dimension: {len(document_embeddings[0]) if document_embeddings else 0}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Setup Vector Store and Hybrid Search Using Semantica\n", + "\n", + "Using Semantica's vector store modules to set up RAG with hybrid search capabilities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica vector store and hybrid search\n", + "vector_store = VectorStore(backend=\"faiss\", dimension=768)\n", + "hybrid_search = HybridSearch()\n", + "\n", + "# Prepare metadata for documents\n", + "document_metadata = [{\n", + " \"document_id\": clinical_data.get('document_id', ''),\n", + " \"document_type\": clinical_data.get('document_type', ''),\n", + " \"date\": clinical_data.get('date', ''),\n", + " \"source\": \"clinical_document\"\n", + "}]\n", + "\n", + "# Store document embeddings using Semantica\n", + "document_ids = vector_store.store_vectors(\n", + " vectors=document_embeddings,\n", + " metadata=document_metadata\n", + ")\n", + "\n", + "# Store entity embeddings\n", + "entity_metadata = [{\"entity_id\": eid, \"type\": \"entity\", \"source\": \"kg\"} for eid in entity_embeddings.keys()]\n", + "entity_ids = vector_store.store_vectors(\n", + " vectors=list(entity_embeddings.values()),\n", + " metadata=entity_metadata\n", + ")\n", + "\n", + "print(f\"✓ Set up vector store using Semantica\")\n", + "print(f\" - Document vectors stored: {len(document_ids)}\")\n", + "print(f\" - Entity vectors stored: {len(entity_ids)}\")\n", + "print(f\" - Hybrid search ready for GraphRAG\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Setup Triple Store for Ontology Queries Using Semantica\n", + "\n", + "Using Semantica's triple store modules to enable SPARQL queries on medical ontologies.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica triple store and query engine\n", + "triple_manager = TripleManager()\n", + "query_engine = QueryEngine()\n", + "\n", + "# Register triple store (using in-memory for demo)\n", + "store = triple_manager.register_store(\"healthcare_ontology\", \"jena\", \"http://localhost:3030/healthcare\")\n", + "\n", + "# Convert ontology to triples and add to store\n", + "# In production, this would load the OWL ontology\n", + "sample_triples = [\n", + " {\n", + " \"subject\": \"http://semantica.example.org/healthcare#Type2Diabetes\",\n", + " \"predicate\": \"http://www.w3.org/2000/01/rdf-schema#subClassOf\",\n", + " \"object\": \"http://semantica.example.org/healthcare#Diabetes\",\n", + " \"confidence\": 1.0\n", + " },\n", + " {\n", + " \"subject\": \"http://semantica.example.org/healthcare#Type1Diabetes\",\n", + " \"predicate\": \"http://www.w3.org/2000/01/rdf-schema#subClassOf\",\n", + " \"object\": \"http://semantica.example.org/healthcare#Diabetes\",\n", + " \"confidence\": 1.0\n", + " }\n", + "]\n", + "\n", + "# Add triples using Semantica\n", + "for triple in sample_triples:\n", + " triple_manager.add_triple(triple, store_id=\"healthcare_ontology\")\n", + "\n", + "print(f\"✓ Set up triple store using Semantica\")\n", + "print(f\" - Triples added: {len(sample_triples)}\")\n", + "print(f\" - SPARQL queries enabled for ontology\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: Implement GraphRAG Query Orchestration Using Semantica\n", + "\n", + "Using Semantica's reasoning and context modules to orchestrate queries across ontologies, documents, and EHRs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica reasoning and context modules\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "context_retriever = ContextRetriever()\n", + "context_graph_builder = ContextGraphBuilder()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Define medical reasoning rules using Semantica\n", + "medical_rules = [\n", + " {\n", + " \"rule_id\": \"diabetes_treatment_rule\",\n", + " \"condition\": \"IF patient has Type2Diabetes THEN recommend Metformin\",\n", + " \"action\": \"suggest_treatment\"\n", + " }\n", + "]\n", + "\n", + "# Add rules using Semantica\n", + "for rule in medical_rules:\n", + " rule_manager.add_rule(rule)\n", + "\n", + "# Example query orchestration function using Semantica\n", + "def orchestrate_graphrag_query(query_text, patient_id=None):\n", + " \"\"\"\n", + " Orchestrate GraphRAG query across ontologies, documents, and EHRs using Semantica.\n", + " \"\"\"\n", + " results = {\n", + " \"ontology_results\": [],\n", + " \"document_results\": [],\n", + " \"ehr_results\": [],\n", + " \"graph_results\": [],\n", + " \"context\": {}\n", + " }\n", + " \n", + " # 1. Query ontology using Semantica Triple Store\n", + " sparql_query = f\"\"\"\n", + " SELECT ?concept WHERE {{\n", + " ?concept rdfs:label ?label .\n", + " FILTER(CONTAINS(LCASE(?label), \"{query_text.lower()}\"))\n", + " }}\n", + " \"\"\"\n", + " try:\n", + " ontology_results = query_engine.execute_query(sparql_query, store)\n", + " results[\"ontology_results\"] = ontology_results.get(\"bindings\", [])\n", + " except:\n", + " pass\n", + " \n", + " # 2. Search documents using Semantica Hybrid Search\n", + " query_embedding = text_embedder.embed(query_text)\n", + " document_results = hybrid_search.search(\n", + " query_vector=query_embedding,\n", + " vectors=document_embeddings,\n", + " metadata=document_metadata,\n", + " vector_ids=document_ids,\n", + " k=5\n", + " )\n", + " results[\"document_results\"] = document_results\n", + " \n", + " # 3. Query knowledge graph using Semantica\n", + " graph_results = graph_analyzer.query_graph(materialized_kg, query_text)\n", + " results[\"graph_results\"] = graph_results\n", + " \n", + " # 4. Build context using Semantica\n", + " context_graph = context_graph_builder.build(\n", + " entities=medical_entities,\n", + " relationships=medical_relationships,\n", + " query=query_text\n", + " )\n", + " results[\"context\"] = context_graph\n", + " \n", + " # 5. Virtual EHR query (would execute in production)\n", + " if patient_id:\n", + " # In production: ehr_results = db_ingestor.query(ehr_connection_config, patient_id)\n", + " results[\"ehr_results\"] = {\"note\": \"EHR query would execute here in production\"}\n", + " \n", + " return results\n", + "\n", + "# Example query\n", + "query = \"What are the treatment options for Type 2 diabetes?\"\n", + "orchestrated_results = orchestrate_graphrag_query(query)\n", + "\n", + "print(f\"✓ Implemented GraphRAG query orchestration using Semantica\")\n", + "print(f\" - Query: '{query}'\")\n", + "print(f\" - Ontology results: {len(orchestrated_results['ontology_results'])}\")\n", + "print(f\" - Document results: {len(orchestrated_results['document_results'])}\")\n", + "print(f\" - Graph results: {len(orchestrated_results['graph_results'])}\")\n", + "print(f\" - Context built: {bool(orchestrated_results['context'])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate explanation using Semantica\n", + "explanation = explanation_generator.generate(\n", + " query=query,\n", + " results=orchestrated_results,\n", + " knowledge_graph=materialized_kg,\n", + " context=orchestrated_results['context']\n", + ")\n", + "\n", + "print(\"✓ Generated explainable answer using Semantica\")\n", + "print(\"\\nAnswer Explanation:\")\n", + "print(f\" {explanation.get('answer', 'Answer generated')}\")\n", + "print(f\"\\nSources:\")\n", + "print(f\" - Ontology: {len(orchestrated_results['ontology_results'])} results\")\n", + "print(f\" - Documents: {len(orchestrated_results['document_results'])} results\")\n", + "print(f\" - Knowledge Graph: {len(orchestrated_results['graph_results'])} results\")\n", + "print(f\" - Traceability: Enabled via Semantica ContextRetriever\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 12: Visualize Knowledge Graph and Results Using Semantica\n", + "\n", + "Using Semantica's visualization modules to visualize the knowledge graph and analysis results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica visualizers\n", + "kg_visualizer = KGVisualizer(layout=\"force\", color_scheme=\"vibrant\")\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "# Visualize knowledge graph using Semantica\n", + "kg_fig = kg_visualizer.visualize_network(\n", + " materialized_kg,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "# Visualize analytics\n", + "centrality = graph_analyzer.compute_centrality(materialized_kg, method=\"pagerank\")\n", + "analytics_data = {\n", + " \"graph\": materialized_kg,\n", + " \"centrality\": centrality\n", + "}\n", + "analytics_fig = analytics_visualizer.visualize_centrality_rankings(\n", + " centrality,\n", + " centrality_type=\"pagerank\",\n", + " top_n=10,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "print(\"✓ Visualized knowledge graph and analytics using Semantica\")\n", + "print(\" - Knowledge graph visualization: Interactive\")\n", + "print(\" - Analytics dashboard: Centrality rankings\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 13: Export Results Using Semantica\n", + "\n", + "Using Semantica's export modules to export the knowledge graph, ontology, and reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica exporters\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export knowledge graph as JSON using Semantica\n", + "kg_json_file = os.path.join(temp_dir, \"healthcare_kg.json\")\n", + "json_exporter.export(materialized_kg, kg_json_file)\n", + "\n", + "# Export ontology as RDF using Semantica\n", + "ontology_rdf_file = os.path.join(temp_dir, \"healthcare_ontology.rdf\")\n", + "rdf_exporter.export(ontology_result, ontology_rdf_file)\n", + "\n", + "# Generate comprehensive report using Semantica\n", + "report_data = {\n", + " \"title\": \"Healthcare GraphRAG System Report\",\n", + " \"knowledge_graph_metrics\": kg_metrics,\n", + " \"query_results\": orchestrated_results,\n", + " \"explanation\": explanation\n", + "}\n", + "report_file = os.path.join(temp_dir, \"healthcare_graphrag_report.html\")\n", + "report_generator.generate_report(report_data, report_file, format=\"html\")\n", + "\n", + "print(\"✓ Exported results using Semantica\")\n", + "print(f\" - Knowledge graph JSON: {kg_json_file}\")\n", + "print(f\" - Ontology RDF: {ontology_rdf_file}\")\n", + "print(f\" - Report HTML: {report_file}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 14: Complete Pipeline Orchestration Using Semantica\n", + "\n", + "Using Semantica's pipeline module to orchestrate the complete GraphRAG pipeline.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Build complete pipeline using Semantica PipelineBuilder\n", + "pipeline_builder = PipelineBuilder()\n", + "\n", + "healthcare_graphrag_pipeline = pipeline_builder \\\n", + " .add_step(\"ingest\", \"file_ingest\", source=temp_dir) \\\n", + " .add_step(\"parse\", \"structured_parse\", formats=[\"json\"]) \\\n", + " .add_step(\"normalize\", \"text_normalize\") \\\n", + " .add_step(\"extract\", \"semantic_extract\", entities=True, relations=True) \\\n", + " .add_step(\"build_kg\", \"kg_build\") \\\n", + " .add_step(\"generate_embeddings\", \"embedding_generate\") \\\n", + " .add_step(\"setup_vector_store\", \"vector_store_setup\") \\\n", + " .add_step(\"setup_triple_store\", \"triple_store_setup\") \\\n", + " .add_step(\"orchestrate_query\", \"graphrag_query\") \\\n", + " .build()\n", + "\n", + "# Execute pipeline using Semantica ExecutionEngine\n", + "execution_engine = ExecutionEngine()\n", + "pipeline_result = execution_engine.execute_pipeline(healthcare_graphrag_pipeline)\n", + "\n", + "print(\"✓ Built and executed complete GraphRAG pipeline using Semantica\")\n", + "print(f\" - Pipeline steps: {len(healthcare_graphrag_pipeline.steps)}\")\n", + "print(f\" - Execution status: {pipeline_result.success if hasattr(pipeline_result, 'success') else 'Completed'}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion and Best Practices\n", + "\n", + "### Key Takeaways\n", + "\n", + "1. **Semantica as Core Framework**: This notebook demonstrated using Semantica as the exclusive framework for building a Healthcare GraphRAG system\n", + "2. **Materialized Knowledge Graphs**: Semantica's KG modules enable building persistent knowledge graphs from medical ontologies and documents\n", + "3. **Virtual Data Integration**: Semantica's DBIngestor allows virtual integration with EHRs without data replication\n", + "4. **Hybrid Search**: Semantica's HybridSearch combines vector similarity with knowledge graph queries\n", + "5. **Query Orchestration**: Semantica's Reasoning and Triple Store modules enable dynamic query orchestration\n", + "6. **Explainability**: Semantica's ExplanationGenerator provides traceable, explainable answers\n", + "\n", + "### Semantica-Specific Performance Considerations\n", + "\n", + "- **Vector Store**: Use Semantica's VectorStore with appropriate backend (FAISS for local, Pinecone/Weaviate for cloud)\n", + "- **Graph Analytics**: Leverage Semantica's GraphAnalyzer for efficient centrality and community detection\n", + "- **Pipeline Execution**: Use Semantica's ExecutionEngine for parallel execution of pipeline steps\n", + "- **Caching**: Utilize Semantica's ContextRetriever caching for frequently accessed contexts\n", + "\n", + "### Deployment Recommendations Using Semantica\n", + "\n", + "1. **Production Setup**:\n", + " - Use Semantica's configuration management for environment-specific settings\n", + " - Leverage Semantica's Pipeline module for production workflows\n", + " - Use Semantica's export modules for data persistence\n", + "\n", + "2. **Scalability**:\n", + " - Use Semantica's batch processing capabilities for large-scale data ingestion\n", + " - Leverage Semantica's vector store adapters for distributed storage\n", + " - Utilize Semantica's parallel execution features\n", + "\n", + "3. **Compliance**:\n", + " - Semantica's virtual data integration ensures EHR data remains in original systems\n", + " - Use Semantica's audit logging for traceability\n", + " - Leverage Semantica's export modules for compliance reporting\n", + "\n", + "### How Semantica's Architecture Benefits Healthcare GraphRAG\n", + "\n", + "- **Unified Framework**: Single framework for all operations reduces integration complexity\n", + "- **Modular Design**: Semantica's modular architecture allows flexible deployment\n", + "- **Extensibility**: Semantica's registry system enables custom method registration\n", + "- **Type Safety**: Semantica's structured data models ensure data consistency\n", + "- **Performance**: Semantica's optimized algorithms provide efficient graph operations\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/healthcare/Medical_Database_Integration.ipynb b/docs/cookbook/use_cases/healthcare/Medical_Database_Integration.ipynb new file mode 100644 index 00000000..89aa6d91 --- /dev/null +++ b/docs/cookbook/use_cases/healthcare/Medical_Database_Integration.ipynb @@ -0,0 +1,601 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Medical Database Integration Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to integrate Python/FastMCP MCP servers as data sources for medical database ingestion. Connect to medical database MCP servers via URL, ingest patient records, drug interactions, and clinical data, then build a healthcare knowledge graph.\n", + "\n", + "**IMPORTANT**: This implementation supports ONLY Python-based MCP servers and FastMCP servers. Users can bring their own Python/FastMCP MCP servers via URL connections.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: MCPIngestor, ingest_mcp, DBIngestor, FileIngestor\n", + "- **Parsing**: MCPParser, JSONParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n", + "- **Triple Store**: TripleStore, TripleManager, QueryEngine\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ValidationEngine\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Connect to Medical MCP Server → Ingest Patient/Drug Data via MCP → Parse MCP Responses → Extract Medical Entities → Build Healthcare KG → Query & Analyze → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Connect to Medical Database MCP Server\n", + "\n", + "Connect to a Python/FastMCP MCP server that provides medical database access via URL. The MCP server can expose resources (patient records, drug databases) and tools (queries, drug interaction checks)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import MCPIngestor, ingest_mcp\n", + "from semantica.parse import MCPParser, JSONParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n", + "from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor, ValidationEngine\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Initialize MCP ingestor\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "# Connect to medical database MCP server via URL\n", + "# Replace with your actual MCP server URL\n", + "# Example: http://localhost:8000/mcp or https://api.example.com/medical-mcp\n", + "medical_mcp_url = \"http://localhost:8000/mcp\"\n", + "\n", + "try:\n", + " # Connect to MCP server with authentication (if required)\n", + " mcp_ingestor.connect(\n", + " \"medical_server\",\n", + " url=medical_mcp_url,\n", + " headers={\n", + " \"Authorization\": \"Bearer your_token\",\n", + " \"X-API-Key\": \"your_api_key\"\n", + " } if \"api.example.com\" in medical_mcp_url else {}\n", + " )\n", + " print(f\"✓ Connected to medical MCP server at {medical_mcp_url}\")\n", + " \n", + " # List available resources (patient records, drug databases)\n", + " resources = mcp_ingestor.list_available_resources(\"medical_server\")\n", + " print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + " for resource in resources[:5]: # Show first 5\n", + " print(f\" - {resource.uri}: {resource.name}\")\n", + " if resource.description:\n", + " print(f\" {resource.description[:80]}...\")\n", + " \n", + " # List available tools (queries, drug interaction checks)\n", + " tools = mcp_ingestor.list_available_tools(\"medical_server\")\n", + " print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + " for tool in tools[:5]: # Show first 5\n", + " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Connection failed: {e}\")\n", + " print(\"Note: This example uses a placeholder URL. Replace with your actual MCP server URL.\")\n", + " print(\"For testing, you can use a mock MCP server or skip connection and use sample data below.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Medical Data from MCP Server\n", + "\n", + "Ingest patient records, drug interactions, and clinical data using both resource-based and tool-based methods.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize parsers\n", + "mcp_parser = MCPParser()\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "medical_data = []\n", + "\n", + "# Method 1: Resource-based ingestion\n", + "# Ingest from MCP resources (patient records, drug databases)\n", + "try:\n", + " # Example: Ingest patient records resource\n", + " patient_data = mcp_ingestor.ingest_resources(\n", + " \"medical_server\",\n", + " resource_uris=[\"resource://patients/records\", \"resource://drugs/interactions\"]\n", + " )\n", + " \n", + " for item in patient_data:\n", + " medical_data.append(item)\n", + " print(f\"✓ Ingested resource: {item.resource_uri}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Resource ingestion: {e}\")\n", + "\n", + "# Method 2: Tool-based ingestion\n", + "# Call MCP tools to retrieve data dynamically\n", + "try:\n", + " # Example: Query patient records\n", + " patient_records = mcp_ingestor.ingest_tool_output(\n", + " \"medical_server\",\n", + " tool_name=\"query_patient_records\",\n", + " arguments={\n", + " \"patient_id\": \"P001\",\n", + " \"date_range\": {\n", + " \"start\": (datetime.now() - timedelta(days=365)).isoformat(),\n", + " \"end\": datetime.now().isoformat()\n", + " }\n", + " }\n", + " )\n", + " \n", + " if patient_records:\n", + " medical_data.append(patient_records)\n", + " print(f\"✓ Retrieved patient records via tool\")\n", + " \n", + " # Example: Check drug interactions\n", + " drug_interactions = mcp_ingestor.ingest_tool_output(\n", + " \"medical_server\",\n", + " tool_name=\"check_drug_interactions\",\n", + " arguments={\n", + " \"medications\": [\"Lisinopril\", \"Metformin\", \"Aspirin\"]\n", + " }\n", + " )\n", + " \n", + " if drug_interactions:\n", + " medical_data.append(drug_interactions)\n", + " print(f\"✓ Retrieved drug interactions via tool\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Tool-based ingestion: {e}\")\n", + " print(\"Note: Using sample data for demonstration\")\n", + "\n", + "# Sample medical data (if MCP server is not available)\n", + "if not medical_data:\n", + " print(\"\\n📝 Using sample medical data for demonstration:\")\n", + " sample_data = {\n", + " \"patient_records\": [\n", + " {\n", + " \"patient_id\": \"P001\",\n", + " \"visit_date\": (datetime.now() - timedelta(days=30)).isoformat(),\n", + " \"diagnosis\": [\"Hypertension\", \"Type 2 Diabetes\"],\n", + " \"medications\": [\"Lisinopril 10mg\", \"Metformin 500mg\"],\n", + " \"procedures\": [\"Blood Pressure Check\", \"HbA1c Test\"],\n", + " \"doctor\": \"Dr. Smith\",\n", + " \"notes\": \"Patient shows improvement in blood pressure control.\"\n", + " },\n", + " {\n", + " \"patient_id\": \"P002\",\n", + " \"visit_date\": (datetime.now() - timedelta(days=15)).isoformat(),\n", + " \"diagnosis\": [\"Asthma\"],\n", + " \"medications\": [\"Albuterol Inhaler\"],\n", + " \"procedures\": [\"Spirometry\"],\n", + " \"doctor\": \"Dr. Johnson\",\n", + " \"notes\": \"Asthma well controlled with current medication.\"\n", + " }\n", + " ],\n", + " \"drug_interactions\": [\n", + " {\n", + " \"drug1\": \"Lisinopril\",\n", + " \"drug2\": \"Aspirin\",\n", + " \"interaction_type\": \"moderate\",\n", + " \"description\": \"May increase risk of kidney problems\"\n", + " },\n", + " {\n", + " \"drug1\": \"Metformin\",\n", + " \"drug2\": \"Alcohol\",\n", + " \"interaction_type\": \"severe\",\n", + " \"description\": \"May cause lactic acidosis\"\n", + " }\n", + " ]\n", + " }\n", + " medical_data.append(sample_data)\n", + " print(f\" Loaded {len(sample_data['patient_records'])} patient records\")\n", + " print(f\" Loaded {len(sample_data['drug_interactions'])} drug interactions\")\n", + "\n", + "print(f\"\\n📊 Total medical data items ingested: {len(medical_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Parse MCP Medical Data\n", + "\n", + "Parse the medical data received from MCP server responses.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parsed_medical_data = []\n", + "\n", + "# Parse MCP responses\n", + "for data_item in medical_data:\n", + " try:\n", + " # Parse MCP response (handles JSON, text, binary)\n", + " if isinstance(data_item, dict):\n", + " parsed_item = data_item\n", + " else:\n", + " parsed_item = mcp_parser.parse_response(data_item, response_type=\"json\")\n", + " \n", + " parsed_medical_data.append(parsed_item)\n", + " \n", + " except Exception as e:\n", + " print(f\"⚠ Parsing error: {e}\")\n", + "\n", + "# Extract patient records and drug interactions\n", + "patient_records = []\n", + "drug_interactions = []\n", + "\n", + "for item in parsed_medical_data:\n", + " if isinstance(item, dict):\n", + " if \"patient_records\" in item:\n", + " patient_records.extend(item[\"patient_records\"])\n", + " elif \"patient_id\" in item:\n", + " patient_records.append(item)\n", + " elif \"drug_interactions\" in item:\n", + " drug_interactions.extend(item[\"drug_interactions\"])\n", + " elif \"drug1\" in item:\n", + " drug_interactions.append(item)\n", + "\n", + "print(f\"✓ Parsed {len(parsed_medical_data)} data items\")\n", + "print(f\"✓ Extracted {len(patient_records)} patient records\")\n", + "print(f\"✓ Extracted {len(drug_interactions)} drug interactions\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Medical Entities and Relationships\n", + "\n", + "Extract medical entities (patients, diagnoses, medications, procedures, doctors) and relationships from MCP data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "medical_entities = []\n", + "medical_relationships = []\n", + "\n", + "# Extract from patient records\n", + "for record in patient_records:\n", + " if isinstance(record, dict):\n", + " patient_id = record.get(\"patient_id\", \"\")\n", + " \n", + " # Patient entity\n", + " medical_entities.append({\n", + " \"id\": patient_id,\n", + " \"type\": \"Patient\",\n", + " \"name\": patient_id,\n", + " \"properties\": {\"visit_date\": record.get(\"visit_date\", \"\")}\n", + " })\n", + " \n", + " # Diagnoses\n", + " for diagnosis in record.get(\"diagnosis\", []):\n", + " medical_entities.append({\n", + " \"id\": diagnosis,\n", + " \"type\": \"Diagnosis\",\n", + " \"name\": diagnosis,\n", + " \"properties\": {}\n", + " })\n", + " medical_relationships.append({\n", + " \"source\": patient_id,\n", + " \"target\": diagnosis,\n", + " \"type\": \"has_diagnosis\",\n", + " \"properties\": {\"timestamp\": record.get(\"visit_date\", \"\")}\n", + " })\n", + " \n", + " # Medications\n", + " for medication in record.get(\"medications\", []):\n", + " medical_entities.append({\n", + " \"id\": medication,\n", + " \"type\": \"Medication\",\n", + " \"name\": medication,\n", + " \"properties\": {}\n", + " })\n", + " medical_relationships.append({\n", + " \"source\": patient_id,\n", + " \"target\": medication,\n", + " \"type\": \"prescribed\",\n", + " \"properties\": {\"timestamp\": record.get(\"visit_date\", \"\")}\n", + " })\n", + " \n", + " # Procedures\n", + " for procedure in record.get(\"procedures\", []):\n", + " medical_entities.append({\n", + " \"id\": procedure,\n", + " \"type\": \"Procedure\",\n", + " \"name\": procedure,\n", + " \"properties\": {}\n", + " })\n", + " medical_relationships.append({\n", + " \"source\": patient_id,\n", + " \"target\": procedure,\n", + " \"type\": \"underwent\",\n", + " \"properties\": {\"timestamp\": record.get(\"visit_date\", \"\")}\n", + " })\n", + " \n", + " # Doctor\n", + " doctor = record.get(\"doctor\", \"\")\n", + " if doctor:\n", + " medical_entities.append({\n", + " \"id\": doctor,\n", + " \"type\": \"Doctor\",\n", + " \"name\": doctor,\n", + " \"properties\": {}\n", + " })\n", + " medical_relationships.append({\n", + " \"source\": doctor,\n", + " \"target\": patient_id,\n", + " \"type\": \"treats\",\n", + " \"properties\": {\"timestamp\": record.get(\"visit_date\", \"\")}\n", + " })\n", + "\n", + "# Extract from drug interactions\n", + "for interaction in drug_interactions:\n", + " if isinstance(interaction, dict):\n", + " drug1 = interaction.get(\"drug1\", \"\")\n", + " drug2 = interaction.get(\"drug2\", \"\")\n", + " interaction_type = interaction.get(\"interaction_type\", \"\")\n", + " \n", + " if drug1 and drug2:\n", + " medical_relationships.append({\n", + " \"source\": drug1,\n", + " \"target\": drug2,\n", + " \"type\": \"interacts_with\",\n", + " \"properties\": {\n", + " \"interaction_type\": interaction_type,\n", + " \"description\": interaction.get(\"description\", \"\")\n", + " }\n", + " })\n", + "\n", + "# Remove duplicates\n", + "seen_entities = set()\n", + "unique_entities = []\n", + "for entity in medical_entities:\n", + " entity_key = (entity[\"id\"], entity[\"type\"])\n", + " if entity_key not in seen_entities:\n", + " seen_entities.add(entity_key)\n", + " unique_entities.append(entity)\n", + "\n", + "medical_entities = unique_entities\n", + "\n", + "print(f\"✓ Extracted {len(medical_entities)} medical entities\")\n", + "print(f\" - Patients: {len([e for e in medical_entities if e['type'] == 'Patient'])}\")\n", + "print(f\" - Diagnoses: {len([e for e in medical_entities if e['type'] == 'Diagnosis'])}\")\n", + "print(f\" - Medications: {len([e for e in medical_entities if e['type'] == 'Medication'])}\")\n", + "print(f\" - Procedures: {len([e for e in medical_entities if e['type'] == 'Procedure'])}\")\n", + "print(f\" - Doctors: {len([e for e in medical_entities if e['type'] == 'Doctor'])}\")\n", + "print(f\"✓ Extracted {len(medical_relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Healthcare Knowledge Graph\n", + "\n", + "Build a knowledge graph from the extracted medical entities and relationships, then store in triple store.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "graph_validator = GraphValidator()\n", + "entity_resolver = EntityResolver()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "# Build knowledge graph\n", + "medical_kg = builder.build(medical_entities, medical_relationships)\n", + "\n", + "# Validate and resolve entities\n", + "validated_kg = graph_validator.validate(medical_kg)\n", + "resolved_kg = entity_resolver.resolve_entities(validated_kg)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(resolved_kg)\n", + "\n", + "# Store in triple store\n", + "triple_store = TripleStore()\n", + "triple_manager = TripleManager()\n", + "query_engine = QueryEngine()\n", + "\n", + "triple_store.add_knowledge_graph(resolved_kg)\n", + "triple_manager.manage_triples(resolved_kg)\n", + "\n", + "print(f\"✓ Built healthcare knowledge graph\")\n", + "print(f\" Entities: {len(resolved_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(resolved_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\"✓ Stored in triple store\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Query and Analyze Medical Data\n", + "\n", + "Query the healthcare knowledge graph and analyze medical patterns.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query patient data\n", + "patient_query = query_engine.query(\n", + " \"SELECT ?patient ?diagnosis WHERE { ?patient has_diagnosis ?diagnosis }\"\n", + ")\n", + "\n", + "# Query drug interactions\n", + "interaction_query = query_engine.query(\n", + " \"SELECT ?drug1 ?drug2 ?type WHERE { ?drug1 interacts_with ?drug2 }\"\n", + ")\n", + "\n", + "# Inference engine for medical rules\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Medical analysis rules\n", + "inference_engine.add_rule(\"IF has_diagnosis(Diabetes) AND prescribed(Metformin) THEN diabetes_treatment\")\n", + "inference_engine.add_rule(\"IF interacts_with(Drug1, Drug2) AND interaction_type(severe) THEN contraindication\")\n", + "\n", + "# Add facts from medical data\n", + "for record in patient_records:\n", + " if isinstance(record, dict):\n", + " for diagnosis in record.get(\"diagnosis\", []):\n", + " inference_engine.add_fact({\n", + " \"patient\": record.get(\"patient_id\", \"\"),\n", + " \"diagnosis\": diagnosis\n", + " })\n", + " for medication in record.get(\"medications\", []):\n", + " inference_engine.add_fact({\n", + " \"patient\": record.get(\"patient_id\", \"\"),\n", + " \"medication\": medication\n", + " })\n", + "\n", + "for interaction in drug_interactions:\n", + " if isinstance(interaction, dict):\n", + " inference_engine.add_fact({\n", + " \"drug1\": interaction.get(\"drug1\", \"\"),\n", + " \"drug2\": interaction.get(\"drug2\", \"\"),\n", + " \"interaction_type\": interaction.get(\"interaction_type\", \"\")\n", + " })\n", + "\n", + "# Generate medical insights\n", + "medical_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"✓ Queried healthcare knowledge graph\")\n", + "print(f\" Patient-diagnosis relationships: {len(patient_query.get('results', []))}\")\n", + "print(f\" Drug interactions: {len(interaction_query.get('results', []))}\")\n", + "print(f\" Medical insights: {len(medical_insights)}\")\n", + "\n", + "# Quality assessment\n", + "quality_assessor = KGQualityAssessor()\n", + "validation_engine = ValidationEngine()\n", + "\n", + "quality_metrics = quality_assessor.assess_quality(resolved_kg)\n", + "validation_results = validation_engine.validate(resolved_kg)\n", + "\n", + "print(f\"✓ Quality assessment completed\")\n", + "print(f\" Quality score: {quality_metrics.get('overall_score', 0):.2f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Export and Visualize\n", + "\n", + "Export the healthcare knowledge graph and generate visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "import os\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export knowledge graph\n", + "json_exporter.export_knowledge_graph(resolved_kg, os.path.join(temp_dir, \"medical_kg.json\"))\n", + "rdf_exporter.export_knowledge_graph(resolved_kg, os.path.join(temp_dir, \"medical_kg.rdf\"))\n", + "owl_exporter.export_knowledge_graph(resolved_kg, os.path.join(temp_dir, \"medical_kg.owl\"))\n", + "\n", + "# Generate report\n", + "report_data = {\n", + " \"summary\": f\"Medical database integration from MCP server identified {len(medical_insights)} insights\",\n", + " \"patients\": len([e for e in medical_entities if e['type'] == 'Patient']),\n", + " \"diagnoses\": len([e for e in medical_entities if e['type'] == 'Diagnosis']),\n", + " \"medications\": len([e for e in medical_entities if e['type'] == 'Medication']),\n", + " \"drug_interactions\": len(drug_interactions),\n", + " \"insights\": len(medical_insights)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"✓ Exported healthcare knowledge graph\")\n", + "print(f\" JSON: {os.path.join(temp_dir, 'medical_kg.json')}\")\n", + "print(f\" RDF: {os.path.join(temp_dir, 'medical_kg.rdf')}\")\n", + "print(f\" OWL: {os.path.join(temp_dir, 'medical_kg.owl')}\")\n", + "print(f\"✓ Generated report ({len(report)} characters)\")\n", + "\n", + "# Visualize\n", + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(resolved_kg, output=\"interactive\")\n", + "ontology_viz = ontology_visualizer.visualize_ontology(resolved_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(resolved_kg, output=\"interactive\")\n", + "\n", + "print(\"✓ Generated visualizations for healthcare knowledge graph\")\n", + "\n", + "# Cleanup: Disconnect from MCP server\n", + "try:\n", + " mcp_ingestor.disconnect(\"medical_server\")\n", + " print(\"\\n✓ Disconnected from MCP server\")\n", + "except:\n", + " pass\n", + "\n", + "print(f\"\\n✅ Pipeline complete: MCP Server → Ingest → Parse → Extract → Build KG → Query → Export → Visualize\")\n", + "print(f\"📊 Total modules used: 20+\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/healthcare/Medical_Literature_GraphRAG.ipynb b/docs/cookbook/use_cases/healthcare/Medical_Literature_GraphRAG.ipynb new file mode 100644 index 00000000..89269788 --- /dev/null +++ b/docs/cookbook/use_cases/healthcare/Medical_Literature_GraphRAG.ipynb @@ -0,0 +1,373 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Medical Literature GraphRAG Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete medical literature GraphRAG pipeline: ingest research papers from multiple sources (PubMed, medical journals, research databases), extract findings, build research knowledge graph, generate embeddings, set up hybrid search (vector + KG), and query medical literature using advanced RAG.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: DocumentParser, PDFParser, HTMLParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder\n", + "- **Vector Store**: VectorStore, HybridSearch\n", + "- **Context**: ContextRetriever, ContextGraphBuilder\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Export**: JSONExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Research Papers → Parse → Extract Findings → Build Research KG → Generate Embeddings → Vector Store → GraphRAG Setup → Q&A → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Research Papers from Multiple Sources\n", + "\n", + "Ingest research papers from PubMed, medical journals, and research databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import DocumentParser, PDFParser, HTMLParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.vector_store import VectorStore, HybridSearch\n", + "from semantica.context import ContextRetriever, ContextGraphBuilder\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "document_parser = DocumentParser()\n", + "pdf_parser = PDFParser()\n", + "html_parser = HTMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real medical literature sources\n", + "medical_literature_sources = [\n", + " \"https://pubmed.ncbi.nlm.nih.gov/\", # PubMed\n", + " \"https://www.ncbi.nlm.nih.gov/pmc/\", # PubMed Central\n", + " \"https://www.biorxiv.org/\", # BioRxiv\n", + " \"https://www.medrxiv.org/\" # MedRxiv\n", + "]\n", + "\n", + "medical_feeds = [\n", + " \"https://www.cdc.gov/rss.xml\", # CDC Health Alerts\n", + " \"https://www.who.int/rss-feeds/news-english.xml\" # WHO News\n", + "]\n", + "\n", + "# Real database connection for medical literature\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/medical_literature_db\"\n", + "db_query = \"SELECT paper_id, title, authors, abstract, publication_date, findings FROM research_papers WHERE publication_date > CURRENT_DATE - INTERVAL '1 year' ORDER BY publication_date DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample research paper data\n", + "research_paper_file = os.path.join(temp_dir, \"research_paper.json\")\n", + "paper_data = {\n", + " \"paper_id\": \"PMID-2024-001\",\n", + " \"title\": \"Novel Treatment Approaches for Type 2 Diabetes\",\n", + " \"authors\": [\"Dr. Smith\", \"Dr. Jones\"],\n", + " \"abstract\": \"This study investigates novel treatment approaches for Type 2 Diabetes, focusing on combination therapies and lifestyle interventions.\",\n", + " \"publication_date\": (datetime.now() - timedelta(days=60)).isoformat(),\n", + " \"findings\": [\n", + " \"Metformin combined with lifestyle changes shows 30% improvement\",\n", + " \"Early intervention reduces complications by 40%\",\n", + " \"Personalized treatment plans improve patient outcomes\"\n", + " ],\n", + " \"keywords\": [\"Type 2 Diabetes\", \"Metformin\", \"Treatment\", \"Lifestyle\"]\n", + "}\n", + "\n", + "with open(research_paper_file, 'w') as f:\n", + " json.dump(paper_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(research_paper_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(research_paper_file)\n", + "\n", + "# Ingest from medical literature sources\n", + "literature_web_list = []\n", + "for source_url in medical_literature_sources[:1]:\n", + " try:\n", + " web_content = web_ingestor.ingest_url(source_url)\n", + " if web_content:\n", + " literature_web_list.append(web_content)\n", + " print(f\"✓ Ingested medical literature source: {web_content.url if hasattr(web_content, 'url') else source_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Medical literature source ingestion for {source_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Research papers: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Medical literature sources: {len(literature_web_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Findings and Build Research Knowledge Graph\n", + "\n", + "Extract findings from research papers and build knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "research_entities = []\n", + "research_relationships = []\n", + "all_documents = []\n", + "\n", + "# Extract from research paper data\n", + "if parsed_data and parsed_data.data:\n", + " paper = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " \n", + " if isinstance(paper, dict):\n", + " paper_text = f\"{paper.get('title', '')} {paper.get('abstract', '')}\"\n", + " all_documents.append(paper_text)\n", + " \n", + " research_entities.append({\n", + " \"id\": paper.get(\"paper_id\", \"\"),\n", + " \"type\": \"Research_Paper\",\n", + " \"name\": paper.get(\"title\", \"\"),\n", + " \"properties\": {\n", + " \"authors\": paper.get(\"authors\", []),\n", + " \"publication_date\": paper.get(\"publication_date\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Findings\n", + " for i, finding in enumerate(paper.get(\"findings\", [])):\n", + " research_entities.append({\n", + " \"id\": f\"{paper.get('paper_id', '')}_finding_{i}\",\n", + " \"type\": \"Finding\",\n", + " \"name\": finding,\n", + " \"properties\": {}\n", + " })\n", + " research_relationships.append({\n", + " \"source\": paper.get(\"paper_id\", \"\"),\n", + " \"target\": f\"{paper.get('paper_id', '')}_finding_{i}\",\n", + " \"type\": \"reports\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Keywords\n", + " for keyword in paper.get(\"keywords\", []):\n", + " research_entities.append({\n", + " \"id\": keyword,\n", + " \"type\": \"Keyword\",\n", + " \"name\": keyword,\n", + " \"properties\": {}\n", + " })\n", + " research_relationships.append({\n", + " \"source\": paper.get(\"paper_id\", \"\"),\n", + " \"target\": keyword,\n", + " \"type\": \"has_keyword\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "research_kg = builder.build(research_entities, research_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(research_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(research_kg)\n", + "\n", + "print(f\"Extracted {len(research_entities)} research entities\")\n", + "print(f\"Extracted {len(research_relationships)} relationships\")\n", + "print(f\"Collected {len(all_documents)} research documents\")\n", + "print(f\"Built research knowledge graph with {len(research_kg.get('entities', []))} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Generate Embeddings and Setup Vector Store\n", + "\n", + "Generate embeddings and setup vector store for GraphRAG.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "vector_store = VectorStore()\n", + "hybrid_search = HybridSearch()\n", + "\n", + "embeddings = embedding_generator.generate(all_documents)\n", + "\n", + "metadata = []\n", + "for i, doc in enumerate(all_documents):\n", + " metadata.append({\n", + " \"id\": f\"doc_{i}\",\n", + " \"text\": doc,\n", + " \"source\": \"medical_literature\"\n", + " })\n", + "\n", + "vector_ids = vector_store.store_vectors(embeddings, metadata)\n", + "\n", + "print(f\"Generated embeddings for {len(all_documents)} documents\")\n", + "print(f\"Stored {len(vector_ids)} vectors in vector store\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Setup GraphRAG and Query Medical Literature\n", + "\n", + "Setup hybrid search and query medical literature using GraphRAG.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "context_retriever = ContextRetriever(\n", + " knowledge_graph=research_kg,\n", + " vector_store=vector_store\n", + ")\n", + "\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Query examples\n", + "queries = [\n", + " \"What are the latest findings on diabetes treatment?\",\n", + " \"Find research on Metformin effectiveness\",\n", + " \"What studies show improvement in patient outcomes?\"\n", + "]\n", + "\n", + "query_results = []\n", + "for query in queries:\n", + " query_embedding = text_embedder.embed_text(query)\n", + " vector_results = vector_store.search_vectors(query_embedding, k=3)\n", + " \n", + " context_results = context_retriever.retrieve(\n", + " query=query,\n", + " top_k=3,\n", + " use_graph_expansion=True\n", + " )\n", + " \n", + " query_results.append({\n", + " \"query\": query,\n", + " \"vector_results\": len(vector_results),\n", + " \"context_results\": len(context_results) if context_results else 0\n", + " })\n", + "\n", + "# Medical research inference rules\n", + "inference_engine.add_rule(\"IF paper reports finding AND finding mentions improvement THEN positive_outcome\")\n", + "inference_engine.add_rule(\"IF paper has_keyword Treatment AND paper has_keyword Diabetes THEN treatment_research\")\n", + "\n", + "if parsed_data and parsed_data.data:\n", + " paper = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " if isinstance(paper, dict):\n", + " inference_engine.add_fact({\n", + " \"paper_id\": paper.get(\"paper_id\", \"\"),\n", + " \"keywords\": paper.get(\"keywords\", [])\n", + " })\n", + "\n", + "research_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Processed {len(queries)} medical literature queries\")\n", + "for result in query_results:\n", + " print(f\" Query: '{result['query']}' - Vector: {result['vector_results']}, Context: {result['context_results']}\")\n", + "print(f\"Generated {len(research_insights)} research insights\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate medical literature analysis reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(research_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(research_kg, os.path.join(temp_dir, \"research_kg.json\"))\n", + "rdf_exporter.export_knowledge_graph(research_kg, os.path.join(temp_dir, \"research_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Medical literature analysis identified {len(research_entities)} entities and {len(research_insights)} insights\",\n", + " \"papers_analyzed\": len([e for e in research_entities if e.get(\"type\") == \"Research_Paper\"]),\n", + " \"findings\": len([e for e in research_entities if e.get(\"type\") == \"Finding\"]),\n", + " \"insights\": len(research_insights),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(research_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(research_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(research_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated medical literature analysis report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Research Papers → Parse → Extract → Build KG → Embeddings → Vector Store → GraphRAG → Q&A → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/healthcare/Patient_Records_Temporal.ipynb b/docs/cookbook/use_cases/healthcare/Patient_Records_Temporal.ipynb new file mode 100644 index 00000000..6c9a66c7 --- /dev/null +++ b/docs/cookbook/use_cases/healthcare/Patient_Records_Temporal.ipynb @@ -0,0 +1,324 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Patient Records Temporal Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete patient records temporal analysis pipeline for healthcare: ingest patient records, extract medical entities, build temporal knowledge graph, query medical history, and generate insights.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, DBIngestor, StreamIngestor\n", + "- **Parsing**: DocumentParser, StructuredDataParser, CSVParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, CoreferenceResolver\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, GraphValidator, EntityResolver\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Triple Store**: TripleStore, TripleManager, QueryEngine\n", + "- **Export**: RDFExporter, OWLExporter, JSONExporter\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Patient Records → Parse → Extract Medical Entities → Build Temporal KG → Generate Ontology → Store in Triple Store → Query History → Export → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Process Patient Records\n", + "\n", + "Ingest and parse patient records from multiple sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, DBIngestor, StreamIngestor\n", + "from semantica.parse import DocumentParser, StructuredDataParser, CSVParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, CoreferenceResolver\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphValidator, EntityResolver\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n", + "from semantica.export import RDFExporter, OWLExporter, JSONExporter\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "stream_ingestor = StreamIngestor()\n", + "document_parser = DocumentParser()\n", + "structured_parser = StructuredDataParser()\n", + "csv_parser = CSVParser()\n", + "\n", + "# Real database connection for patient records (HIPAA compliant example)\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/patient_records_db\"\n", + "db_query = \"SELECT patient_id, visit_date, diagnosis, medication, doctor FROM patient_visits WHERE visit_date > CURRENT_DATE - INTERVAL '1 year' ORDER BY visit_date DESC\"\n", + "\n", + "# Real HL7/FHIR API endpoints (examples)\n", + "healthcare_apis = [\n", + " \"https://api.logicahealth.org/fhir/R4/Patient\", # Logica Health FHIR API\n", + " \"https://hapi.fhir.org/baseR4/Patient\" # HAPI FHIR Server\n", + "]\n", + "\n", + "# Real medical feed URLs\n", + "medical_feeds = [\n", + " \"https://www.cdc.gov/rss.xml\", # CDC Health Alerts\n", + " \"https://www.who.int/rss-feeds/news-english.xml\" # WHO News\n", + "]\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "patient_records_file = os.path.join(temp_dir, \"patient_records.csv\")\n", + "patient_data = \"\"\"patient_id,visit_date,diagnosis,medication,doctor\n", + "P001,2024-01-15,Hypertension,Lisinopril,Dr. Smith\n", + "P001,2024-02-20,Diabetes,Metformin,Dr. Jones\n", + "P002,2024-01-10,Fever,Acetaminophen,Dr. Smith\"\"\"\n", + "\n", + "with open(patient_records_file, 'w') as f:\n", + " f.write(patient_data)\n", + "\n", + "file_objects = file_ingestor.ingest_file(patient_records_file, read_content=True)\n", + "parsed_csv = csv_parser.parse(patient_records_file)\n", + "\n", + "print(f\"Ingested {len([file_objects]) if file_objects else 0} patient record files\")\n", + "print(f\"Parsed {len(parsed_csv.rows) if parsed_csv else 0} patient records\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Medical Entities\n", + "\n", + "Extract medical entities and relationships from patient records.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "coreference_resolver = CoreferenceResolver()\n", + "\n", + "patient_entities = []\n", + "relationships = []\n", + "\n", + "if parsed_csv and parsed_csv.rows:\n", + " for row in parsed_csv.rows:\n", + " patient_id = row.get(\"patient_id\", \"\")\n", + " diagnosis = row.get(\"diagnosis\", \"\")\n", + " medication = row.get(\"medication\", \"\")\n", + " doctor = row.get(\"doctor\", \"\")\n", + " visit_date = row.get(\"visit_date\", \"\")\n", + "\n", + " patient_entities.append({\n", + " \"id\": patient_id,\n", + " \"type\": \"Patient\",\n", + " \"name\": patient_id,\n", + " \"properties\": {}\n", + " })\n", + "\n", + " patient_entities.append({\n", + " \"id\": diagnosis,\n", + " \"type\": \"Diagnosis\",\n", + " \"name\": diagnosis,\n", + " \"properties\": {}\n", + " })\n", + "\n", + " patient_entities.append({\n", + " \"id\": medication,\n", + " \"type\": \"Medication\",\n", + " \"name\": medication,\n", + " \"properties\": {}\n", + " })\n", + "\n", + " patient_entities.append({\n", + " \"id\": doctor,\n", + " \"type\": \"Doctor\",\n", + " \"name\": doctor,\n", + " \"properties\": {}\n", + " })\n", + "\n", + " relationships.append({\n", + " \"source\": patient_id,\n", + " \"target\": diagnosis,\n", + " \"type\": \"has_diagnosis\",\n", + " \"properties\": {\"timestamp\": visit_date}\n", + " })\n", + "\n", + " relationships.append({\n", + " \"source\": patient_id,\n", + " \"target\": medication,\n", + " \"type\": \"prescribed\",\n", + " \"properties\": {\"timestamp\": visit_date}\n", + " })\n", + "\n", + " relationships.append({\n", + " \"source\": doctor,\n", + " \"target\": patient_id,\n", + " \"type\": \"treats\",\n", + " \"properties\": {\"timestamp\": visit_date}\n", + " })\n", + "\n", + "print(f\"Extracted {len(patient_entities)} medical entities\")\n", + "print(f\"Extracted {len(relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Temporal Patient Knowledge Graph\n", + "\n", + "Build a temporal knowledge graph from patient data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "entity_resolver = EntityResolver()\n", + "graph_validator = GraphValidator()\n", + "\n", + "resolved_entities = entity_resolver.resolve(patient_entities)\n", + "\n", + "patient_kg = builder.build(resolved_entities, relationships)\n", + "\n", + "validation_result = graph_validator.validate(patient_kg)\n", + "\n", + "print(f\"Built temporal patient knowledge graph\")\n", + "print(f\" Entities: {len(patient_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(patient_kg.get('relationships', []))}\")\n", + "print(f\" Graph valid: {validation_result.get('valid', False)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Generate Medical Ontology\n", + "\n", + "Generate ontology from medical entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "ontology = ontology_generator.generate(resolved_entities, relationships)\n", + "\n", + "classes = class_inferrer.infer_classes(resolved_entities)\n", + "properties = property_generator.infer_properties(resolved_entities, relationships, classes)\n", + "\n", + "validation_result = ontology_validator.validate_ontology(ontology)\n", + "\n", + "print(f\"Generated medical ontology\")\n", + "print(f\" Classes: {len(ontology.get('classes', []))}\")\n", + "print(f\" Properties: {len(ontology.get('properties', []))}\")\n", + "print(f\" Ontology valid: {validation_result.valid}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Store in Triple Store and Query\n", + "\n", + "Store knowledge graph in triple store and query medical history.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "triple_store = TripleStore()\n", + "triple_manager = TripleManager()\n", + "query_engine = QueryEngine()\n", + "temporal_query = TemporalGraphQuery()\n", + "\n", + "triple_store.store_knowledge_graph(patient_kg)\n", + "\n", + "patient_id = \"P001\"\n", + "start_time = \"2024-01-01\"\n", + "end_time = \"2024-12-31\"\n", + "\n", + "medical_history = temporal_query.query_time_range(\n", + " graph=patient_kg,\n", + " query=f\"Find medical history for patient {patient_id}\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "print(f\"Stored patient knowledge graph in triple store\")\n", + "print(f\"Retrieved {len(medical_history.get('entities', []))} medical events for patient {patient_id}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Export and Visualize\n", + "\n", + "Export patient data and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "json_exporter = JSONExporter()\n", + "\n", + "rdf_exporter.export_knowledge_graph(patient_kg, os.path.join(temp_dir, \"patient_kg.rdf\"))\n", + "owl_exporter.export(ontology, os.path.join(temp_dir, \"medical_ontology.owl\"))\n", + "json_exporter.export_knowledge_graph(patient_kg, os.path.join(temp_dir, \"patient_kg.json\"))\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(patient_kg, output=\"interactive\")\n", + "ontology_viz = ontology_visualizer.visualize_hierarchy(ontology, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(patient_kg, output=\"interactive\")\n", + "\n", + "print(\"Exported patient knowledge graph and ontology\")\n", + "print(\"Generated visualizations for knowledge graph, ontology, and temporal timeline\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Patient Records → Parse → Extract → Temporal KG → Ontology → Triple Store → Query → Export → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/intelligence/Network_Analysis_Intelligence_Reports.ipynb b/docs/cookbook/use_cases/intelligence/Network_Analysis_Intelligence_Reports.ipynb new file mode 100644 index 00000000..93a1e998 --- /dev/null +++ b/docs/cookbook/use_cases/intelligence/Network_Analysis_Intelligence_Reports.ipynb @@ -0,0 +1,779 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Network Analysis and Intelligence Reports with Semantica\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates using **Semantica as the core framework** to combine graph analytics with AI to analyze relational data and generate professional intelligence reports for fraud detection, cybersecurity, supply chain analysis, and criminal networks.\n", + "\n", + "### Why Semantica?\n", + "\n", + "Semantica provides a complete framework for network analysis and intelligence reporting:\n", + "\n", + "- **Graph Analytics**: Semantica's GraphAnalyzer provides community detection (Louvain) and centrality measures (PageRank, Betweenness, Closeness, Eigenvector)\n", + "- **Association Strength**: Semantica's algorithms calculate co-occurrence networks using association strength\n", + "- **Agent Coordination**: Semantica's Pipeline module enables parallel agent coordination for intelligence gathering\n", + "- **Pattern Detection**: Semantica's Reasoning modules identify patterns in complex networks\n", + "- **Report Generation**: Semantica's ReportGenerator creates professional HTML intelligence reports\n", + "- **Entity Resolution**: Semantica's Deduplication modules resolve entities in networks\n", + "\n", + "### Key Features\n", + "\n", + "- Transform relational data into co-occurrence networks using Semantica's association strength calculations\n", + "- Community detection (Louvain) and centrality measures (PageRank, Betweenness) using Semantica\n", + "- Parallel agent coordination using Semantica's Pipeline module\n", + "- Systematic prompt engineering techniques\n", + "- LLMs-as-judge evaluation systems\n", + "- Professional HTML report generation through coordinated AI agents using Semantica\n", + "- Graph data science + agentic AI coordination via Semantica\n", + "\n", + "### Semantica Modules Used (20+)\n", + "\n", + "- **Ingest**: FileIngestor, DBIngestor, WebIngestor (relational data from various sources)\n", + "- **Parse**: StructuredDataParser, CSVParser, JSONParser, DocumentParser (for various data formats)\n", + "- **Normalize**: TextNormalizer, DataNormalizer (for data cleaning and standardization)\n", + "- **Semantic Extract**: NERExtractor, RelationExtractor, TripleExtractor (relationship extraction, entity extraction)\n", + "- **KG**: GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer (network construction and analysis)\n", + "- **Graph Analytics**: Use Semantica's GraphAnalyzer for community detection (Louvain), centrality calculations (PageRank, Betweenness, Closeness, Eigenvector)\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder (for similarity calculations, entity embeddings)\n", + "- **Vector Store**: VectorStore, HybridSearch, MetadataFilter (for RAG and similarity search)\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator (for pattern detection, rule-based analysis)\n", + "- **Context**: ContextRetriever, ContextGraphBuilder (for contextual intelligence gathering)\n", + "- **Pipeline**: PipelineBuilder, ExecutionEngine, ParallelismManager (for orchestrating agent workflows)\n", + "- **Export**: ReportGenerator, HTMLExporter, JSONExporter (for professional intelligence reports)\n", + "- **Visualization**: KGVisualizer, AnalyticsVisualizer, QualityVisualizer (network visualization, analytics dashboards, report visualizations)\n", + "- **Deduplication**: DuplicateDetector, EntityMerger (for entity resolution in networks)\n", + "\n", + "### Pipeline Overview\n", + "\n", + "**Relational Data → Parse → Extract Entities/Relationships → Build Network Graph → Graph Analytics → Pattern Detection → Generate Intelligence Report → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Setup and Import Semantica Modules\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import all Semantica modules - using Semantica as the core framework\n", + "from semantica.ingest import FileIngestor, DBIngestor, WebIngestor\n", + "from semantica.parse import StructuredDataParser, CSVParser, JSONParser, DocumentParser\n", + "from semantica.normalize import TextNormalizer, DataNormalizer\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.vector_store import VectorStore, HybridSearch, MetadataFilter\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.context import ContextRetriever, ContextGraphBuilder\n", + "from semantica.pipeline import PipelineBuilder, ExecutionEngine, ParallelismManager\n", + "from semantica.export import ReportGenerator, JSONExporter\n", + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, QualityVisualizer\n", + "from semantica.deduplication import DuplicateDetector, EntityMerger\n", + "\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime\n", + "import numpy as np\n", + "\n", + "print(\"✓ All Semantica modules imported successfully\")\n", + "print(\"✓ Using Semantica as the core framework for Network Analysis and Intelligence Reports\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Relational Data Using Semantica\n", + "\n", + "Using Semantica's ingest modules to load relational data from various sources (CSV, JSON, databases).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica ingestors\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "web_ingestor = WebIngestor()\n", + "\n", + "# Create temporary directory for sample data\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample relational data (e.g., transaction network, communication network, etc.)\n", + "relational_data = {\n", + " \"entities\": [\n", + " {\"id\": \"E001\", \"name\": \"Entity A\", \"type\": \"Person\", \"attributes\": {\"age\": 35, \"location\": \"City1\"}},\n", + " {\"id\": \"E002\", \"name\": \"Entity B\", \"type\": \"Person\", \"attributes\": {\"age\": 42, \"location\": \"City1\"}},\n", + " {\"id\": \"E003\", \"name\": \"Entity C\", \"type\": \"Organization\", \"attributes\": {\"location\": \"City2\"}},\n", + " {\"id\": \"E004\", \"name\": \"Entity D\", \"type\": \"Person\", \"attributes\": {\"age\": 28, \"location\": \"City2\"}},\n", + " {\"id\": \"E005\", \"name\": \"Entity E\", \"type\": \"Person\", \"attributes\": {\"age\": 50, \"location\": \"City1\"}}\n", + " ],\n", + " \"relationships\": [\n", + " {\"source\": \"E001\", \"target\": \"E002\", \"type\": \"communicates_with\", \"frequency\": 15, \"date\": \"2024-01-15\"},\n", + " {\"source\": \"E001\", \"target\": \"E003\", \"type\": \"associated_with\", \"frequency\": 8, \"date\": \"2024-02-10\"},\n", + " {\"source\": \"E002\", \"target\": \"E003\", \"type\": \"communicates_with\", \"frequency\": 12, \"date\": \"2024-01-20\"},\n", + " {\"source\": \"E002\", \"target\": \"E004\", \"type\": \"communicates_with\", \"frequency\": 5, \"date\": \"2024-03-05\"},\n", + " {\"source\": \"E003\", \"target\": \"E004\", \"type\": \"associated_with\", \"frequency\": 20, \"date\": \"2024-02-15\"},\n", + " {\"source\": \"E004\", \"target\": \"E005\", \"type\": \"communicates_with\", \"frequency\": 3, \"date\": \"2024-03-10\"},\n", + " {\"source\": \"E001\", \"target\": \"E005\", \"type\": \"communicates_with\", \"frequency\": 10, \"date\": \"2024-01-25\"}\n", + " ]\n", + "}\n", + "\n", + "# Save sample data\n", + "relational_file = os.path.join(temp_dir, \"relational_data.json\")\n", + "with open(relational_file, 'w') as f:\n", + " json.dump(relational_data, f, indent=2)\n", + "\n", + "# Ingest using Semantica FileIngestor\n", + "relational_file_obj = file_ingestor.ingest_file(relational_file, read_content=True)\n", + "\n", + "print(f\"✓ Ingested relational data using Semantica\")\n", + "print(f\" - Entities: {len(relational_data['entities'])}\")\n", + "print(f\" - Relationships: {len(relational_data['relationships'])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Parse and Normalize Data Using Semantica\n", + "\n", + "Using Semantica's parse and normalize modules to process the relational data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica parsers and normalizers\n", + "structured_parser = StructuredDataParser()\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "text_normalizer = TextNormalizer()\n", + "data_normalizer = DataNormalizer()\n", + "\n", + "# Parse relational data using Semantica\n", + "parsed_data = structured_parser.parse_json(relational_file)\n", + "relational_data_parsed = parsed_data.data if hasattr(parsed_data, 'data') else parsed_data\n", + "\n", + "# Normalize entity names using Semantica\n", + "if isinstance(relational_data_parsed, dict):\n", + " for entity in relational_data_parsed.get('entities', []):\n", + " entity['normalized_name'] = text_normalizer.normalize(entity.get('name', ''))\n", + "\n", + "print(f\"✓ Parsed relational data using Semantica\")\n", + "print(f\" - Entities parsed: {len(relational_data_parsed.get('entities', [])) if isinstance(relational_data_parsed, dict) else 0}\")\n", + "print(f\" - Relationships parsed: {len(relational_data_parsed.get('relationships', [])) if isinstance(relational_data_parsed, dict) else 0}\")\n", + "print(f\"✓ Normalized entity data using Semantica\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Entities and Relationships Using Semantica\n", + "\n", + "Using Semantica's semantic extraction modules to extract entities, relationships, and build co-occurrence networks with association strength.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica extractors\n", + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "# Extract entities and relationships from relational data\n", + "network_entities = []\n", + "network_relationships = []\n", + "\n", + "# Process entities using Semantica\n", + "if isinstance(relational_data_parsed, dict):\n", + " for entity in relational_data_parsed.get('entities', []):\n", + " network_entities.append({\n", + " \"id\": entity.get('id', ''),\n", + " \"type\": entity.get('type', 'Entity'),\n", + " \"name\": entity.get('name', ''),\n", + " \"properties\": entity.get('attributes', {})\n", + " })\n", + "\n", + "# Process relationships and calculate association strength using Semantica\n", + "if isinstance(relational_data_parsed, dict):\n", + " # Calculate association strength (frequency-based)\n", + " relationship_strength = {}\n", + " for rel in relational_data_parsed.get('relationships', []):\n", + " source = rel.get('source', '')\n", + " target = rel.get('target', '')\n", + " rel_type = rel.get('type', 'related_to')\n", + " frequency = rel.get('frequency', 1)\n", + " \n", + " # Association strength calculation (normalized frequency)\n", + " key = f\"{source}_{target}_{rel_type}\"\n", + " relationship_strength[key] = relationship_strength.get(key, 0) + frequency\n", + " \n", + " network_relationships.append({\n", + " \"source\": source,\n", + " \"target\": target,\n", + " \"type\": rel_type,\n", + " \"properties\": {\n", + " \"frequency\": frequency,\n", + " \"association_strength\": relationship_strength[key] / max(relationship_strength.values()) if relationship_strength else 1.0,\n", + " \"date\": rel.get('date', '')\n", + " }\n", + " })\n", + "\n", + "print(f\"✓ Extracted {len(network_entities)} network entities using Semantica\")\n", + "print(f\"✓ Extracted {len(network_relationships)} relationships using Semantica\")\n", + "print(f\"✓ Calculated association strength for relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Network Knowledge Graph Using Semantica\n", + "\n", + "Using Semantica's KG modules to build the network knowledge graph from extracted entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica KG builders and analyzers\n", + "graph_builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "# Build network knowledge graph using Semantica\n", + "network_kg = graph_builder.build(network_entities, network_relationships)\n", + "\n", + "# Analyze the graph using Semantica\n", + "kg_metrics = graph_analyzer.compute_metrics(network_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(network_kg)\n", + "\n", + "print(f\"✓ Built network knowledge graph using Semantica\")\n", + "print(f\" - Entities: {len(network_kg.get('entities', []))}\")\n", + "print(f\" - Relationships: {len(network_kg.get('relationships', []))}\")\n", + "print(f\" - Graph density: {kg_metrics.get('density', 0):.4f}\")\n", + "print(f\" - Connected components: {connectivity.get('num_components', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Perform Graph Analytics Using Semantica\n", + "\n", + "Using Semantica's GraphAnalyzer to perform community detection (Louvain) and centrality measures (PageRank, Betweenness, Closeness, Eigenvector).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Perform graph analytics using Semantica GraphAnalyzer\n", + "\n", + "# 1. Community detection using Louvain algorithm (via Semantica)\n", + "communities = graph_analyzer.detect_communities(network_kg, method=\"louvain\")\n", + "\n", + "# 2. Centrality measures using Semantica\n", + "pagerank_centrality = graph_analyzer.compute_centrality(network_kg, method=\"pagerank\")\n", + "betweenness_centrality = graph_analyzer.compute_centrality(network_kg, method=\"betweenness\")\n", + "closeness_centrality = graph_analyzer.compute_centrality(network_kg, method=\"closeness\")\n", + "eigenvector_centrality = graph_analyzer.compute_centrality(network_kg, method=\"eigenvector\")\n", + "\n", + "# 3. Identify key entities (high centrality)\n", + "key_entities_pagerank = sorted(\n", + " [(node, score) for node, score in pagerank_centrality.items()],\n", + " key=lambda x: x[1],\n", + " reverse=True\n", + ")[:5]\n", + "\n", + "key_entities_betweenness = sorted(\n", + " [(node, score) for node, score in betweenness_centrality.items()],\n", + " key=lambda x: x[1],\n", + " reverse=True\n", + ")[:5]\n", + "\n", + "print(f\"✓ Performed graph analytics using Semantica\")\n", + "print(f\" - Communities detected (Louvain): {communities.get('num_communities', 0)}\")\n", + "print(f\" - PageRank centrality computed: {len(pagerank_centrality)} entities\")\n", + "print(f\" - Betweenness centrality computed: {len(betweenness_centrality)} entities\")\n", + "print(f\" - Closeness centrality computed: {len(closeness_centrality)} entities\")\n", + "print(f\" - Eigenvector centrality computed: {len(eigenvector_centrality)} entities\")\n", + "print(f\"\\nTop Key Entities (PageRank):\")\n", + "for entity_id, score in key_entities_pagerank:\n", + " entity_name = next((e.get('name', '') for e in network_entities if e.get('id') == entity_id), 'Unknown')\n", + " print(f\" - {entity_name} ({entity_id}): {score:.4f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Detect Patterns Using Semantica Reasoning\n", + "\n", + "Using Semantica's reasoning modules to detect patterns in the network.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica reasoning modules\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Define pattern detection rules using Semantica\n", + "pattern_rules = [\n", + " {\n", + " \"rule_id\": \"high_centrality_pattern\",\n", + " \"condition\": \"IF entity has high_pagerank AND entity has high_betweenness THEN entity is key_player\",\n", + " \"action\": \"flag_key_player\"\n", + " },\n", + " {\n", + " \"rule_id\": \"community_pattern\",\n", + " \"condition\": \"IF entities in same_community AND high_communication_frequency THEN entities form_cluster\",\n", + " \"action\": \"identify_cluster\"\n", + " },\n", + " {\n", + " \"rule_id\": \"bridge_pattern\",\n", + " \"condition\": \"IF entity has high_betweenness AND connects_communities THEN entity is_bridge\",\n", + " \"action\": \"flag_bridge_entity\"\n", + " }\n", + "]\n", + "\n", + "# Add rules using Semantica\n", + "for rule in pattern_rules:\n", + " rule_manager.add_rule(rule)\n", + "\n", + "# Apply pattern detection using Semantica InferenceEngine\n", + "pattern_results = inference_engine.infer(\n", + " knowledge_graph=network_kg,\n", + " rules=pattern_rules,\n", + " facts={\n", + " \"centrality\": pagerank_centrality,\n", + " \"communities\": communities,\n", + " \"betweenness\": betweenness_centrality\n", + " }\n", + ")\n", + "\n", + "print(f\"✓ Detected patterns using Semantica\")\n", + "print(f\" - Rules defined: {len(pattern_rules)}\")\n", + "print(f\" - Pattern detection results: {len(pattern_results) if isinstance(pattern_results, list) else 1}\")\n", + "print(f\" - Key players identified: {len(key_entities_pagerank)}\")\n", + "print(f\" - Clusters identified: {communities.get('num_communities', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Generate Embeddings and Setup Vector Store Using Semantica\n", + "\n", + "Using Semantica's embedding and vector store modules for similarity search and RAG capabilities.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica embedding and vector store modules\n", + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "vector_store = VectorStore(backend=\"faiss\", dimension=768)\n", + "hybrid_search = HybridSearch()\n", + "\n", + "# Generate embeddings for entities using Semantica\n", + "entity_embeddings = {}\n", + "for entity in network_entities:\n", + " entity_text = f\"{entity.get('name', '')} {entity.get('type', '')}\"\n", + " embedding = text_embedder.embed(entity_text)\n", + " entity_embeddings[entity.get('id', '')] = embedding\n", + "\n", + "# Store entity embeddings using Semantica\n", + "entity_metadata = [{\"entity_id\": eid, \"type\": \"entity\", \"name\": next((e.get('name', '') for e in network_entities if e.get('id') == eid), '')} for eid in entity_embeddings.keys()]\n", + "entity_ids = vector_store.store_vectors(\n", + " vectors=list(entity_embeddings.values()),\n", + " metadata=entity_metadata\n", + ")\n", + "\n", + "print(f\"✓ Generated embeddings using Semantica\")\n", + "print(f\" - Entity embeddings: {len(entity_embeddings)}\")\n", + "print(f\" - Vectors stored: {len(entity_ids)}\")\n", + "print(f\" - Hybrid search ready for intelligence queries\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Orchestrate Agent Workflows Using Semantica Pipeline\n", + "\n", + "Using Semantica's Pipeline module to coordinate parallel agents for intelligence gathering and analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica pipeline modules\n", + "pipeline_builder = PipelineBuilder()\n", + "execution_engine = ExecutionEngine()\n", + "parallelism_manager = ParallelismManager(max_workers=4)\n", + "\n", + "# Define agent workflows using Semantica Pipeline\n", + "# Agent 1: Network Structure Analysis\n", + "def agent_network_analysis(graph, analyzer):\n", + " \"\"\"Agent for network structure analysis.\"\"\"\n", + " metrics = analyzer.compute_metrics(graph)\n", + " centrality = analyzer.compute_centrality(graph, method=\"pagerank\")\n", + " return {\"metrics\": metrics, \"centrality\": centrality}\n", + "\n", + "# Agent 2: Community Detection\n", + "def agent_community_detection(graph, analyzer):\n", + " \"\"\"Agent for community detection.\"\"\"\n", + " communities = analyzer.detect_communities(graph, method=\"louvain\")\n", + " return {\"communities\": communities}\n", + "\n", + "# Agent 3: Pattern Detection\n", + "def agent_pattern_detection(graph, inference_engine, rules):\n", + " \"\"\"Agent for pattern detection.\"\"\"\n", + " patterns = inference_engine.infer(knowledge_graph=graph, rules=rules)\n", + " return {\"patterns\": patterns}\n", + "\n", + "# Build parallel agent pipeline using Semantica\n", + "intelligence_pipeline = pipeline_builder \\\n", + " .add_step(\"network_analysis\", \"custom\", func=agent_network_analysis, args=(network_kg, graph_analyzer)) \\\n", + " .add_step(\"community_detection\", \"custom\", func=agent_community_detection, args=(network_kg, graph_analyzer)) \\\n", + " .add_step(\"pattern_detection\", \"custom\", func=agent_pattern_detection, args=(network_kg, inference_engine, pattern_rules)) \\\n", + " .build()\n", + "\n", + "# Execute pipeline with parallel execution using Semantica\n", + "pipeline_result = execution_engine.execute_pipeline(intelligence_pipeline, parallel=True)\n", + "\n", + "print(\"✓ Orchestrated agent workflows using Semantica Pipeline\")\n", + "print(f\" - Pipeline steps: {len(intelligence_pipeline.steps)}\")\n", + "print(f\" - Parallel execution: Enabled\")\n", + "print(f\" - Execution status: {pipeline_result.success if hasattr(pipeline_result, 'success') else 'Completed'}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica context modules\n", + "context_retriever = ContextRetriever()\n", + "context_graph_builder = ContextGraphBuilder()\n", + "\n", + "# Build contextual intelligence using Semantica\n", + "context_graph = context_graph_builder.build(\n", + " entities=network_entities,\n", + " relationships=network_relationships,\n", + " query=\"Network intelligence analysis\"\n", + ")\n", + "\n", + "# Retrieve relevant context using Semantica\n", + "intelligence_context = context_retriever.retrieve(\n", + " query=\"key entities and relationships\",\n", + " knowledge_graph=network_kg,\n", + " top_k=10\n", + ")\n", + "\n", + "print(\"✓ Built contextual intelligence using Semantica\")\n", + "print(f\" - Context graph built: {bool(context_graph)}\")\n", + "print(f\" - Intelligence context retrieved: {len(intelligence_context) if isinstance(intelligence_context, list) else 1} items\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 11: Resolve Entity Duplicates Using Semantica Deduplication\n", + "\n", + "Using Semantica's deduplication modules to resolve entity duplicates in the network.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica deduplication modules\n", + "duplicate_detector = DuplicateDetector()\n", + "entity_merger = EntityMerger()\n", + "\n", + "# Detect duplicates using Semantica\n", + "duplicates = duplicate_detector.detect(network_entities, similarity_threshold=0.8)\n", + "\n", + "# Merge duplicate entities using Semantica\n", + "if duplicates:\n", + " merged_entities = entity_merger.merge(network_entities, duplicates)\n", + " print(f\"✓ Resolved entity duplicates using Semantica\")\n", + " print(f\" - Duplicates detected: {len(duplicates)}\")\n", + " print(f\" - Merged entities: {len(merged_entities) if merged_entities else len(network_entities)}\")\n", + "else:\n", + " print(f\"✓ No duplicates detected using Semantica\")\n", + " merged_entities = network_entities\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 12: Visualize Network and Analytics Using Semantica\n", + "\n", + "Using Semantica's visualization modules to visualize the network, communities, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica visualizers\n", + "kg_visualizer = KGVisualizer(layout=\"force\", color_scheme=\"vibrant\")\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "quality_visualizer = QualityVisualizer()\n", + "\n", + "# Visualize network using Semantica\n", + "network_fig = kg_visualizer.visualize_network(\n", + " network_kg,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "# Visualize communities using Semantica\n", + "communities_fig = kg_visualizer.visualize_communities(\n", + " network_kg,\n", + " communities,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "# Visualize centrality rankings using Semantica\n", + "centrality_fig = analytics_visualizer.visualize_centrality_rankings(\n", + " pagerank_centrality,\n", + " centrality_type=\"pagerank\",\n", + " top_n=10,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "# Visualize centrality comparison using Semantica\n", + "centrality_comparison = {\n", + " \"pagerank\": pagerank_centrality,\n", + " \"betweenness\": betweenness_centrality,\n", + " \"closeness\": closeness_centrality\n", + "}\n", + "comparison_fig = analytics_visualizer.visualize_centrality_comparison(\n", + " centrality_comparison,\n", + " top_n=10,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "print(\"✓ Visualized network and analytics using Semantica\")\n", + "print(\" - Network visualization: Interactive\")\n", + "print(\" - Community visualization: Louvain communities\")\n", + "print(\" - Centrality rankings: PageRank\")\n", + "print(\" - Centrality comparison: Multi-measure comparison\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica report generator\n", + "report_generator = ReportGenerator()\n", + "json_exporter = JSONExporter()\n", + "\n", + "# Prepare intelligence report data\n", + "intelligence_report_data = {\n", + " \"title\": \"Network Analysis Intelligence Report\",\n", + " \"executive_summary\": \"Analysis of network structure, key entities, communities, and patterns\",\n", + " \"knowledge_graph_metrics\": kg_metrics,\n", + " \"communities\": communities,\n", + " \"key_entities\": {\n", + " \"pagerank\": key_entities_pagerank,\n", + " \"betweenness\": key_entities_betweenness\n", + " },\n", + " \"centrality_measures\": {\n", + " \"pagerank\": pagerank_centrality,\n", + " \"betweenness\": betweenness_centrality,\n", + " \"closeness\": closeness_centrality,\n", + " \"eigenvector\": eigenvector_centrality\n", + " },\n", + " \"pattern_detection\": pattern_results,\n", + " \"network_structure\": {\n", + " \"entities\": len(network_entities),\n", + " \"relationships\": len(network_relationships),\n", + " \"density\": kg_metrics.get('density', 0),\n", + " \"components\": connectivity.get('num_components', 0)\n", + " },\n", + " \"context\": intelligence_context\n", + "}\n", + "\n", + "# Generate professional HTML report using Semantica\n", + "intelligence_report_file = os.path.join(temp_dir, \"intelligence_report.html\")\n", + "report_generator.generate_report(\n", + " intelligence_report_data,\n", + " intelligence_report_file,\n", + " format=\"html\"\n", + ")\n", + "\n", + "# Export network data as JSON using Semantica\n", + "network_json_file = os.path.join(temp_dir, \"network_data.json\")\n", + "json_exporter.export(network_kg, network_json_file)\n", + "\n", + "print(\"✓ Generated professional intelligence report using Semantica\")\n", + "print(f\" - Intelligence report HTML: {intelligence_report_file}\")\n", + "print(f\" - Network data JSON: {network_json_file}\")\n", + "print(f\" - Report includes: Executive summary, metrics, key entities, communities, patterns\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Build complete pipeline using Semantica PipelineBuilder\n", + "pipeline_builder = PipelineBuilder()\n", + "\n", + "network_analysis_pipeline = pipeline_builder \\\n", + " .add_step(\"ingest\", \"file_ingest\", source=temp_dir) \\\n", + " .add_step(\"parse\", \"structured_parse\", formats=[\"json\"]) \\\n", + " .add_step(\"normalize\", \"data_normalize\") \\\n", + " .add_step(\"extract\", \"semantic_extract\", entities=True, relations=True) \\\n", + " .add_step(\"build_kg\", \"kg_build\") \\\n", + " .add_step(\"graph_analytics\", \"graph_analyze\") \\\n", + " .add_step(\"pattern_detection\", \"reasoning_infer\") \\\n", + " .add_step(\"generate_embeddings\", \"embedding_generate\") \\\n", + " .add_step(\"setup_vector_store\", \"vector_store_setup\") \\\n", + " .add_step(\"build_context\", \"context_build\") \\\n", + " .add_step(\"deduplication\", \"deduplication_detect\") \\\n", + " .add_step(\"visualize\", \"visualize_network\") \\\n", + " .add_step(\"generate_report\", \"export_report\") \\\n", + " .build()\n", + "\n", + "# Execute pipeline using Semantica ExecutionEngine with parallel execution\n", + "execution_engine = ExecutionEngine()\n", + "pipeline_result = execution_engine.execute_pipeline(\n", + " network_analysis_pipeline,\n", + " parallel=True,\n", + " max_workers=4\n", + ")\n", + "\n", + "print(\"✓ Built and executed complete network analysis pipeline using Semantica\")\n", + "print(f\" - Pipeline steps: {len(network_analysis_pipeline.steps)}\")\n", + "print(f\" - Parallel execution: Enabled (4 workers)\")\n", + "print(f\" - Execution status: {pipeline_result.success if hasattr(pipeline_result, 'success') else 'Completed'}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion and Best Practices\n", + "\n", + "### Key Takeaways\n", + "\n", + "1. **Semantica as Core Framework**: This notebook demonstrated using Semantica as the exclusive framework for network analysis and intelligence reporting\n", + "2. **Graph Analytics**: Semantica's GraphAnalyzer provides comprehensive algorithms (Louvain, PageRank, Betweenness, Closeness, Eigenvector)\n", + "3. **Association Strength**: Semantica's algorithms calculate co-occurrence networks using association strength\n", + "4. **Agent Coordination**: Semantica's Pipeline module enables parallel agent coordination for intelligence gathering\n", + "5. **Pattern Detection**: Semantica's Reasoning modules identify patterns in complex networks\n", + "6. **Professional Reports**: Semantica's ReportGenerator creates professional HTML intelligence reports\n", + "7. **Entity Resolution**: Semantica's Deduplication modules resolve entities in networks\n", + "\n", + "### Semantica-Specific Performance Considerations\n", + "\n", + "- **Graph Analytics**: Use Semantica's GraphAnalyzer for efficient community detection and centrality calculations on large networks\n", + "- **Parallel Execution**: Leverage Semantica's ParallelismManager for concurrent agent execution\n", + "- **Vector Search**: Use Semantica's HybridSearch for efficient entity similarity search\n", + "- **Caching**: Utilize Semantica's ContextRetriever caching for frequently accessed contexts\n", + "\n", + "### Deployment Recommendations Using Semantica\n", + "\n", + "1. **Production Setup**:\n", + " - Use Semantica's configuration management for data source settings\n", + " - Leverage Semantica's Pipeline module for automated intelligence workflows\n", + " - Use Semantica's export modules for report persistence\n", + "\n", + "2. **Scalability**:\n", + " - Use Semantica's batch processing for large-scale network data\n", + " - Leverage Semantica's graph analytics optimizations\n", + " - Utilize Semantica's parallel execution for concurrent analysis\n", + "\n", + "3. **Quality Assurance**:\n", + " - Use Semantica's Deduplication modules for entity resolution\n", + " - Leverage Semantica's ExplanationGenerator for report traceability\n", + " - Utilize Semantica's quality modules for data validation\n", + "\n", + "### How Semantica's Architecture Benefits Network Analysis\n", + "\n", + "- **Comprehensive Analytics**: Semantica's GraphAnalyzer provides all necessary algorithms in one framework\n", + "- **Unified Pipeline**: Semantica's Pipeline module orchestrates complex multi-agent workflows\n", + "- **Extensibility**: Semantica's registry system enables custom analysis methods\n", + "- **Integration**: Semantica's unified framework simplifies integration with existing systems\n", + "- **Performance**: Semantica's optimized algorithms handle large-scale network analysis efficiently\n", + "- **Explainability**: Semantica's ExplanationGenerator provides traceable intelligence reports\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/renewable_energy/Energy_Market_Analysis.ipynb b/docs/cookbook/use_cases/renewable_energy/Energy_Market_Analysis.ipynb new file mode 100644 index 00000000..cc8bd870 --- /dev/null +++ b/docs/cookbook/use_cases/renewable_energy/Energy_Market_Analysis.ipynb @@ -0,0 +1,433 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Energy Market Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete energy market analysis pipeline: ingest energy market data from multiple sources (energy APIs, market data streams, databases), extract energy entities, build temporal market knowledge graph, analyze pricing trends, and predict market movements.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, StreamIngestor, DBIngestor, FeedIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Energy Market Sources → Parse → Extract Entities → Build Temporal Market KG → Analyze Pricing → Predict Trends → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Energy Market Data from Multiple Sources\n", + "\n", + "Ingest energy market data from energy APIs, market data streams, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, StreamIngestor, DBIngestor, FeedIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "stream_ingestor = StreamIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real energy market data sources\n", + "energy_apis = [\n", + " \"https://api.eia.gov/v2/electricity/retail-sales/data/\", # EIA Energy Information Administration\n", + " \"https://www.energy.gov/data\", # US Energy Department Data\n", + " \"https://api.github.com/repos/energy-data/aggregator\" # Energy data aggregator\n", + "]\n", + "\n", + "energy_feeds = [\n", + " \"https://www.energy.gov/rss\", # US Energy Department RSS\n", + " \"https://feeds.reuters.com/reuters/businessNews\" # Reuters Business (energy news)\n", + "]\n", + "\n", + "# Real streaming sources for energy market data\n", + "stream_sources = [\n", + " {\n", + " \"type\": \"kafka\",\n", + " \"topic\": \"energy_market\",\n", + " \"bootstrap_servers\": [\"localhost:9092\"],\n", + " \"consumer_config\": {\"group_id\": \"energy_analysis\"}\n", + " }\n", + "]\n", + "\n", + "# Real database connection for energy market data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/energy_market_db\"\n", + "db_query = \"SELECT energy_type, price, volume, timestamp, region FROM energy_prices WHERE timestamp > NOW() - INTERVAL '24 hours' ORDER BY timestamp DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample energy market data (real-world structure)\n", + "energy_market_file = os.path.join(temp_dir, \"energy_market.json\")\n", + "energy_market_data = [\n", + " {\n", + " \"energy_type\": \"Solar\",\n", + " \"price\": 0.045,\n", + " \"volume\": 1000000,\n", + " \"timestamp\": (datetime.now() - timedelta(hours=2)).isoformat(),\n", + " \"region\": \"California\"\n", + " },\n", + " {\n", + " \"energy_type\": \"Wind\",\n", + " \"price\": 0.035,\n", + " \"volume\": 800000,\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"region\": \"Texas\"\n", + " },\n", + " {\n", + " \"energy_type\": \"Hydroelectric\",\n", + " \"price\": 0.040,\n", + " \"volume\": 600000,\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"region\": \"Pacific Northwest\"\n", + " }\n", + "]\n", + "\n", + "with open(energy_market_file, 'w') as f:\n", + " json.dump(energy_market_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(energy_market_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(energy_market_file)\n", + "\n", + "# Ingest from energy APIs\n", + "energy_api_list = []\n", + "for api_url in energy_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " energy_api_list.append(api_content)\n", + " print(f\"✓ Ingested energy API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Energy API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "# Ingest from energy feeds\n", + "energy_feed_list = []\n", + "for feed_url in energy_feeds:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " energy_feed_list.append(feed_data)\n", + " print(f\"✓ Ingested energy feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Energy Market Ingestion Summary:\")\n", + "print(f\" Energy market files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Energy APIs: {len(energy_api_list)}\")\n", + "print(f\" Energy feeds: {len(energy_feed_list)}\")\n", + "print(f\" Streaming sources: {len(stream_sources)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Energy Entities and Build Temporal Market Knowledge Graph\n", + "\n", + "Extract energy entities and build temporal market knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "energy_entities = []\n", + "energy_relationships = []\n", + "\n", + "# Extract from energy market data\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " energy_type = entry.get(\"energy_type\", \"\")\n", + " region = entry.get(\"region\", \"\")\n", + " timestamp = entry.get(\"timestamp\", \"\")\n", + " \n", + " energy_entities.append({\n", + " \"id\": f\"{energy_type}_{region}_{timestamp}\",\n", + " \"type\": \"Energy_Price\",\n", + " \"name\": f\"{energy_type} in {region}\",\n", + " \"properties\": {\n", + " \"energy_type\": energy_type,\n", + " \"price\": entry.get(\"price\", 0),\n", + " \"volume\": entry.get(\"volume\", 0),\n", + " \"region\": region,\n", + " \"timestamp\": timestamp\n", + " }\n", + " })\n", + " \n", + " energy_entities.append({\n", + " \"id\": energy_type,\n", + " \"type\": \"Energy_Source\",\n", + " \"name\": energy_type,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " energy_entities.append({\n", + " \"id\": region,\n", + " \"type\": \"Region\",\n", + " \"name\": region,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " energy_relationships.append({\n", + " \"source\": energy_type,\n", + " \"target\": f\"{energy_type}_{region}_{timestamp}\",\n", + " \"type\": \"has_price_in\",\n", + " \"properties\": {\"timestamp\": timestamp}\n", + " })\n", + " \n", + " energy_relationships.append({\n", + " \"source\": f\"{energy_type}_{region}_{timestamp}\",\n", + " \"target\": region,\n", + " \"type\": \"in_region\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "energy_market_kg = builder.build(energy_entities, energy_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(energy_market_kg)\n", + "\n", + "print(f\"Extracted {len(energy_entities)} energy entities\")\n", + "print(f\"Extracted {len(energy_relationships)} relationships\")\n", + "print(f\"Built temporal energy market knowledge graph with {len(energy_market_kg.get('entities', []))} entities\")\n", + "print(f\"Graph density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Analyze Energy Pricing Trends\n", + "\n", + "Analyze energy pricing trends using temporal queries and pattern detection.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "start_time = (datetime.now() - timedelta(hours=24)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "# Query pricing trends\n", + "pricing_trends = temporal_query.query_time_range(\n", + " graph=energy_market_kg,\n", + " query=\"Find energy pricing trends in the last 24 hours\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "# Detect temporal patterns\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " energy_market_kg,\n", + " pattern_type=\"trend\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "# Analyze pricing by energy type\n", + "pricing_analysis = {}\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " energy_type = entry.get(\"energy_type\", \"\")\n", + " price = entry.get(\"price\", 0)\n", + " \n", + " if energy_type not in pricing_analysis:\n", + " pricing_analysis[energy_type] = {\n", + " \"prices\": [],\n", + " \"volumes\": [],\n", + " \"regions\": []\n", + " }\n", + " \n", + " pricing_analysis[energy_type][\"prices\"].append(price)\n", + " pricing_analysis[energy_type][\"volumes\"].append(entry.get(\"volume\", 0))\n", + " pricing_analysis[energy_type][\"regions\"].append(entry.get(\"region\", \"\"))\n", + "\n", + "# Calculate average prices\n", + "for energy_type, data in pricing_analysis.items():\n", + " if data[\"prices\"]:\n", + " pricing_analysis[energy_type][\"avg_price\"] = sum(data[\"prices\"]) / len(data[\"prices\"])\n", + " pricing_analysis[energy_type][\"total_volume\"] = sum(data[\"volumes\"])\n", + "\n", + "centrality_scores = centrality_calculator.calculate_centrality(energy_market_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(energy_market_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(energy_market_kg)\n", + "\n", + "print(f\"Pricing analysis complete\")\n", + "print(f\" Temporal patterns: {len(temporal_patterns)}\")\n", + "print(f\" Energy types analyzed: {len(pricing_analysis)}\")\n", + "for energy_type, data in pricing_analysis.items():\n", + " print(f\" {energy_type}: Avg Price ${data.get('avg_price', 0):.4f}/kWh, Total Volume {data.get('total_volume', 0):,}\")\n", + "print(f\" Central energy sources: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Predict Energy Market Trends\n", + "\n", + "Predict energy market trends using inference engine.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Energy market trend prediction rules\n", + "inference_engine.add_rule(\"IF price < 0.04 AND volume > 500000 THEN competitive_pricing\")\n", + "inference_engine.add_rule(\"IF price > 0.05 AND volume < 500000 THEN high_pricing\")\n", + "inference_engine.add_rule(\"IF multiple regions show same energy_type THEN market_trend\")\n", + "\n", + "# Add facts from energy market data\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " inference_engine.add_fact({\n", + " \"energy_type\": entry.get(\"energy_type\", \"\"),\n", + " \"price\": entry.get(\"price\", 0),\n", + " \"volume\": entry.get(\"volume\", 0),\n", + " \"region\": entry.get(\"region\", \"\")\n", + " })\n", + "\n", + "trend_predictions = inference_engine.forward_chain()\n", + "\n", + "# Generate trend predictions\n", + "market_predictions = []\n", + "for energy_type, data in pricing_analysis.items():\n", + " avg_price = data.get(\"avg_price\", 0)\n", + " total_volume = data.get(\"total_volume\", 0)\n", + " \n", + " prediction = {\n", + " \"energy_type\": energy_type,\n", + " \"current_avg_price\": avg_price,\n", + " \"total_volume\": total_volume,\n", + " \"trend\": \"increasing\" if avg_price > 0.04 else \"stable\" if avg_price > 0.035 else \"decreasing\",\n", + " \"market_share\": total_volume / sum(p.get(\"total_volume\", 0) for p in pricing_analysis.values()) if sum(p.get(\"total_volume\", 0) for p in pricing_analysis.values()) > 0 else 0\n", + " }\n", + " market_predictions.append(prediction)\n", + "\n", + "print(f\"Generated {len(trend_predictions)} trend predictions\")\n", + "print(f\"Market predictions for {len(market_predictions)} energy types:\")\n", + "for prediction in market_predictions:\n", + " print(f\" {prediction['energy_type']}: {prediction['trend']} trend, Market Share: {prediction['market_share']*100:.1f}%\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate energy market analysis reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(energy_market_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(energy_market_kg, os.path.join(temp_dir, \"energy_market_kg.json\"))\n", + "csv_exporter.export_entities(energy_entities, os.path.join(temp_dir, \"energy_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(energy_market_kg, os.path.join(temp_dir, \"energy_market_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Energy market analysis identified {len(trend_predictions)} trends and {len(market_predictions)} market predictions\",\n", + " \"energy_types_analyzed\": len(pricing_analysis),\n", + " \"patterns\": len(temporal_patterns),\n", + " \"predictions\": len(market_predictions),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(energy_market_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(energy_market_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(energy_market_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated energy market analysis report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Energy Market Sources → Parse → Extract → Build Temporal KG → Analyze Pricing → Predict Trends → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/renewable_energy/Environmental_Impact.ipynb b/docs/cookbook/use_cases/renewable_energy/Environmental_Impact.ipynb new file mode 100644 index 00000000..e4c7aa19 --- /dev/null +++ b/docs/cookbook/use_cases/renewable_energy/Environmental_Impact.ipynb @@ -0,0 +1,472 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Environmental Impact Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete environmental impact analysis pipeline: ingest environmental data from multiple sources (EPA APIs, climate databases, sustainability feeds), extract environmental entities, build impact knowledge graph, analyze relationships, and assess environmental impact.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Environmental Data Sources → Parse → Extract Entities → Build Impact KG → Analyze Relationships → Assess Impact → Generate Ontology → Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Environmental Data from Multiple Sources\n", + "\n", + "Ingest environmental data from EPA APIs, climate databases, and sustainability feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "# Real environmental data sources\n", + "environmental_apis = [\n", + " \"https://www.epa.gov/enviro/facts-service\", # EPA Environmental Facts Service\n", + " \"https://www.epa.gov/airdata\", # EPA Air Data\n", + " \"https://api.github.com/repos/climate-data/aggregator\" # Climate data aggregator\n", + "]\n", + "\n", + "environmental_feeds = [\n", + " \"https://www.epa.gov/rss\", # EPA RSS Feed\n", + " \"https://www.energy.gov/rss\", # US Energy Department RSS\n", + " \"https://feeds.reuters.com/reuters/environment\" # Reuters Environment News\n", + "]\n", + "\n", + "# Real database connection for environmental data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/environmental_db\"\n", + "db_query = \"SELECT project_id, energy_type, co2_reduction, water_saved, land_impact, timestamp FROM environmental_impact WHERE timestamp > CURRENT_DATE - INTERVAL '1 year' ORDER BY timestamp DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample environmental impact data\n", + "environmental_file = os.path.join(temp_dir, \"environmental_impact.json\")\n", + "environmental_data = [\n", + " {\n", + " \"project_id\": \"PROJ-001\",\n", + " \"energy_type\": \"Solar\",\n", + " \"co2_reduction_tons\": 5000,\n", + " \"water_saved_gallons\": 1000000,\n", + " \"land_impact_acres\": 50,\n", + " \"carbon_offset\": 5000,\n", + " \"timestamp\": (datetime.now() - timedelta(days=60)).isoformat(),\n", + " \"region\": \"California\"\n", + " },\n", + " {\n", + " \"project_id\": \"PROJ-002\",\n", + " \"energy_type\": \"Wind\",\n", + " \"co2_reduction_tons\": 8000,\n", + " \"water_saved_gallons\": 2000000,\n", + " \"land_impact_acres\": 100,\n", + " \"carbon_offset\": 8000,\n", + " \"timestamp\": (datetime.now() - timedelta(days=30)).isoformat(),\n", + " \"region\": \"Texas\"\n", + " },\n", + " {\n", + " \"project_id\": \"PROJ-003\",\n", + " \"energy_type\": \"Hydroelectric\",\n", + " \"co2_reduction_tons\": 3000,\n", + " \"water_saved_gallons\": 500000,\n", + " \"land_impact_acres\": 200,\n", + " \"carbon_offset\": 3000,\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"region\": \"Pacific Northwest\"\n", + " }\n", + "]\n", + "\n", + "with open(environmental_file, 'w') as f:\n", + " json.dump(environmental_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(environmental_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(environmental_file)\n", + "\n", + "# Ingest from environmental APIs\n", + "environmental_api_list = []\n", + "for api_url in environmental_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " environmental_api_list.append(api_content)\n", + " print(f\"✓ Ingested environmental API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Environmental API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "# Ingest from environmental feeds\n", + "environmental_feed_list = []\n", + "for feed_url in environmental_feeds:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " environmental_feed_list.append(feed_data)\n", + " print(f\"✓ Ingested environmental feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Environmental Data Ingestion Summary:\")\n", + "print(f\" Environmental data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Environmental APIs: {len(environmental_api_list)}\")\n", + "print(f\" Environmental feeds: {len(environmental_feed_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Environmental Entities and Build Impact Knowledge Graph\n", + "\n", + "Extract environmental entities and build impact knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "environmental_entities = []\n", + "environmental_relationships = []\n", + "\n", + "# Extract from environmental data\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " project_id = entry.get(\"project_id\", \"\")\n", + " energy_type = entry.get(\"energy_type\", \"\")\n", + " region = entry.get(\"region\", \"\")\n", + " \n", + " environmental_entities.append({\n", + " \"id\": project_id,\n", + " \"type\": \"Project\",\n", + " \"name\": project_id,\n", + " \"properties\": {\n", + " \"energy_type\": energy_type,\n", + " \"region\": region,\n", + " \"timestamp\": entry.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " environmental_entities.append({\n", + " \"id\": energy_type,\n", + " \"type\": \"Energy_Source\",\n", + " \"name\": energy_type,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " environmental_entities.append({\n", + " \"id\": f\"{project_id}_co2_reduction\",\n", + " \"type\": \"Environmental_Impact\",\n", + " \"name\": \"CO2 Reduction\",\n", + " \"properties\": {\n", + " \"metric\": \"CO2\",\n", + " \"value\": entry.get(\"co2_reduction_tons\", 0),\n", + " \"unit\": \"tons\"\n", + " }\n", + " })\n", + " \n", + " environmental_entities.append({\n", + " \"id\": f\"{project_id}_water_saved\",\n", + " \"type\": \"Environmental_Impact\",\n", + " \"name\": \"Water Saved\",\n", + " \"properties\": {\n", + " \"metric\": \"Water\",\n", + " \"value\": entry.get(\"water_saved_gallons\", 0),\n", + " \"unit\": \"gallons\"\n", + " }\n", + " })\n", + " \n", + " environmental_relationships.append({\n", + " \"source\": project_id,\n", + " \"target\": energy_type,\n", + " \"type\": \"uses\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " environmental_relationships.append({\n", + " \"source\": project_id,\n", + " \"target\": f\"{project_id}_co2_reduction\",\n", + " \"type\": \"reduces\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " environmental_relationships.append({\n", + " \"source\": project_id,\n", + " \"target\": f\"{project_id}_water_saved\",\n", + " \"type\": \"saves\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "\n", + "impact_kg = builder.build(environmental_entities, environmental_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(impact_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(impact_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(impact_kg)\n", + "\n", + "print(f\"Extracted {len(environmental_entities)} environmental entities\")\n", + "print(f\"Extracted {len(environmental_relationships)} relationships\")\n", + "print(f\"Built impact knowledge graph with {len(impact_kg.get('entities', []))} entities\")\n", + "print(f\"Graph density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Analyze Environmental Relationships\n", + "\n", + "Analyze environmental relationships using graph analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "\n", + "connectivity = connectivity_analyzer.analyze_connectivity(impact_kg)\n", + "\n", + "start_time = (datetime.now() - timedelta(days=365)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=impact_kg,\n", + " query=\"Find environmental impacts in the last year\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " impact_kg,\n", + " pattern_type=\"trend\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "# Analyze impact by energy type\n", + "impact_analysis = {}\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " energy_type = entry.get(\"energy_type\", \"\")\n", + " \n", + " if energy_type not in impact_analysis:\n", + " impact_analysis[energy_type] = {\n", + " \"co2_reduction\": [],\n", + " \"water_saved\": [],\n", + " \"projects\": []\n", + " }\n", + " \n", + " impact_analysis[energy_type][\"co2_reduction\"].append(entry.get(\"co2_reduction_tons\", 0))\n", + " impact_analysis[energy_type][\"water_saved\"].append(entry.get(\"water_saved_gallons\", 0))\n", + " impact_analysis[energy_type][\"projects\"].append(entry.get(\"project_id\", \"\"))\n", + "\n", + "# Calculate totals\n", + "for energy_type, data in impact_analysis.items():\n", + " impact_analysis[energy_type][\"total_co2_reduction\"] = sum(data[\"co2_reduction\"])\n", + " impact_analysis[energy_type][\"total_water_saved\"] = sum(data[\"water_saved\"])\n", + " impact_analysis[energy_type][\"project_count\"] = len(data[\"projects\"])\n", + "\n", + "print(f\"Environmental relationships analyzed\")\n", + "print(f\" Connected components: {len(connectivity.get('components', []))}\")\n", + "print(f\" Temporal patterns: {len(temporal_patterns)}\")\n", + "print(f\" Energy types analyzed: {len(impact_analysis)}\")\n", + "for energy_type, data in impact_analysis.items():\n", + " print(f\" {energy_type}: {data['total_co2_reduction']:,} tons CO2 reduced, {data['total_water_saved']:,} gallons saved, {data['project_count']} projects\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Assess Environmental Impact\n", + "\n", + "Assess environmental impact using inference engine and generate ontology.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "# Environmental impact assessment rules\n", + "inference_engine.add_rule(\"IF co2_reduction > 5000 AND water_saved > 1000000 THEN high_impact_project\")\n", + "inference_engine.add_rule(\"IF energy_type is Solar AND co2_reduction > 3000 THEN sustainable_solar\")\n", + "inference_engine.add_rule(\"IF multiple projects use same energy_type THEN scalable_solution\")\n", + "\n", + "# Assess impact\n", + "impact_assessments = []\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " co2_reduction = entry.get(\"co2_reduction_tons\", 0)\n", + " water_saved = entry.get(\"water_saved_gallons\", 0)\n", + " \n", + " impact_score = (co2_reduction / 1000) + (water_saved / 100000)\n", + " impact_level = \"high\" if impact_score > 10 else \"medium\" if impact_score > 5 else \"low\"\n", + " \n", + " assessment = {\n", + " \"project_id\": entry.get(\"project_id\", \"\"),\n", + " \"energy_type\": entry.get(\"energy_type\", \"\"),\n", + " \"co2_reduction\": co2_reduction,\n", + " \"water_saved\": water_saved,\n", + " \"impact_score\": impact_score,\n", + " \"impact_level\": impact_level\n", + " }\n", + " impact_assessments.append(assessment)\n", + " \n", + " inference_engine.add_fact({\n", + " \"project_id\": entry.get(\"project_id\", \"\"),\n", + " \"energy_type\": entry.get(\"energy_type\", \"\"),\n", + " \"co2_reduction\": co2_reduction,\n", + " \"water_saved\": water_saved\n", + " })\n", + "\n", + "impact_insights = inference_engine.forward_chain()\n", + "\n", + "# Generate environmental ontology\n", + "impact_ontology = ontology_generator.generate(environmental_entities, environmental_relationships)\n", + "classes = class_inferrer.infer_classes(environmental_entities)\n", + "properties = property_generator.infer_properties(environmental_entities, environmental_relationships, classes)\n", + "validation_result = ontology_validator.validate_ontology(impact_ontology)\n", + "\n", + "print(f\"Environmental impact assessment complete\")\n", + "print(f\" Projects assessed: {len(impact_assessments)}\")\n", + "print(f\" High impact projects: {len([a for a in impact_assessments if a.get('impact_level') == 'high'])}\")\n", + "print(f\" Generated {len(impact_insights)} impact insights\")\n", + "print(f\" Ontology valid: {validation_result.valid}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate environmental impact reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(impact_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(impact_kg, os.path.join(temp_dir, \"environmental_impact_kg.json\"))\n", + "csv_exporter.export_entities(environmental_entities, os.path.join(temp_dir, \"environmental_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(impact_kg, os.path.join(temp_dir, \"environmental_impact_kg.rdf\"))\n", + "owl_exporter.export(impact_ontology, os.path.join(temp_dir, \"environmental_ontology.owl\"))\n", + "\n", + "total_co2_reduction = sum(a.get(\"co2_reduction\", 0) for a in impact_assessments)\n", + "total_water_saved = sum(a.get(\"water_saved\", 0) for a in impact_assessments)\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Environmental impact analysis identified {len(impact_assessments)} projects with {total_co2_reduction:,} tons CO2 reduction and {total_water_saved:,} gallons water saved\",\n", + " \"projects_analyzed\": len(impact_assessments),\n", + " \"total_co2_reduction\": total_co2_reduction,\n", + " \"total_water_saved\": total_water_saved,\n", + " \"high_impact_projects\": len([a for a in impact_assessments if a.get(\"impact_level\") == \"high\"]),\n", + " \"insights\": len(impact_insights),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(impact_kg, output=\"interactive\")\n", + "ontology_viz = ontology_visualizer.visualize_hierarchy(impact_ontology, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(impact_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated environmental impact report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Environmental Data → Parse → Extract → Build Impact KG → Analyze Relationships → Assess Impact → Generate Ontology → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/renewable_energy/Grid_Management.ipynb b/docs/cookbook/use_cases/renewable_energy/Grid_Management.ipynb new file mode 100644 index 00000000..1580ff53 --- /dev/null +++ b/docs/cookbook/use_cases/renewable_energy/Grid_Management.ipynb @@ -0,0 +1,455 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Smart Grid Management Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete smart grid management pipeline: stream grid data from multiple sources (grid sensors, SCADA systems, databases), build temporal grid knowledge graph, monitor grid health in real-time, detect anomalies, and predict failures.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: StreamIngestor, FileIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser, CSVParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, AutomatedFixer\n", + "- **Export**: JSONExporter, CSVExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Stream Grid Data → Parse → Extract Entities → Build Temporal Grid KG → Monitor Grid Health → Detect Anomalies → Predict Failures → Generate Alerts → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Stream Grid Data from Multiple Sources\n", + "\n", + "Stream grid data from grid sensors, SCADA systems, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import StreamIngestor, FileIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, StructuredDataParser, CSVParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor, AutomatedFixer\n", + "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "import time\n", + "from datetime import datetime, timedelta\n", + "from collections import deque\n", + "\n", + "stream_ingestor = StreamIngestor()\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "csv_parser = CSVParser()\n", + "\n", + "# Real streaming sources for grid data\n", + "stream_sources = [\n", + " {\n", + " \"type\": \"kafka\",\n", + " \"topic\": \"grid_sensors\",\n", + " \"bootstrap_servers\": [\"localhost:9092\"],\n", + " \"consumer_config\": {\"group_id\": \"grid_monitor\"}\n", + " },\n", + " {\n", + " \"type\": \"rabbitmq\",\n", + " \"queue\": \"scada_events\",\n", + " \"connection_url\": \"amqp://user:password@localhost:5672/\"\n", + " }\n", + "]\n", + "\n", + "# Real grid monitoring APIs\n", + "grid_apis = [\n", + " \"https://api.eia.gov/v2/electricity/operating-generator-capacity/data/\", # EIA Grid Capacity\n", + " \"https://www.energy.gov/data\" # US Energy Department Grid Data\n", + "]\n", + "\n", + "# Real database connection for grid data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/grid_db\"\n", + "db_query = \"SELECT node_id, voltage, current, power, frequency, status, timestamp FROM grid_nodes WHERE timestamp > NOW() - INTERVAL '1 hour' ORDER BY timestamp DESC LIMIT 1000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample real-time grid data (simulating SCADA system)\n", + "grid_stream_file = os.path.join(temp_dir, \"grid_stream.json\")\n", + "grid_stream = [\n", + " {\n", + " \"node_id\": \"GRID-001\",\n", + " \"voltage\": 230.0,\n", + " \"current\": 100.0,\n", + " \"power\": 23000.0,\n", + " \"frequency\": 60.0,\n", + " \"status\": \"normal\",\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=5)).isoformat(),\n", + " \"location\": \"Substation A\"\n", + " },\n", + " {\n", + " \"node_id\": \"GRID-002\",\n", + " \"voltage\": 225.0,\n", + " \"current\": 95.0,\n", + " \"power\": 21375.0,\n", + " \"frequency\": 59.8,\n", + " \"status\": \"warning\",\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=4)).isoformat(),\n", + " \"location\": \"Substation B\"\n", + " },\n", + " {\n", + " \"node_id\": \"GRID-003\",\n", + " \"voltage\": 235.0,\n", + " \"current\": 105.0,\n", + " \"power\": 24675.0,\n", + " \"frequency\": 60.2,\n", + " \"status\": \"normal\",\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=3)).isoformat(),\n", + " \"location\": \"Substation C\"\n", + " },\n", + " {\n", + " \"node_id\": \"GRID-004\",\n", + " \"voltage\": 200.0,\n", + " \"current\": 80.0,\n", + " \"power\": 16000.0,\n", + " \"frequency\": 58.5,\n", + " \"status\": \"critical\",\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=2)).isoformat(),\n", + " \"location\": \"Substation D\"\n", + " }\n", + "]\n", + "\n", + "with open(grid_stream_file, 'w') as f:\n", + " json.dump(grid_stream, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(grid_stream_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(grid_stream_file)\n", + "\n", + "print(f\"\\n📊 Grid Data Ingestion Summary:\")\n", + "print(f\" Grid stream files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Streaming sources: {len(stream_sources)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Grid Entities and Build Temporal Grid Knowledge Graph\n", + "\n", + "Extract grid entities and build temporal grid knowledge graph for real-time monitoring.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "grid_entities = []\n", + "grid_relationships = []\n", + "\n", + "# Extract from grid stream data\n", + "if parsed_data and parsed_data.data:\n", + " for node in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(node, dict):\n", + " node_id = node.get(\"node_id\", \"\")\n", + " location = node.get(\"location\", \"\")\n", + " \n", + " grid_entities.append({\n", + " \"id\": node_id,\n", + " \"type\": \"Grid_Node\",\n", + " \"name\": node_id,\n", + " \"properties\": {\n", + " \"voltage\": node.get(\"voltage\", 0),\n", + " \"current\": node.get(\"current\", 0),\n", + " \"power\": node.get(\"power\", 0),\n", + " \"frequency\": node.get(\"frequency\", 0),\n", + " \"status\": node.get(\"status\", \"\"),\n", + " \"location\": location,\n", + " \"timestamp\": node.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " grid_entities.append({\n", + " \"id\": location,\n", + " \"type\": \"Location\",\n", + " \"name\": location,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " grid_relationships.append({\n", + " \"source\": node_id,\n", + " \"target\": location,\n", + " \"type\": \"located_at\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Status relationships\n", + " if node.get(\"status\") != \"normal\":\n", + " grid_entities.append({\n", + " \"id\": f\"{node_id}_status\",\n", + " \"type\": \"Grid_Status\",\n", + " \"name\": node.get(\"status\", \"\"),\n", + " \"properties\": {}\n", + " })\n", + " grid_relationships.append({\n", + " \"source\": node_id,\n", + " \"target\": f\"{node_id}_status\",\n", + " \"type\": \"has_status\",\n", + " \"properties\": {\"timestamp\": node.get(\"timestamp\", \"\")}\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "grid_kg = builder.build(grid_entities, grid_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(grid_kg)\n", + "\n", + "print(f\"Extracted {len(grid_entities)} grid entities\")\n", + "print(f\"Extracted {len(grid_relationships)} relationships\")\n", + "print(f\"Built temporal grid knowledge graph with {len(grid_kg.get('entities', []))} entities\")\n", + "print(f\"Graph density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Monitor Grid Health in Real-Time\n", + "\n", + "Monitor grid health using temporal queries and pattern detection.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "current_time = datetime.now().isoformat()\n", + "start_time = (datetime.now() - timedelta(minutes=10)).isoformat()\n", + "\n", + "# Query current grid status\n", + "current_grid_status = temporal_query.query_time_range(\n", + " graph=grid_kg,\n", + " query=\"Find current grid status\",\n", + " start_time=start_time,\n", + " end_time=current_time\n", + ")\n", + "\n", + "# Monitor grid health metrics\n", + "grid_health_metrics = []\n", + "if parsed_data and parsed_data.data:\n", + " for node in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(node, dict):\n", + " voltage = node.get(\"voltage\", 0)\n", + " frequency = node.get(\"frequency\", 0)\n", + " status = node.get(\"status\", \"\")\n", + " \n", + " health_score = 100\n", + " issues = []\n", + " \n", + " # Voltage check (normal: 220-240V)\n", + " if voltage < 210 or voltage > 250:\n", + " health_score -= 20\n", + " issues.append(\"Voltage out of range\")\n", + " \n", + " # Frequency check (normal: 59.5-60.5 Hz)\n", + " if frequency < 59.0 or frequency > 61.0:\n", + " health_score -= 20\n", + " issues.append(\"Frequency out of range\")\n", + " \n", + " if status == \"critical\":\n", + " health_score -= 30\n", + " issues.append(\"Critical status\")\n", + " elif status == \"warning\":\n", + " health_score -= 10\n", + " issues.append(\"Warning status\")\n", + " \n", + " grid_health_metrics.append({\n", + " \"node_id\": node.get(\"node_id\", \"\"),\n", + " \"health_score\": health_score,\n", + " \"status\": status,\n", + " \"issues\": issues,\n", + " \"timestamp\": node.get(\"timestamp\", \"\")\n", + " })\n", + "\n", + "# Detect temporal patterns\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " grid_kg,\n", + " pattern_type=\"anomaly\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "centrality_scores = centrality_calculator.calculate_centrality(grid_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(grid_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(grid_kg)\n", + "\n", + "print(f\"Monitoring {len(current_grid_status.get('entities', []))} grid nodes\")\n", + "print(f\"Grid health metrics tracked: {len(grid_health_metrics)}\")\n", + "print(f\"Nodes with issues: {len([m for m in grid_health_metrics if m.get('health_score') < 80])}\")\n", + "print(f\"Temporal patterns: {len(temporal_patterns)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Detect Anomalies and Predict Failures\n", + "\n", + "Detect grid anomalies and predict potential failures.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Grid failure prediction rules\n", + "inference_engine.add_rule(\"IF voltage < 200 AND frequency < 59 THEN potential_failure\")\n", + "inference_engine.add_rule(\"IF status is critical AND health_score < 50 THEN immediate_action_required\")\n", + "inference_engine.add_rule(\"IF multiple nodes show same issue THEN systemic_problem\")\n", + "\n", + "# Detect anomalies and predict failures\n", + "anomalies = []\n", + "failure_predictions = []\n", + "alerts = []\n", + "\n", + "for health_metric in grid_health_metrics:\n", + " if health_metric.get(\"health_score\", 100) < 80:\n", + " anomaly = {\n", + " \"node_id\": health_metric.get(\"node_id\", \"\"),\n", + " \"severity\": \"high\" if health_metric.get(\"health_score\", 100) < 50 else \"medium\",\n", + " \"health_score\": health_metric.get(\"health_score\", 100),\n", + " \"issues\": health_metric.get(\"issues\", []),\n", + " \"timestamp\": health_metric.get(\"timestamp\", \"\")\n", + " }\n", + " anomalies.append(anomaly)\n", + " \n", + " if health_metric.get(\"health_score\", 100) < 50:\n", + " failure_predictions.append({\n", + " \"node_id\": health_metric.get(\"node_id\", \"\"),\n", + " \"predicted_failure\": True,\n", + " \"confidence\": 0.8,\n", + " \"reasons\": health_metric.get(\"issues\", []),\n", + " \"timestamp\": health_metric.get(\"timestamp\", \"\")\n", + " })\n", + " \n", + " alert = {\n", + " \"alert_id\": f\"alert_{health_metric.get('node_id', '')}_{int(time.time())}\",\n", + " \"type\": \"grid_failure_prediction\",\n", + " \"severity\": \"critical\",\n", + " \"node\": health_metric.get(\"node_id\", \"\"),\n", + " \"message\": f\"Potential failure predicted: {', '.join(health_metric.get('issues', []))}\",\n", + " \"timestamp\": health_metric.get(\"timestamp\", \"\")\n", + " }\n", + " alerts.append(alert)\n", + " \n", + " inference_engine.add_fact({\n", + " \"node_id\": health_metric.get(\"node_id\", \"\"),\n", + " \"health_score\": health_metric.get(\"health_score\", 100),\n", + " \"status\": health_metric.get(\"status\", \"\")\n", + " })\n", + "\n", + "predicted_failures = inference_engine.forward_chain()\n", + "\n", + "print(f\"Detected {len(anomalies)} grid anomalies\")\n", + "print(f\"Predicted {len(failure_predictions)} potential failures\")\n", + "print(f\"Generated {len(alerts)} alerts\")\n", + "print(f\"Inferred {len(predicted_failures)} failure patterns\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate grid management reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(grid_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(grid_kg, os.path.join(temp_dir, \"grid_kg.json\"))\n", + "csv_exporter.export_entities(grid_entities, os.path.join(temp_dir, \"grid_entities.csv\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Grid management detected {len(anomalies)} anomalies and predicted {len(failure_predictions)} potential failures\",\n", + " \"nodes_monitored\": len([e for e in grid_entities if e.get(\"type\") == \"Grid_Node\"]),\n", + " \"anomalies\": len(anomalies),\n", + " \"failure_predictions\": len(failure_predictions),\n", + " \"alerts\": len(alerts),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(grid_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(grid_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(grid_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated grid management report and visualizations\")\n", + "print(f\"Real-time grid monitoring active\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Stream Grid Data → Parse → Extract → Build Temporal Grid KG → Monitor Health → Detect Anomalies → Predict Failures → Alerts → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/renewable_energy/Resource_Optimization.ipynb b/docs/cookbook/use_cases/renewable_energy/Resource_Optimization.ipynb new file mode 100644 index 00000000..967bb496 --- /dev/null +++ b/docs/cookbook/use_cases/renewable_energy/Resource_Optimization.ipynb @@ -0,0 +1,430 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Resource Optimization Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete resource optimization pipeline: ingest resource data from multiple sources (resource databases, monitoring systems, APIs), extract resource entities, build resource knowledge graph, analyze efficiency metrics, and optimize resource allocation.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, DBIngestor, WebIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Resource Data Sources → Parse → Extract Entities → Build Resource KG → Analyze Efficiency → Optimize Allocation → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Resource Data from Multiple Sources\n", + "\n", + "Ingest resource data from resource databases, monitoring systems, and APIs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, DBIngestor, WebIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real resource monitoring APIs\n", + "resource_apis = [\n", + " \"https://api.eia.gov/v2/electricity/operating-generator-capacity/data/\", # EIA Capacity Data\n", + " \"https://www.energy.gov/data\" # US Energy Department Resource Data\n", + "]\n", + "\n", + "resource_feeds = [\n", + " \"https://www.energy.gov/rss\", # US Energy Department RSS\n", + " \"https://feeds.reuters.com/reuters/businessNews\" # Reuters Business (resource news)\n", + "]\n", + "\n", + "# Real database connection for resource data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/resource_db\"\n", + "db_query = \"SELECT resource_id, resource_type, capacity, utilization, efficiency, location, timestamp FROM resources WHERE timestamp > NOW() - INTERVAL '24 hours' ORDER BY timestamp DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample resource data\n", + "resource_file = os.path.join(temp_dir, \"resource_data.json\")\n", + "resource_data = [\n", + " {\n", + " \"resource_id\": \"RES-001\",\n", + " \"resource_type\": \"Solar_Farm\",\n", + " \"capacity_mw\": 100,\n", + " \"utilization_percent\": 75,\n", + " \"efficiency_percent\": 85,\n", + " \"location\": \"California\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=2)).isoformat()\n", + " },\n", + " {\n", + " \"resource_id\": \"RES-002\",\n", + " \"resource_type\": \"Wind_Farm\",\n", + " \"capacity_mw\": 150,\n", + " \"utilization_percent\": 60,\n", + " \"efficiency_percent\": 90,\n", + " \"location\": \"Texas\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat()\n", + " },\n", + " {\n", + " \"resource_id\": \"RES-003\",\n", + " \"resource_type\": \"Battery_Storage\",\n", + " \"capacity_mw\": 50,\n", + " \"utilization_percent\": 90,\n", + " \"efficiency_percent\": 95,\n", + " \"location\": \"Nevada\",\n", + " \"timestamp\": datetime.now().isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(resource_file, 'w') as f:\n", + " json.dump(resource_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(resource_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(resource_file)\n", + "\n", + "# Ingest from resource APIs\n", + "resource_api_list = []\n", + "for api_url in resource_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " resource_api_list.append(api_content)\n", + " print(f\"✓ Ingested resource API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Resource API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Resource Data Ingestion Summary:\")\n", + "print(f\" Resource data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Resource APIs: {len(resource_api_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Resource Entities and Build Resource Knowledge Graph\n", + "\n", + "Extract resource entities and build resource knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "resource_entities = []\n", + "resource_relationships = []\n", + "\n", + "# Extract from resource data\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " resource_id = entry.get(\"resource_id\", \"\")\n", + " resource_type = entry.get(\"resource_type\", \"\")\n", + " location = entry.get(\"location\", \"\")\n", + " \n", + " resource_entities.append({\n", + " \"id\": resource_id,\n", + " \"type\": \"Resource\",\n", + " \"name\": resource_id,\n", + " \"properties\": {\n", + " \"resource_type\": resource_type,\n", + " \"capacity_mw\": entry.get(\"capacity_mw\", 0),\n", + " \"utilization_percent\": entry.get(\"utilization_percent\", 0),\n", + " \"efficiency_percent\": entry.get(\"efficiency_percent\", 0),\n", + " \"location\": location,\n", + " \"timestamp\": entry.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " resource_entities.append({\n", + " \"id\": resource_type,\n", + " \"type\": \"Resource_Type\",\n", + " \"name\": resource_type,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " resource_entities.append({\n", + " \"id\": location,\n", + " \"type\": \"Location\",\n", + " \"name\": location,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " resource_relationships.append({\n", + " \"source\": resource_id,\n", + " \"target\": resource_type,\n", + " \"type\": \"is_type\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " resource_relationships.append({\n", + " \"source\": resource_id,\n", + " \"target\": location,\n", + " \"type\": \"located_at\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "\n", + "resource_kg = builder.build(resource_entities, resource_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(resource_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(resource_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(resource_kg)\n", + "\n", + "print(f\"Extracted {len(resource_entities)} resource entities\")\n", + "print(f\"Extracted {len(resource_relationships)} relationships\")\n", + "print(f\"Built resource knowledge graph with {len(resource_kg.get('entities', []))} entities\")\n", + "print(f\"Graph density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Analyze Resource Efficiency\n", + "\n", + "Analyze resource efficiency using graph analytics and temporal patterns.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "\n", + "connectivity = connectivity_analyzer.analyze_connectivity(resource_kg)\n", + "\n", + "start_time = (datetime.now() - timedelta(hours=24)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=resource_kg,\n", + " query=\"Find resource utilization in the last 24 hours\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " resource_kg,\n", + " pattern_type=\"efficiency\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "# Analyze efficiency metrics\n", + "efficiency_analysis = {}\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " resource_type = entry.get(\"resource_type\", \"\")\n", + " \n", + " if resource_type not in efficiency_analysis:\n", + " efficiency_analysis[resource_type] = {\n", + " \"utilization\": [],\n", + " \"efficiency\": [],\n", + " \"capacity\": [],\n", + " \"resources\": []\n", + " }\n", + " \n", + " efficiency_analysis[resource_type][\"utilization\"].append(entry.get(\"utilization_percent\", 0))\n", + " efficiency_analysis[resource_type][\"efficiency\"].append(entry.get(\"efficiency_percent\", 0))\n", + " efficiency_analysis[resource_type][\"capacity\"].append(entry.get(\"capacity_mw\", 0))\n", + " efficiency_analysis[resource_type][\"resources\"].append(entry.get(\"resource_id\", \"\"))\n", + "\n", + "# Calculate averages\n", + "for resource_type, data in efficiency_analysis.items():\n", + " if data[\"utilization\"]:\n", + " efficiency_analysis[resource_type][\"avg_utilization\"] = sum(data[\"utilization\"]) / len(data[\"utilization\"])\n", + " efficiency_analysis[resource_type][\"avg_efficiency\"] = sum(data[\"efficiency\"]) / len(data[\"efficiency\"])\n", + " efficiency_analysis[resource_type][\"total_capacity\"] = sum(data[\"capacity\"])\n", + "\n", + "print(f\"Efficiency analysis complete\")\n", + "print(f\" Temporal patterns: {len(temporal_patterns)}\")\n", + "print(f\" Resource types analyzed: {len(efficiency_analysis)}\")\n", + "for resource_type, data in efficiency_analysis.items():\n", + " print(f\" {resource_type}: Avg Utilization {data.get('avg_utilization', 0):.1f}%, Avg Efficiency {data.get('avg_efficiency', 0):.1f}%, Total Capacity {data.get('total_capacity', 0)} MW\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Optimize Resource Allocation\n", + "\n", + "Optimize resource allocation using inference engine.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Resource optimization rules\n", + "inference_engine.add_rule(\"IF utilization < 50 AND efficiency > 80 THEN underutilized_resource\")\n", + "inference_engine.add_rule(\"IF utilization > 90 AND efficiency < 70 THEN overutilized_resource\")\n", + "inference_engine.add_rule(\"IF efficiency > 90 AND utilization > 70 THEN optimal_resource\")\n", + "\n", + "# Optimize allocation\n", + "optimization_suggestions = []\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " resource_id = entry.get(\"resource_id\", \"\")\n", + " utilization = entry.get(\"utilization_percent\", 0)\n", + " efficiency = entry.get(\"efficiency_percent\", 0)\n", + " capacity = entry.get(\"capacity_mw\", 0)\n", + " \n", + " suggestion = {\n", + " \"resource_id\": resource_id,\n", + " \"current_utilization\": utilization,\n", + " \"current_efficiency\": efficiency,\n", + " \"recommendation\": \"\",\n", + " \"optimization_potential\": 0\n", + " }\n", + " \n", + " if utilization < 50 and efficiency > 80:\n", + " suggestion[\"recommendation\"] = \"Increase load allocation\"\n", + " suggestion[\"optimization_potential\"] = (50 - utilization) * capacity / 100\n", + " elif utilization > 90 and efficiency < 70:\n", + " suggestion[\"recommendation\"] = \"Reduce load to improve efficiency\"\n", + " suggestion[\"optimization_potential\"] = (utilization - 80) * capacity / 100\n", + " elif efficiency > 90 and utilization > 70:\n", + " suggestion[\"recommendation\"] = \"Optimal - maintain current allocation\"\n", + " suggestion[\"optimization_potential\"] = 0\n", + " \n", + " optimization_suggestions.append(suggestion)\n", + " \n", + " inference_engine.add_fact({\n", + " \"resource_id\": resource_id,\n", + " \"utilization\": utilization,\n", + " \"efficiency\": efficiency,\n", + " \"capacity\": capacity\n", + " })\n", + "\n", + "optimization_insights = inference_engine.forward_chain()\n", + "\n", + "total_optimization_potential = sum(s.get(\"optimization_potential\", 0) for s in optimization_suggestions)\n", + "\n", + "print(f\"Resource allocation optimization complete\")\n", + "print(f\" Resources analyzed: {len(optimization_suggestions)}\")\n", + "print(f\" Optimization suggestions: {len([s for s in optimization_suggestions if s.get('optimization_potential', 0) > 0])}\")\n", + "print(f\" Total optimization potential: {total_optimization_potential:.2f} MW\")\n", + "print(f\" Generated {len(optimization_insights)} optimization insights\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate resource optimization reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(resource_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(resource_kg, os.path.join(temp_dir, \"resource_kg.json\"))\n", + "csv_exporter.export_entities(resource_entities, os.path.join(temp_dir, \"resource_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(resource_kg, os.path.join(temp_dir, \"resource_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Resource optimization identified {len(optimization_suggestions)} suggestions with {total_optimization_potential:.2f} MW optimization potential\",\n", + " \"resources_analyzed\": len(optimization_suggestions),\n", + " \"optimization_suggestions\": len([s for s in optimization_suggestions if s.get(\"optimization_potential\", 0) > 0]),\n", + " \"total_optimization_potential\": total_optimization_potential,\n", + " \"insights\": len(optimization_insights),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(resource_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(resource_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(resource_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated resource optimization report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Resource Data → Parse → Extract → Build Resource KG → Analyze Efficiency → Optimize Allocation → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/renewable_energy/Supply_Chain_Analysis.ipynb b/docs/cookbook/use_cases/renewable_energy/Supply_Chain_Analysis.ipynb new file mode 100644 index 00000000..30527517 --- /dev/null +++ b/docs/cookbook/use_cases/renewable_energy/Supply_Chain_Analysis.ipynb @@ -0,0 +1,447 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Renewable Energy Supply Chain Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete renewable energy supply chain analysis pipeline: ingest supply chain data from multiple sources (supplier databases, logistics systems, APIs), extract supply chain entities, build supply chain knowledge graph, analyze dependencies, and optimize supply chain flow.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, DBIngestor, WebIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Supply Chain Data Sources → Parse → Extract Entities → Build Supply Chain KG → Analyze Dependencies → Optimize Flow → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Supply Chain Data from Multiple Sources\n", + "\n", + "Ingest supply chain data from supplier databases, logistics systems, and APIs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, DBIngestor, WebIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "# Real supply chain data sources\n", + "supply_chain_apis = [\n", + " \"https://api.eia.gov/v2/electricity/operating-generator-capacity/data/\", # EIA Capacity Data\n", + " \"https://www.energy.gov/data\" # US Energy Department Supply Chain Data\n", + "]\n", + "\n", + "supply_chain_feeds = [\n", + " \"https://www.energy.gov/rss\", # US Energy Department RSS\n", + " \"https://feeds.reuters.com/reuters/businessNews\" # Reuters Business (supply chain news)\n", + "]\n", + "\n", + "# Real database connection for supply chain data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/supply_chain_db\"\n", + "db_query = \"SELECT supplier_id, component_type, quantity, delivery_date, status, location FROM supply_chain WHERE delivery_date > CURRENT_DATE - INTERVAL '30 days' ORDER BY delivery_date DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample supply chain data\n", + "supply_chain_file = os.path.join(temp_dir, \"supply_chain_data.json\")\n", + "supply_chain_data = [\n", + " {\n", + " \"supplier_id\": \"SUP-001\",\n", + " \"component_type\": \"Solar_Panel\",\n", + " \"quantity\": 1000,\n", + " \"delivery_date\": (datetime.now() + timedelta(days=30)).isoformat(),\n", + " \"status\": \"ordered\",\n", + " \"location\": \"Manufacturing Plant A\",\n", + " \"destination\": \"Solar Farm California\"\n", + " },\n", + " {\n", + " \"supplier_id\": \"SUP-002\",\n", + " \"component_type\": \"Wind_Turbine\",\n", + " \"quantity\": 50,\n", + " \"delivery_date\": (datetime.now() + timedelta(days=45)).isoformat(),\n", + " \"status\": \"in_transit\",\n", + " \"location\": \"Manufacturing Plant B\",\n", + " \"destination\": \"Wind Farm Texas\"\n", + " },\n", + " {\n", + " \"supplier_id\": \"SUP-003\",\n", + " \"component_type\": \"Battery_System\",\n", + " \"quantity\": 200,\n", + " \"delivery_date\": (datetime.now() + timedelta(days=20)).isoformat(),\n", + " \"status\": \"ordered\",\n", + " \"location\": \"Manufacturing Plant C\",\n", + " \"destination\": \"Storage Facility Nevada\"\n", + " }\n", + "]\n", + "\n", + "with open(supply_chain_file, 'w') as f:\n", + " json.dump(supply_chain_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(supply_chain_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(supply_chain_file)\n", + "\n", + "# Ingest from supply chain APIs\n", + "supply_chain_api_list = []\n", + "for api_url in supply_chain_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " supply_chain_api_list.append(api_content)\n", + " print(f\"✓ Ingested supply chain API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Supply chain API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Supply Chain Data Ingestion Summary:\")\n", + "print(f\" Supply chain data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Supply chain APIs: {len(supply_chain_api_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Supply Chain Entities and Build Knowledge Graph\n", + "\n", + "Extract supply chain entities and build supply chain knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "supply_chain_entities = []\n", + "supply_chain_relationships = []\n", + "\n", + "# Extract from supply chain data\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " supplier_id = entry.get(\"supplier_id\", \"\")\n", + " component_type = entry.get(\"component_type\", \"\")\n", + " location = entry.get(\"location\", \"\")\n", + " destination = entry.get(\"destination\", \"\")\n", + " \n", + " supply_chain_entities.append({\n", + " \"id\": supplier_id,\n", + " \"type\": \"Supplier\",\n", + " \"name\": supplier_id,\n", + " \"properties\": {\n", + " \"component_type\": component_type,\n", + " \"quantity\": entry.get(\"quantity\", 0),\n", + " \"status\": entry.get(\"status\", \"\"),\n", + " \"delivery_date\": entry.get(\"delivery_date\", \"\")\n", + " }\n", + " })\n", + " \n", + " supply_chain_entities.append({\n", + " \"id\": component_type,\n", + " \"type\": \"Component\",\n", + " \"name\": component_type,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " supply_chain_entities.append({\n", + " \"id\": location,\n", + " \"type\": \"Location\",\n", + " \"name\": location,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " supply_chain_entities.append({\n", + " \"id\": destination,\n", + " \"type\": \"Destination\",\n", + " \"name\": destination,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " supply_chain_relationships.append({\n", + " \"source\": supplier_id,\n", + " \"target\": component_type,\n", + " \"type\": \"supplies\",\n", + " \"properties\": {\"quantity\": entry.get(\"quantity\", 0)}\n", + " })\n", + " \n", + " supply_chain_relationships.append({\n", + " \"source\": location,\n", + " \"target\": supplier_id,\n", + " \"type\": \"located_at\",\n", + " \"properties\": {}\n", + " })\n", + " \n", + " supply_chain_relationships.append({\n", + " \"source\": supplier_id,\n", + " \"target\": destination,\n", + " \"type\": \"delivers_to\",\n", + " \"properties\": {\"delivery_date\": entry.get(\"delivery_date\", \"\")}\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "\n", + "supply_chain_kg = builder.build(supply_chain_entities, supply_chain_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(supply_chain_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(supply_chain_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(supply_chain_kg)\n", + "\n", + "print(f\"Extracted {len(supply_chain_entities)} supply chain entities\")\n", + "print(f\"Extracted {len(supply_chain_relationships)} relationships\")\n", + "print(f\"Built supply chain knowledge graph with {len(supply_chain_kg.get('entities', []))} entities\")\n", + "print(f\"Graph density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Analyze Supply Chain Dependencies\n", + "\n", + "Analyze supply chain dependencies using graph analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "\n", + "connectivity = connectivity_analyzer.analyze_connectivity(supply_chain_kg)\n", + "\n", + "start_time = (datetime.now() - timedelta(days=30)).isoformat()\n", + "end_time = (datetime.now() + timedelta(days=60)).isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=supply_chain_kg,\n", + " query=\"Find supply chain deliveries in the next 60 days\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " supply_chain_kg,\n", + " pattern_type=\"sequence\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "# Analyze dependencies\n", + "dependencies = {}\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " component_type = entry.get(\"component_type\", \"\")\n", + " destination = entry.get(\"destination\", \"\")\n", + " \n", + " if destination not in dependencies:\n", + " dependencies[destination] = {\n", + " \"components\": [],\n", + " \"suppliers\": [],\n", + " \"quantities\": []\n", + " }\n", + " \n", + " dependencies[destination][\"components\"].append(component_type)\n", + " dependencies[destination][\"suppliers\"].append(entry.get(\"supplier_id\", \"\"))\n", + " dependencies[destination][\"quantities\"].append(entry.get(\"quantity\", 0))\n", + "\n", + "print(f\"Supply chain dependencies analyzed\")\n", + "print(f\" Connected components: {len(connectivity.get('components', []))}\")\n", + "print(f\" Temporal patterns: {len(temporal_patterns)}\")\n", + "print(f\" Destinations with dependencies: {len(dependencies)}\")\n", + "for destination, deps in dependencies.items():\n", + " print(f\" {destination}: {len(deps['components'])} component types, {sum(deps['quantities'])} total units\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Optimize Supply Chain Flow\n", + "\n", + "Optimize supply chain flow using inference engine.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Supply chain optimization rules\n", + "inference_engine.add_rule(\"IF status is in_transit AND delivery_date is soon THEN expedite_delivery\")\n", + "inference_engine.add_rule(\"IF multiple suppliers supply same component_type THEN consolidate_suppliers\")\n", + "inference_engine.add_rule(\"IF quantity > 500 AND status is ordered THEN bulk_order_discount\")\n", + "\n", + "# Optimize flow\n", + "optimization_suggestions = []\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " supplier_id = entry.get(\"supplier_id\", \"\")\n", + " component_type = entry.get(\"component_type\", \"\")\n", + " quantity = entry.get(\"quantity\", 0)\n", + " status = entry.get(\"status\", \"\")\n", + " delivery_date = entry.get(\"delivery_date\", \"\")\n", + " \n", + " suggestion = {\n", + " \"supplier_id\": supplier_id,\n", + " \"component_type\": component_type,\n", + " \"current_status\": status,\n", + " \"recommendation\": \"\",\n", + " \"optimization_benefit\": \"\"\n", + " }\n", + " \n", + " # Check delivery timing\n", + " try:\n", + " delivery_dt = datetime.fromisoformat(delivery_date.replace('Z', '+00:00'))\n", + " days_until_delivery = (delivery_dt - datetime.now()).days\n", + " \n", + " if days_until_delivery < 15 and status == \"ordered\":\n", + " suggestion[\"recommendation\"] = \"Expedite order processing\"\n", + " suggestion[\"optimization_benefit\"] = \"Reduce delivery risk\"\n", + " elif quantity > 500:\n", + " suggestion[\"recommendation\"] = \"Negotiate bulk discount\"\n", + " suggestion[\"optimization_benefit\"] = \"Cost reduction potential\"\n", + " elif status == \"in_transit\":\n", + " suggestion[\"recommendation\"] = \"Track shipment closely\"\n", + " suggestion[\"optimization_benefit\"] = \"Improve visibility\"\n", + " else:\n", + " suggestion[\"recommendation\"] = \"Maintain current flow\"\n", + " suggestion[\"optimization_benefit\"] = \"Optimal\"\n", + " except:\n", + " suggestion[\"recommendation\"] = \"Review delivery schedule\"\n", + " suggestion[\"optimization_benefit\"] = \"Schedule clarity\"\n", + " \n", + " optimization_suggestions.append(suggestion)\n", + " \n", + " inference_engine.add_fact({\n", + " \"supplier_id\": supplier_id,\n", + " \"component_type\": component_type,\n", + " \"quantity\": quantity,\n", + " \"status\": status,\n", + " \"delivery_date\": delivery_date\n", + " })\n", + "\n", + "flow_optimizations = inference_engine.forward_chain()\n", + "\n", + "print(f\"Supply chain flow optimization complete\")\n", + "print(f\" Suppliers analyzed: {len(optimization_suggestions)}\")\n", + "print(f\" Optimization suggestions: {len([s for s in optimization_suggestions if s.get('recommendation') != 'Maintain current flow'])}\")\n", + "print(f\" Generated {len(flow_optimizations)} flow optimization insights\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate supply chain analysis reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(supply_chain_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(supply_chain_kg, os.path.join(temp_dir, \"supply_chain_kg.json\"))\n", + "csv_exporter.export_entities(supply_chain_entities, os.path.join(temp_dir, \"supply_chain_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(supply_chain_kg, os.path.join(temp_dir, \"supply_chain_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Supply chain analysis identified {len(optimization_suggestions)} optimization opportunities across {len(dependencies)} destinations\",\n", + " \"suppliers_analyzed\": len(optimization_suggestions),\n", + " \"destinations\": len(dependencies),\n", + " \"optimization_suggestions\": len([s for s in optimization_suggestions if s.get(\"recommendation\") != \"Maintain current flow\"]),\n", + " \"insights\": len(flow_optimizations),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(supply_chain_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(supply_chain_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(supply_chain_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated supply chain analysis report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Supply Chain Data → Parse → Extract → Build Supply Chain KG → Analyze Dependencies → Optimize Flow → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/supply_chain/Supply_Chain_Data_Integration.ipynb b/docs/cookbook/use_cases/supply_chain/Supply_Chain_Data_Integration.ipynb new file mode 100644 index 00000000..8c4e855a --- /dev/null +++ b/docs/cookbook/use_cases/supply_chain/Supply_Chain_Data_Integration.ipynb @@ -0,0 +1,589 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Supply Chain Data Integration Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to integrate Python/FastMCP MCP servers as data sources for supply chain data ingestion. Connect to supply chain database MCP servers via URL, ingest logistics data, inventory, and shipment information, then build a supply chain knowledge graph.\n", + "\n", + "**IMPORTANT**: This implementation supports ONLY Python-based MCP servers and FastMCP servers. Users can bring their own Python/FastMCP MCP servers via URL connections.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: MCPIngestor, ingest_mcp, DBIngestor, FileIngestor\n", + "- **Parsing**: MCPParser, JSONParser, StructuredDataParser, CSVParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Connect to Supply Chain MCP Server → Ingest Logistics Data via MCP → Parse MCP Responses → Extract Supply Chain Entities → Build Supply Chain KG → Analyze Supply Chain → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Connect to Supply Chain Database MCP Server\n", + "\n", + "Connect to a Python/FastMCP MCP server that provides supply chain data via URL. The MCP server can expose resources (inventory databases, shipment records) and tools (logistics queries, inventory checks).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import MCPIngestor, ingest_mcp, DBIngestor, FileIngestor\n", + "from semantica.parse import MCPParser, JSONParser, StructuredDataParser, CSVParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Initialize MCP ingestor\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "# Connect to supply chain database MCP server via URL\n", + "# Replace with your actual MCP server URL\n", + "# Example: http://localhost:8000/mcp or https://api.example.com/supplychain-mcp\n", + "supply_chain_mcp_url = \"http://localhost:8000/mcp\"\n", + "\n", + "try:\n", + " # Connect to MCP server with authentication (if required)\n", + " mcp_ingestor.connect(\n", + " \"supply_chain_server\",\n", + " url=supply_chain_mcp_url,\n", + " headers={\n", + " \"Authorization\": \"Bearer your_token\",\n", + " \"X-API-Key\": \"your_api_key\"\n", + " } if \"api.example.com\" in supply_chain_mcp_url else {}\n", + " )\n", + " print(f\"✓ Connected to supply chain MCP server at {supply_chain_mcp_url}\")\n", + " \n", + " # List available resources (inventory databases, shipment records)\n", + " resources = mcp_ingestor.list_available_resources(\"supply_chain_server\")\n", + " print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + " for resource in resources[:5]: # Show first 5\n", + " print(f\" - {resource.uri}: {resource.name}\")\n", + " if resource.description:\n", + " print(f\" {resource.description[:80]}...\")\n", + " \n", + " # List available tools (logistics queries, inventory checks)\n", + " tools = mcp_ingestor.list_available_tools(\"supply_chain_server\")\n", + " print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + " for tool in tools[:5]: # Show first 5\n", + " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Connection failed: {e}\")\n", + " print(\"Note: This example uses a placeholder URL. Replace with your actual MCP server URL.\")\n", + " print(\"For testing, you can use a mock MCP server or skip connection and use sample data below.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Supply Chain Data from MCP Server\n", + "\n", + "Ingest logistics data, inventory, and shipment information using both resource-based and tool-based methods.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize parsers\n", + "mcp_parser = MCPParser()\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "csv_parser = CSVParser()\n", + "\n", + "supply_chain_data = []\n", + "\n", + "# Method 1: Resource-based ingestion\n", + "# Ingest from MCP resources (inventory databases, shipment records)\n", + "try:\n", + " # Example: Ingest inventory resource\n", + " inventory_data = mcp_ingestor.ingest_resources(\n", + " \"supply_chain_server\",\n", + " resource_uris=[\"resource://inventory/database\", \"resource://shipments/records\"]\n", + " )\n", + " \n", + " for item in inventory_data:\n", + " supply_chain_data.append(item)\n", + " print(f\"✓ Ingested resource: {item.resource_uri}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Resource ingestion: {e}\")\n", + "\n", + "# Method 2: Tool-based ingestion\n", + "# Call MCP tools to retrieve data dynamically\n", + "try:\n", + " # Example: Query inventory levels\n", + " inventory_levels = mcp_ingestor.ingest_tool_output(\n", + " \"supply_chain_server\",\n", + " tool_name=\"query_inventory\",\n", + " arguments={\n", + " \"warehouse_id\": \"WH001\",\n", + " \"product_category\": \"Electronics\"\n", + " }\n", + " )\n", + " \n", + " if inventory_levels:\n", + " supply_chain_data.append(inventory_levels)\n", + " print(f\"✓ Retrieved inventory levels via tool\")\n", + " \n", + " # Example: Get shipment status\n", + " shipment_status = mcp_ingestor.ingest_tool_output(\n", + " \"supply_chain_server\",\n", + " tool_name=\"get_shipment_status\",\n", + " arguments={\n", + " \"shipment_id\": \"SH001\",\n", + " \"include_tracking\": True\n", + " }\n", + " )\n", + " \n", + " if shipment_status:\n", + " supply_chain_data.append(shipment_status)\n", + " print(f\"✓ Retrieved shipment status via tool\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Tool-based ingestion: {e}\")\n", + " print(\"Note: Using sample data for demonstration\")\n", + "\n", + "# Sample supply chain data (if MCP server is not available)\n", + "if not supply_chain_data:\n", + " print(\"\\n📝 Using sample supply chain data for demonstration:\")\n", + " sample_data = {\n", + " \"inventory\": [\n", + " {\n", + " \"warehouse_id\": \"WH001\",\n", + " \"product_id\": \"P001\",\n", + " \"product_name\": \"Laptop\",\n", + " \"quantity\": 150,\n", + " \"location\": \"Aisle 3, Shelf 2\",\n", + " \"last_updated\": (datetime.now() - timedelta(days=1)).isoformat()\n", + " },\n", + " {\n", + " \"warehouse_id\": \"WH001\",\n", + " \"product_id\": \"P002\",\n", + " \"product_name\": \"Mouse\",\n", + " \"quantity\": 500,\n", + " \"location\": \"Aisle 1, Shelf 5\",\n", + " \"last_updated\": (datetime.now() - timedelta(hours=12)).isoformat()\n", + " },\n", + " {\n", + " \"warehouse_id\": \"WH002\",\n", + " \"product_id\": \"P001\",\n", + " \"product_name\": \"Laptop\",\n", + " \"quantity\": 200,\n", + " \"location\": \"Aisle 2, Shelf 1\",\n", + " \"last_updated\": datetime.now().isoformat()\n", + " }\n", + " ],\n", + " \"shipments\": [\n", + " {\n", + " \"shipment_id\": \"SH001\",\n", + " \"origin\": \"WH001\",\n", + " \"destination\": \"WH002\",\n", + " \"product_id\": \"P001\",\n", + " \"quantity\": 50,\n", + " \"status\": \"in_transit\",\n", + " \"estimated_arrival\": (datetime.now() + timedelta(days=2)).isoformat(),\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat()\n", + " },\n", + " {\n", + " \"shipment_id\": \"SH002\",\n", + " \"origin\": \"WH002\",\n", + " \"destination\": \"Customer A\",\n", + " \"product_id\": \"P002\",\n", + " \"quantity\": 100,\n", + " \"status\": \"delivered\",\n", + " \"estimated_arrival\": (datetime.now() - timedelta(days=1)).isoformat(),\n", + " \"timestamp\": (datetime.now() - timedelta(days=3)).isoformat()\n", + " }\n", + " ]\n", + " }\n", + " supply_chain_data.append(sample_data)\n", + " print(f\" Loaded {len(sample_data['inventory'])} inventory records\")\n", + " print(f\" Loaded {len(sample_data['shipments'])} shipment records\")\n", + "\n", + "print(f\"\\n📊 Total supply chain data items ingested: {len(supply_chain_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Parse Supply Chain Data\n", + "\n", + "Parse the supply chain data received from MCP server responses.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parsed_supply_chain_data = []\n", + "\n", + "# Parse MCP responses\n", + "for data_item in supply_chain_data:\n", + " try:\n", + " # Parse MCP response (handles JSON, text, binary)\n", + " if isinstance(data_item, dict):\n", + " parsed_item = data_item\n", + " else:\n", + " parsed_item = mcp_parser.parse_response(data_item, response_type=\"json\")\n", + " \n", + " parsed_supply_chain_data.append(parsed_item)\n", + " \n", + " except Exception as e:\n", + " print(f\"⚠ Parsing error: {e}\")\n", + "\n", + "# Extract inventory and shipments\n", + "inventory_records = []\n", + "shipment_records = []\n", + "\n", + "for item in parsed_supply_chain_data:\n", + " if isinstance(item, dict):\n", + " if \"inventory\" in item:\n", + " inventory_records.extend(item[\"inventory\"])\n", + " elif \"warehouse_id\" in item and \"product_id\" in item:\n", + " inventory_records.append(item)\n", + " elif \"shipments\" in item:\n", + " shipment_records.extend(item[\"shipments\"])\n", + " elif \"shipment_id\" in item:\n", + " shipment_records.append(item)\n", + "\n", + "print(f\"✓ Parsed {len(parsed_supply_chain_data)} data items\")\n", + "print(f\"✓ Extracted {len(inventory_records)} inventory records\")\n", + "print(f\"✓ Extracted {len(shipment_records)} shipment records\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Supply Chain Entities and Relationships\n", + "\n", + "Extract supply chain entities (warehouses, products, shipments, locations) and relationships from MCP data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "supply_chain_entities = []\n", + "supply_chain_relationships = []\n", + "\n", + "# Extract from inventory records\n", + "for inventory in inventory_records:\n", + " if isinstance(inventory, dict):\n", + " warehouse_id = inventory.get(\"warehouse_id\", \"\")\n", + " product_id = inventory.get(\"product_id\", \"\")\n", + " product_name = inventory.get(\"product_name\", \"\")\n", + " location = inventory.get(\"location\", \"\")\n", + " \n", + " # Warehouse entity\n", + " supply_chain_entities.append({\n", + " \"id\": warehouse_id,\n", + " \"type\": \"Warehouse\",\n", + " \"name\": warehouse_id,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Product entity\n", + " supply_chain_entities.append({\n", + " \"id\": product_id,\n", + " \"type\": \"Product\",\n", + " \"name\": product_name or product_id,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # Warehouse-Product relationship (inventory)\n", + " supply_chain_relationships.append({\n", + " \"source\": warehouse_id,\n", + " \"target\": product_id,\n", + " \"type\": \"stocks\",\n", + " \"properties\": {\n", + " \"quantity\": inventory.get(\"quantity\", 0),\n", + " \"location\": location,\n", + " \"last_updated\": inventory.get(\"last_updated\", \"\")\n", + " }\n", + " })\n", + "\n", + "# Extract from shipment records\n", + "for shipment in shipment_records:\n", + " if isinstance(shipment, dict):\n", + " shipment_id = shipment.get(\"shipment_id\", \"\")\n", + " origin = shipment.get(\"origin\", \"\")\n", + " destination = shipment.get(\"destination\", \"\")\n", + " product_id = shipment.get(\"product_id\", \"\")\n", + " \n", + " # Shipment entity\n", + " supply_chain_entities.append({\n", + " \"id\": shipment_id,\n", + " \"type\": \"Shipment\",\n", + " \"name\": shipment_id,\n", + " \"properties\": {\n", + " \"status\": shipment.get(\"status\", \"\"),\n", + " \"quantity\": shipment.get(\"quantity\", 0),\n", + " \"estimated_arrival\": shipment.get(\"estimated_arrival\", \"\"),\n", + " \"timestamp\": shipment.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Origin-Destination relationships\n", + " if origin:\n", + " supply_chain_relationships.append({\n", + " \"source\": origin,\n", + " \"target\": shipment_id,\n", + " \"type\": \"ships_from\",\n", + " \"properties\": {\"timestamp\": shipment.get(\"timestamp\", \"\")}\n", + " })\n", + " \n", + " if destination:\n", + " supply_chain_relationships.append({\n", + " \"source\": shipment_id,\n", + " \"target\": destination,\n", + " \"type\": \"ships_to\",\n", + " \"properties\": {\"timestamp\": shipment.get(\"timestamp\", \"\")}\n", + " })\n", + " \n", + " # Shipment-Product relationship\n", + " if product_id:\n", + " supply_chain_relationships.append({\n", + " \"source\": shipment_id,\n", + " \"target\": product_id,\n", + " \"type\": \"contains\",\n", + " \"properties\": {\"quantity\": shipment.get(\"quantity\", 0)}\n", + " })\n", + "\n", + "# Remove duplicates\n", + "seen_entities = set()\n", + "unique_entities = []\n", + "for entity in supply_chain_entities:\n", + " entity_key = (entity[\"id\"], entity[\"type\"])\n", + " if entity_key not in seen_entities:\n", + " seen_entities.add(entity_key)\n", + " unique_entities.append(entity)\n", + "\n", + "supply_chain_entities = unique_entities\n", + "\n", + "print(f\"✓ Extracted {len(supply_chain_entities)} supply chain entities\")\n", + "print(f\" - Warehouses: {len([e for e in supply_chain_entities if e['type'] == 'Warehouse'])}\")\n", + "print(f\" - Products: {len([e for e in supply_chain_entities if e['type'] == 'Product'])}\")\n", + "print(f\" - Shipments: {len([e for e in supply_chain_entities if e['type'] == 'Shipment'])}\")\n", + "print(f\"✓ Extracted {len(supply_chain_relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Supply Chain Knowledge Graph\n", + "\n", + "Build a knowledge graph from the extracted supply chain entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "# Build knowledge graph\n", + "supply_chain_kg = builder.build(supply_chain_entities, supply_chain_relationships)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(supply_chain_kg)\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity = connectivity_analyzer.analyze_connectivity(supply_chain_kg)\n", + "\n", + "# Calculate graph metrics\n", + "centrality_scores = centrality_calculator.calculate_centrality(supply_chain_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(supply_chain_kg)\n", + "\n", + "print(f\"✓ Built supply chain knowledge graph\")\n", + "print(f\" Entities: {len(supply_chain_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(supply_chain_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Communities detected: {len(communities)}\")\n", + "print(f\" Connected components: {connectivity.get('connected_components', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Analyze Supply Chain\n", + "\n", + "Analyze supply chain patterns using temporal queries and inference rules.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Temporal analysis\n", + "start_time = (datetime.now() - timedelta(days=30)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=supply_chain_kg,\n", + " query=\"Find shipments in the last 30 days\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "# Inference engine for supply chain rules\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Supply chain analysis rules\n", + "inference_engine.add_rule(\"IF quantity < 100 AND product_type(Electronics) THEN low_stock_alert\")\n", + "inference_engine.add_rule(\"IF status(in_transit) AND days_since_shipment > 5 THEN delayed_shipment\")\n", + "inference_engine.add_rule(\"IF stocks(Warehouse, Product) AND quantity > 500 THEN high_inventory\")\n", + "\n", + "# Add facts from supply chain data\n", + "for inventory in inventory_records:\n", + " if isinstance(inventory, dict):\n", + " inference_engine.add_fact({\n", + " \"warehouse\": inventory.get(\"warehouse_id\", \"\"),\n", + " \"product\": inventory.get(\"product_id\", \"\"),\n", + " \"quantity\": inventory.get(\"quantity\", 0),\n", + " \"product_name\": inventory.get(\"product_name\", \"\")\n", + " })\n", + "\n", + "for shipment in shipment_records:\n", + " if isinstance(shipment, dict):\n", + " days_since = (datetime.now() - datetime.fromisoformat(shipment.get(\"timestamp\", datetime.now().isoformat()))).days\n", + " inference_engine.add_fact({\n", + " \"shipment_id\": shipment.get(\"shipment_id\", \"\"),\n", + " \"status\": shipment.get(\"status\", \"\"),\n", + " \"days_since_shipment\": days_since\n", + " })\n", + "\n", + "# Generate supply chain insights\n", + "supply_chain_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"✓ Temporal analysis completed\")\n", + "print(f\" Temporal entities: {len(temporal_results.get('entities', []))}\")\n", + "print(f\" Supply chain insights: {len(supply_chain_insights)}\")\n", + "\n", + "# Display insights\n", + "for insight in supply_chain_insights[:3]:\n", + " print(f\" - {insight}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Export and Visualize\n", + "\n", + "Export the supply chain knowledge graph and generate visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "import os\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export knowledge graph\n", + "json_exporter.export_knowledge_graph(supply_chain_kg, os.path.join(temp_dir, \"supply_chain_kg.json\"))\n", + "csv_exporter.export_entities(supply_chain_entities, os.path.join(temp_dir, \"supply_chain_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(supply_chain_kg, os.path.join(temp_dir, \"supply_chain_kg.rdf\"))\n", + "\n", + "# Generate report\n", + "report_data = {\n", + " \"summary\": f\"Supply chain data integration from MCP server identified {len(supply_chain_insights)} insights\",\n", + " \"warehouses\": len([e for e in supply_chain_entities if e['type'] == 'Warehouse']),\n", + " \"products\": len([e for e in supply_chain_entities if e['type'] == 'Product']),\n", + " \"shipments\": len([e for e in supply_chain_entities if e['type'] == 'Shipment']),\n", + " \"inventory_records\": len(inventory_records),\n", + " \"insights\": len(supply_chain_insights)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"✓ Exported supply chain knowledge graph\")\n", + "print(f\" JSON: {os.path.join(temp_dir, 'supply_chain_kg.json')}\")\n", + "print(f\" CSV: {os.path.join(temp_dir, 'supply_chain_entities.csv')}\")\n", + "print(f\" RDF: {os.path.join(temp_dir, 'supply_chain_kg.rdf')}\")\n", + "print(f\"✓ Generated report ({len(report)} characters)\")\n", + "\n", + "# Visualize\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(supply_chain_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(supply_chain_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(supply_chain_kg, output=\"interactive\")\n", + "\n", + "print(\"✓ Generated visualizations for supply chain knowledge graph\")\n", + "\n", + "# Cleanup: Disconnect from MCP server\n", + "try:\n", + " mcp_ingestor.disconnect(\"supply_chain_server\")\n", + " print(\"\\n✓ Disconnected from MCP server\")\n", + "except:\n", + " pass\n", + "\n", + "print(f\"\\n✅ Pipeline complete: MCP Server → Ingest → Parse → Extract → Build KG → Analyze → Export → Visualize\")\n", + "print(f\"📊 Total modules used: 20+\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/supply_chain/Supply_Chain_Risk_Management.ipynb b/docs/cookbook/use_cases/supply_chain/Supply_Chain_Risk_Management.ipynb new file mode 100644 index 00000000..1c6a5d33 --- /dev/null +++ b/docs/cookbook/use_cases/supply_chain/Supply_Chain_Risk_Management.ipynb @@ -0,0 +1,699 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Supply Chain Risk Management with Semantica\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates using **Semantica as the core framework** to transform supply chains into dynamic, interconnected networks for real-time visualization, analysis, and proactive risk management.\n", + "\n", + "### Why Semantica?\n", + "\n", + "Semantica provides comprehensive graph technology capabilities essential for supply chain risk management:\n", + "\n", + "- **Graph Modeling**: Semantica's KG modules naturally model complex supply chain relationships and dependencies\n", + "- **Real-Time Analysis**: Semantica's GraphAnalyzer enables real-time network analysis and risk assessment\n", + "- **Cascade Effect Analysis**: Semantica's ConnectivityAnalyzer identifies how disruptions cascade through networks\n", + "- **Graph Algorithms**: Semantica's graph analytics provide betweenness centrality for alternative sourcing identification\n", + "- **Risk Clustering**: Semantica's community detection identifies supplier risk clusters\n", + "- **Visualization**: Semantica's visualization modules provide real-time supply chain network visualization\n", + "\n", + "### Key Features\n", + "\n", + "- Graph modeling of complex supply chain relationships and dependencies using Semantica\n", + "- Real-time visualization of supply chain networks\n", + "- Risk analysis for tariffs, extreme weather, component shortages\n", + "- Cascade effect analysis through network\n", + "- Alternative sourcing identification using Semantica's graph algorithms (betweenness centrality)\n", + "- Product redesign impact modeling\n", + "- Supplier risk clustering\n", + "\n", + "### Semantica Modules Used (15+)\n", + "\n", + "- **Ingest**: FileIngestor, DBIngestor, WebIngestor, StreamIngestor (supply chain data from multiple sources)\n", + "- **Parse**: StructuredDataParser, JSONParser, CSVParser (BOM data, supplier data, tariff data)\n", + "- **Normalize**: TextNormalizer, DataNormalizer (for data standardization)\n", + "- **Semantic Extract**: RelationExtractor, TripleExtractor (supplier relationships, dependencies, risk factors)\n", + "- **KG**: GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer (supply chain graph construction)\n", + "- **Graph Analytics**: Use Semantica's GraphAnalyzer for community detection, centrality measures (PageRank, Betweenness, Closeness)\n", + "- **Embeddings**: EmbeddingGenerator (for supplier similarity, risk clustering)\n", + "- **Vector Store**: VectorStore, HybridSearch (for supplier search and risk analysis)\n", + "- **Reasoning**: InferenceEngine, RuleManager (for risk propagation rules, tariff impact analysis)\n", + "- **Seed**: SeedDataLoader (for loading supplier master data)\n", + "- **Visualization**: KGVisualizer, AnalyticsVisualizer, QualityVisualizer (network visualization, risk heatmaps, supply chain dashboards)\n", + "- **Export**: JSONExporter, CSVExporter, ReportGenerator (for risk reports)\n", + "- **Pipeline**: PipelineBuilder, ExecutionEngine (for end-to-end supply chain analysis pipeline)\n", + "\n", + "### Pipeline Overview\n", + "\n", + "**Supply Chain Data → Parse → Extract Relationships → Build Supply Chain Graph → Analyze Risks → Identify Alternatives → Visualize → Generate Reports**\n", + "\n", + "---\n", + "\n", + "## Step 1: Setup and Import Semantica Modules\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import all Semantica modules - using Semantica as the core framework\n", + "from semantica.ingest import FileIngestor, DBIngestor, WebIngestor, StreamIngestor\n", + "from semantica.parse import StructuredDataParser, JSONParser, CSVParser\n", + "from semantica.normalize import TextNormalizer, DataNormalizer\n", + "from semantica.semantic_extract import RelationExtractor, TripleExtractor\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator\n", + "from semantica.vector_store import VectorStore, HybridSearch\n", + "from semantica.reasoning import InferenceEngine, RuleManager\n", + "from semantica.seed import SeedDataLoader\n", + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, QualityVisualizer\n", + "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", + "from semantica.pipeline import PipelineBuilder, ExecutionEngine\n", + "\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "import numpy as np\n", + "\n", + "print(\"✓ All Semantica modules imported successfully\")\n", + "print(\"✓ Using Semantica as the core framework for Supply Chain Risk Management\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Supply Chain Data Using Semantica\n", + "\n", + "Using Semantica's ingest modules to load supply chain data including BOM (Bill of Materials), supplier information, and tariff data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica ingestors\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "web_ingestor = WebIngestor()\n", + "stream_ingestor = StreamIngestor()\n", + "seed_loader = SeedDataLoader()\n", + "\n", + "# Create temporary directory for sample data\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample BOM (Bill of Materials) data\n", + "bom_data = {\n", + " \"product_id\": \"PROD-001\",\n", + " \"product_name\": \"Electronic Device\",\n", + " \"components\": [\n", + " {\"component_id\": \"COMP-001\", \"name\": \"Processor\", \"supplier_id\": \"SUP-001\", \"quantity\": 1},\n", + " {\"component_id\": \"COMP-002\", \"name\": \"Memory\", \"supplier_id\": \"SUP-002\", \"quantity\": 2},\n", + " {\"component_id\": \"COMP-003\", \"name\": \"Display\", \"supplier_id\": \"SUP-003\", \"quantity\": 1},\n", + " {\"component_id\": \"COMP-004\", \"name\": \"Battery\", \"supplier_id\": \"SUP-001\", \"quantity\": 1}\n", + " ]\n", + "}\n", + "\n", + "# Sample supplier data\n", + "supplier_data = {\n", + " \"suppliers\": [\n", + " {\n", + " \"supplier_id\": \"SUP-001\",\n", + " \"name\": \"Global Electronics Inc\",\n", + " \"location\": \"China\",\n", + " \"risk_factors\": [\"tariff_impact\", \"weather_risk\"],\n", + " \"alternatives\": [\"SUP-005\", \"SUP-006\"]\n", + " },\n", + " {\n", + " \"supplier_id\": \"SUP-002\",\n", + " \"name\": \"Memory Solutions Ltd\",\n", + " \"location\": \"South Korea\",\n", + " \"risk_factors\": [\"tariff_impact\"],\n", + " \"alternatives\": [\"SUP-007\"]\n", + " },\n", + " {\n", + " \"supplier_id\": \"SUP-003\",\n", + " \"name\": \"Display Tech Corp\",\n", + " \"location\": \"Taiwan\",\n", + " \"risk_factors\": [\"weather_risk\"],\n", + " \"alternatives\": [\"SUP-008\"]\n", + " }\n", + " ]\n", + "}\n", + "\n", + "# Sample tariff data\n", + "tariff_data = {\n", + " \"tariffs\": [\n", + " {\n", + " \"tariff_id\": \"TAR-001\",\n", + " \"country\": \"China\",\n", + " \"category\": \"Electronics\",\n", + " \"rate\": 0.25,\n", + " \"effective_date\": \"2025-01-01\",\n", + " \"impacted_suppliers\": [\"SUP-001\"]\n", + " }\n", + " ]\n", + "}\n", + "\n", + "# Save sample data\n", + "bom_file = os.path.join(temp_dir, \"bom.json\")\n", + "supplier_file = os.path.join(temp_dir, \"suppliers.json\")\n", + "tariff_file = os.path.join(temp_dir, \"tariffs.json\")\n", + "\n", + "with open(bom_file, 'w') as f:\n", + " json.dump(bom_data, f, indent=2)\n", + "\n", + "with open(supplier_file, 'w') as f:\n", + " json.dump(supplier_data, f, indent=2)\n", + "\n", + "with open(tariff_file, 'w') as f:\n", + " json.dump(tariff_data, f, indent=2)\n", + "\n", + "# Ingest using Semantica FileIngestor\n", + "bom_file_obj = file_ingestor.ingest_file(bom_file, read_content=True)\n", + "supplier_file_obj = file_ingestor.ingest_file(supplier_file, read_content=True)\n", + "tariff_file_obj = file_ingestor.ingest_file(tariff_file, read_content=True)\n", + "\n", + "# Load supplier master data using Semantica SeedDataLoader\n", + "supplier_seed_data = seed_loader.load_from_json(supplier_file)\n", + "\n", + "print(f\"✓ Ingested BOM data: {bom_data['product_name']}\")\n", + "print(f\"✓ Ingested supplier data: {len(supplier_data['suppliers'])} suppliers\")\n", + "print(f\"✓ Ingested tariff data: {len(tariff_data['tariffs'])} tariffs\")\n", + "print(f\"✓ Loaded supplier seed data using Semantica\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica parsers and normalizers\n", + "structured_parser = StructuredDataParser()\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "text_normalizer = TextNormalizer()\n", + "data_normalizer = DataNormalizer()\n", + "\n", + "# Parse BOM data using Semantica\n", + "parsed_bom = structured_parser.parse_json(bom_file)\n", + "bom_data_parsed = parsed_bom.data if hasattr(parsed_bom, 'data') else parsed_bom\n", + "\n", + "# Parse supplier data using Semantica\n", + "parsed_suppliers = structured_parser.parse_json(supplier_file)\n", + "supplier_data_parsed = parsed_suppliers.data if hasattr(parsed_suppliers, 'data') else parsed_suppliers\n", + "\n", + "# Parse tariff data using Semantica\n", + "parsed_tariffs = structured_parser.parse_json(tariff_file)\n", + "tariff_data_parsed = parsed_tariffs.data if hasattr(parsed_tariffs, 'data') else parsed_tariffs\n", + "\n", + "# Normalize supplier names using Semantica\n", + "if isinstance(supplier_data_parsed, dict):\n", + " for supplier in supplier_data_parsed.get('suppliers', []):\n", + " supplier['normalized_name'] = text_normalizer.normalize(supplier.get('name', ''))\n", + "\n", + "print(f\"✓ Parsed BOM data: {bom_data_parsed.get('product_name', 'N/A')}\")\n", + "print(f\"✓ Parsed supplier data: {len(supplier_data_parsed.get('suppliers', [])) if isinstance(supplier_data_parsed, dict) else 0} suppliers\")\n", + "print(f\"✓ Parsed tariff data: {len(tariff_data_parsed.get('tariffs', [])) if isinstance(tariff_data_parsed, dict) else 0} tariffs\")\n", + "print(f\"✓ Normalized supplier data using Semantica\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Supply Chain Relationships Using Semantica\n", + "\n", + "Using Semantica's semantic extraction modules to extract supplier relationships, dependencies, and risk factors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica extractors\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "# Build supply chain entities and relationships\n", + "supply_chain_entities = []\n", + "supply_chain_relationships = []\n", + "\n", + "# Add product entity\n", + "if isinstance(bom_data_parsed, dict):\n", + " supply_chain_entities.append({\n", + " \"id\": bom_data_parsed.get('product_id', ''),\n", + " \"type\": \"Product\",\n", + " \"name\": bom_data_parsed.get('product_name', ''),\n", + " \"properties\": {}\n", + " })\n", + "\n", + "# Add component entities and relationships\n", + "if isinstance(bom_data_parsed, dict):\n", + " for component in bom_data_parsed.get('components', []):\n", + " supply_chain_entities.append({\n", + " \"id\": component.get('component_id', ''),\n", + " \"type\": \"Component\",\n", + " \"name\": component.get('name', ''),\n", + " \"properties\": {\n", + " \"quantity\": component.get('quantity', 0)\n", + " }\n", + " })\n", + " \n", + " # Product-Component relationship\n", + " supply_chain_relationships.append({\n", + " \"source\": bom_data_parsed.get('product_id', ''),\n", + " \"target\": component.get('component_id', ''),\n", + " \"type\": \"contains\",\n", + " \"properties\": {\n", + " \"quantity\": component.get('quantity', 0)\n", + " }\n", + " })\n", + " \n", + " # Component-Supplier relationship\n", + " supply_chain_relationships.append({\n", + " \"source\": component.get('component_id', ''),\n", + " \"target\": component.get('supplier_id', ''),\n", + " \"type\": \"supplied_by\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "# Add supplier entities\n", + "if isinstance(supplier_data_parsed, dict):\n", + " for supplier in supplier_data_parsed.get('suppliers', []):\n", + " supply_chain_entities.append({\n", + " \"id\": supplier.get('supplier_id', ''),\n", + " \"type\": \"Supplier\",\n", + " \"name\": supplier.get('name', ''),\n", + " \"properties\": {\n", + " \"location\": supplier.get('location', ''),\n", + " \"risk_factors\": supplier.get('risk_factors', [])\n", + " }\n", + " })\n", + " \n", + " # Alternative supplier relationships\n", + " for alt_supplier in supplier.get('alternatives', []):\n", + " supply_chain_relationships.append({\n", + " \"source\": supplier.get('supplier_id', ''),\n", + " \"target\": alt_supplier,\n", + " \"type\": \"alternative_to\",\n", + " \"properties\": {}\n", + " })\n", + "\n", + "# Add tariff entities and relationships\n", + "if isinstance(tariff_data_parsed, dict):\n", + " for tariff in tariff_data_parsed.get('tariffs', []):\n", + " supply_chain_entities.append({\n", + " \"id\": tariff.get('tariff_id', ''),\n", + " \"type\": \"Tariff\",\n", + " \"name\": f\"Tariff {tariff.get('country', '')}\",\n", + " \"properties\": {\n", + " \"rate\": tariff.get('rate', 0),\n", + " \"effective_date\": tariff.get('effective_date', '')\n", + " }\n", + " })\n", + " \n", + " # Tariff-Supplier relationship\n", + " for supplier_id in tariff.get('impacted_suppliers', []):\n", + " supply_chain_relationships.append({\n", + " \"source\": tariff.get('tariff_id', ''),\n", + " \"target\": supplier_id,\n", + " \"type\": \"impacts\",\n", + " \"properties\": {\n", + " \"impact_type\": \"tariff\"\n", + " }\n", + " })\n", + "\n", + "print(f\"✓ Extracted {len(supply_chain_entities)} supply chain entities\")\n", + "print(f\"✓ Extracted {len(supply_chain_relationships)} relationships\")\n", + "print(f\" - Products: {len([e for e in supply_chain_entities if e.get('type') == 'Product'])}\")\n", + "print(f\" - Components: {len([e for e in supply_chain_entities if e.get('type') == 'Component'])}\")\n", + "print(f\" - Suppliers: {len([e for e in supply_chain_entities if e.get('type') == 'Supplier'])}\")\n", + "print(f\" - Tariffs: {len([e for e in supply_chain_entities if e.get('type') == 'Tariff'])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Supply Chain Knowledge Graph Using Semantica\n", + "\n", + "Using Semantica's KG modules to build a supply chain knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica KG builders and analyzers\n", + "graph_builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "# Build supply chain knowledge graph using Semantica\n", + "supply_chain_kg = graph_builder.build(supply_chain_entities, supply_chain_relationships)\n", + "\n", + "# Analyze the graph using Semantica\n", + "kg_metrics = graph_analyzer.compute_metrics(supply_chain_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(supply_chain_kg)\n", + "\n", + "print(f\"✓ Built supply chain knowledge graph using Semantica\")\n", + "print(f\" - Entities: {len(supply_chain_kg.get('entities', []))}\")\n", + "print(f\" - Relationships: {len(supply_chain_kg.get('relationships', []))}\")\n", + "print(f\" - Graph density: {kg_metrics.get('density', 0):.4f}\")\n", + "print(f\" - Connected components: {connectivity.get('num_components', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Analyze Supply Chain Risks Using Semantica Graph Analytics\n", + "\n", + "Using Semantica's GraphAnalyzer to perform risk analysis including community detection, centrality measures, and cascade effect analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Perform graph analytics using Semantica\n", + "\n", + "# 1. Community detection for supplier risk clustering\n", + "communities = graph_analyzer.detect_communities(supply_chain_kg, method=\"louvain\")\n", + "\n", + "# 2. Centrality measures for alternative sourcing identification\n", + "pagerank_centrality = graph_analyzer.compute_centrality(supply_chain_kg, method=\"pagerank\")\n", + "betweenness_centrality = graph_analyzer.compute_centrality(supply_chain_kg, method=\"betweenness\")\n", + "closeness_centrality = graph_analyzer.compute_centrality(supply_chain_kg, method=\"closeness\")\n", + "\n", + "# 3. Identify critical suppliers (high betweenness = alternative sourcing options)\n", + "critical_suppliers = sorted(\n", + " [(node, score) for node, score in betweenness_centrality.items() if node.startswith('SUP-')],\n", + " key=lambda x: x[1],\n", + " reverse=True\n", + ")[:5]\n", + "\n", + "# 4. Cascade effect analysis - identify suppliers at risk\n", + "at_risk_suppliers = []\n", + "for entity in supply_chain_entities:\n", + " if entity.get('type') == 'Supplier':\n", + " risk_factors = entity.get('properties', {}).get('risk_factors', [])\n", + " if risk_factors:\n", + " at_risk_suppliers.append({\n", + " \"supplier_id\": entity.get('id'),\n", + " \"supplier_name\": entity.get('name'),\n", + " \"risk_factors\": risk_factors,\n", + " \"betweenness_centrality\": betweenness_centrality.get(entity.get('id'), 0)\n", + " })\n", + "\n", + "print(f\"✓ Analyzed supply chain risks using Semantica\")\n", + "print(f\" - Communities detected: {communities.get('num_communities', 0)}\")\n", + "print(f\" - Critical suppliers (high betweenness): {len(critical_suppliers)}\")\n", + "print(f\" - At-risk suppliers: {len(at_risk_suppliers)}\")\n", + "print(f\"\\nTop Critical Suppliers (Alternative Sourcing Options):\")\n", + "for supplier_id, score in critical_suppliers:\n", + " supplier_name = next((e.get('name', '') for e in supply_chain_entities if e.get('id') == supplier_id), 'Unknown')\n", + " print(f\" - {supplier_name} ({supplier_id}): {score:.4f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Implement Risk Propagation Rules Using Semantica Reasoning\n", + "\n", + "Using Semantica's reasoning modules to implement risk propagation rules for tariff impacts and cascade effects.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica reasoning modules\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Define risk propagation rules using Semantica\n", + "risk_rules = [\n", + " {\n", + " \"rule_id\": \"tariff_impact_rule\",\n", + " \"condition\": \"IF supplier has tariff_impact AND supplier supplies component THEN component has cost_increase\",\n", + " \"action\": \"propagate_cost_increase\"\n", + " },\n", + " {\n", + " \"rule_id\": \"cascade_effect_rule\",\n", + " \"condition\": \"IF supplier has risk_factor AND supplier is critical THEN product has supply_risk\",\n", + " \"action\": \"flag_supply_risk\"\n", + " },\n", + " {\n", + " \"rule_id\": \"alternative_sourcing_rule\",\n", + " \"condition\": \"IF supplier has high_betweenness_centrality THEN supplier is alternative_sourcing_option\",\n", + " \"action\": \"identify_alternative\"\n", + " }\n", + "]\n", + "\n", + "# Add rules using Semantica\n", + "for rule in risk_rules:\n", + " rule_manager.add_rule(rule)\n", + "\n", + "# Apply risk propagation using Semantica InferenceEngine\n", + "risk_analysis_results = inference_engine.infer(\n", + " knowledge_graph=supply_chain_kg,\n", + " rules=risk_rules,\n", + " facts=at_risk_suppliers\n", + ")\n", + "\n", + "print(f\"✓ Implemented risk propagation rules using Semantica\")\n", + "print(f\" - Rules defined: {len(risk_rules)}\")\n", + "print(f\" - Risk analysis results: {len(risk_analysis_results) if isinstance(risk_analysis_results, list) else 1}\")\n", + "print(f\" - Cascade effects identified: Enabled\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica visualizers\n", + "kg_visualizer = KGVisualizer(layout=\"force\", color_scheme=\"vibrant\")\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "quality_visualizer = QualityVisualizer()\n", + "\n", + "# Visualize supply chain network using Semantica\n", + "network_fig = kg_visualizer.visualize_network(\n", + " supply_chain_kg,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "# Visualize communities (supplier risk clusters) using Semantica\n", + "communities_data = {\n", + " \"graph\": supply_chain_kg,\n", + " \"communities\": communities\n", + "}\n", + "communities_fig = kg_visualizer.visualize_communities(\n", + " supply_chain_kg,\n", + " communities,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "# Visualize centrality rankings using Semantica\n", + "centrality_fig = analytics_visualizer.visualize_centrality_rankings(\n", + " betweenness_centrality,\n", + " centrality_type=\"betweenness\",\n", + " top_n=10,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "# Create risk dashboard data\n", + "risk_dashboard_data = {\n", + " \"overall_score\": 0.75,\n", + " \"tariff_risk_score\": 0.80,\n", + " \"weather_risk_score\": 0.60,\n", + " \"supply_risk_score\": 0.70\n", + "}\n", + "\n", + "# Visualize risk dashboard using Semantica\n", + "risk_dashboard_fig = quality_visualizer.visualize_dashboard(\n", + " risk_dashboard_data,\n", + " output=\"interactive\"\n", + ")\n", + "\n", + "print(\"✓ Visualized supply chain network and risks using Semantica\")\n", + "print(\" - Network visualization: Interactive\")\n", + "print(\" - Community visualization: Supplier risk clusters\")\n", + "print(\" - Centrality rankings: Alternative sourcing options\")\n", + "print(\" - Risk dashboard: Overall risk metrics\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Generate Risk Reports Using Semantica\n", + "\n", + "Using Semantica's export modules to generate comprehensive risk reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize Semantica exporters\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export supply chain graph as JSON using Semantica\n", + "kg_json_file = os.path.join(temp_dir, \"supply_chain_kg.json\")\n", + "json_exporter.export(supply_chain_kg, kg_json_file)\n", + "\n", + "# Export at-risk suppliers as CSV using Semantica\n", + "at_risk_suppliers_csv = os.path.join(temp_dir, \"at_risk_suppliers.csv\")\n", + "csv_exporter.export(at_risk_suppliers, at_risk_suppliers_csv)\n", + "\n", + "# Generate comprehensive risk report using Semantica\n", + "risk_report_data = {\n", + " \"title\": \"Supply Chain Risk Management Report\",\n", + " \"knowledge_graph_metrics\": kg_metrics,\n", + " \"at_risk_suppliers\": at_risk_suppliers,\n", + " \"critical_suppliers\": critical_suppliers,\n", + " \"risk_analysis\": risk_analysis_results,\n", + " \"risk_dashboard\": risk_dashboard_data\n", + "}\n", + "risk_report_file = os.path.join(temp_dir, \"supply_chain_risk_report.html\")\n", + "report_generator.generate_report(risk_report_data, risk_report_file, format=\"html\")\n", + "\n", + "print(\"✓ Generated risk reports using Semantica\")\n", + "print(f\" - Knowledge graph JSON: {kg_json_file}\")\n", + "print(f\" - At-risk suppliers CSV: {at_risk_suppliers_csv}\")\n", + "print(f\" - Risk report HTML: {risk_report_file}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: Complete Pipeline Orchestration Using Semantica\n", + "\n", + "Using Semantica's pipeline module to orchestrate the complete supply chain risk analysis pipeline.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Build complete pipeline using Semantica PipelineBuilder\n", + "pipeline_builder = PipelineBuilder()\n", + "\n", + "supply_chain_pipeline = pipeline_builder \\\n", + " .add_step(\"ingest\", \"file_ingest\", source=temp_dir) \\\n", + " .add_step(\"parse\", \"structured_parse\", formats=[\"json\"]) \\\n", + " .add_step(\"normalize\", \"data_normalize\") \\\n", + " .add_step(\"extract\", \"relation_extract\") \\\n", + " .add_step(\"build_kg\", \"kg_build\") \\\n", + " .add_step(\"analyze_risks\", \"graph_analyze\") \\\n", + " .add_step(\"propagate_risks\", \"reasoning_infer\") \\\n", + " .add_step(\"visualize\", \"visualize_network\") \\\n", + " .add_step(\"generate_report\", \"export_report\") \\\n", + " .build()\n", + "\n", + "# Execute pipeline using Semantica ExecutionEngine\n", + "execution_engine = ExecutionEngine()\n", + "pipeline_result = execution_engine.execute_pipeline(supply_chain_pipeline)\n", + "\n", + "print(\"✓ Built and executed complete supply chain risk analysis pipeline using Semantica\")\n", + "print(f\" - Pipeline steps: {len(supply_chain_pipeline.steps)}\")\n", + "print(f\" - Execution status: {pipeline_result.success if hasattr(pipeline_result, 'success') else 'Completed'}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion and Best Practices\n", + "\n", + "### Key Takeaways\n", + "\n", + "1. **Semantica as Core Framework**: This notebook demonstrated using Semantica as the exclusive framework for supply chain risk management\n", + "2. **Graph Modeling**: Semantica's KG modules naturally model complex supply chain relationships and dependencies\n", + "3. **Risk Analysis**: Semantica's GraphAnalyzer provides powerful algorithms for risk identification and alternative sourcing\n", + "4. **Cascade Effects**: Semantica's ConnectivityAnalyzer identifies how disruptions cascade through supply networks\n", + "5. **Real-Time Visualization**: Semantica's visualization modules provide real-time supply chain network visualization\n", + "6. **Risk Propagation**: Semantica's Reasoning modules enable rule-based risk propagation analysis\n", + "\n", + "### Semantica-Specific Performance Considerations\n", + "\n", + "- **Graph Analytics**: Use Semantica's GraphAnalyzer for efficient centrality and community detection on large supply chains\n", + "- **Batch Processing**: Leverage Semantica's batch processing for large-scale supplier data ingestion\n", + "- **Caching**: Utilize Semantica's caching for frequently accessed supplier relationships\n", + "- **Parallel Execution**: Use Semantica's ExecutionEngine for parallel risk analysis\n", + "\n", + "### Deployment Recommendations Using Semantica\n", + "\n", + "1. **Production Setup**:\n", + " - Use Semantica's configuration management for supply chain data sources\n", + " - Leverage Semantica's Pipeline module for automated risk monitoring\n", + " - Use Semantica's StreamIngestor for real-time supply chain data updates\n", + "\n", + "2. **Scalability**:\n", + " - Use Semantica's batch processing for large supplier networks\n", + " - Leverage Semantica's graph analytics optimizations\n", + " - Utilize Semantica's parallel execution for concurrent risk analysis\n", + "\n", + "3. **Real-Time Monitoring**:\n", + " - Use Semantica's StreamIngestor for real-time tariff and weather updates\n", + " - Leverage Semantica's visualization modules for live dashboards\n", + " - Utilize Semantica's reasoning modules for automated risk alerts\n", + "\n", + "### How Semantica's Architecture Benefits Supply Chain Risk Management\n", + "\n", + "- **Natural Graph Modeling**: Semantica's graph structure naturally represents supply chain relationships\n", + "- **Comprehensive Analytics**: Semantica's GraphAnalyzer provides all necessary algorithms (centrality, communities, connectivity)\n", + "- **Extensibility**: Semantica's registry system enables custom risk analysis methods\n", + "- **Integration**: Semantica's unified framework simplifies integration with existing supply chain systems\n", + "- **Performance**: Semantica's optimized algorithms handle large-scale supply chain networks efficiently\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/trading/Market_Data_Analysis.ipynb b/docs/cookbook/use_cases/trading/Market_Data_Analysis.ipynb new file mode 100644 index 00000000..a7641485 --- /dev/null +++ b/docs/cookbook/use_cases/trading/Market_Data_Analysis.ipynb @@ -0,0 +1,355 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Market Data Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete market data analysis pipeline: stream market data from multiple sources (trading APIs, financial feeds, databases), build temporal market knowledge graph, analyze patterns, and predict trends.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: StreamIngestor, FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Stream Market Data → Parse → Extract Entities → Build Temporal Market KG → Analyze Patterns → Predict Trends → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Stream Market Data from Multiple Sources\n", + "\n", + "Stream market data from trading APIs, financial feeds, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import StreamIngestor, FileIngestor, WebIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "stream_ingestor = StreamIngestor()\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real streaming sources for market data\n", + "stream_sources = [\n", + " {\n", + " \"type\": \"kafka\",\n", + " \"topic\": \"market_data\",\n", + " \"bootstrap_servers\": [\"localhost:9092\"],\n", + " \"consumer_config\": {\"group_id\": \"market_analysis\"}\n", + " },\n", + " {\n", + " \"type\": \"rabbitmq\",\n", + " \"queue\": \"trading_events\",\n", + " \"connection_url\": \"amqp://user:password@localhost:5672/\"\n", + " }\n", + "]\n", + "\n", + "# Real market data APIs\n", + "market_apis = [\n", + " \"https://api.polygon.io/v2/aggs/ticker/AAPL/range/1/minute/2024-01-15/2024-01-15\", # Polygon.io\n", + " \"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol=AAPL&interval=1min&apikey=demo\", # Alpha Vantage\n", + " \"https://api.github.com/repos/ranaroussi/yfinance\" # Yahoo Finance API\n", + "]\n", + "\n", + "financial_feeds = [\n", + " \"https://feeds.reuters.com/reuters/businessNews\",\n", + " \"https://rss.cnn.com/rss/money_latest.rss\",\n", + " \"https://feeds.bloomberg.com/markets/news.rss\"\n", + "]\n", + "\n", + "# Real database connection for market data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/market_data_db\"\n", + "db_query = \"SELECT symbol, price, volume, timestamp FROM market_data WHERE timestamp > NOW() - INTERVAL '1 hour' ORDER BY timestamp DESC LIMIT 1000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample streaming market data\n", + "market_data_file = os.path.join(temp_dir, \"market_data_stream.json\")\n", + "market_stream = [\n", + " {\n", + " \"symbol\": \"AAPL\",\n", + " \"price\": 175.50,\n", + " \"volume\": 1000000,\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=5)).isoformat()\n", + " },\n", + " {\n", + " \"symbol\": \"MSFT\",\n", + " \"price\": 380.25,\n", + " \"volume\": 800000,\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=4)).isoformat()\n", + " },\n", + " {\n", + " \"symbol\": \"GOOGL\",\n", + " \"price\": 142.80,\n", + " \"volume\": 1200000,\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=3)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(market_data_file, 'w') as f:\n", + " json.dump(market_stream, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(market_data_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(market_data_file)\n", + "\n", + "# Ingest from financial feeds\n", + "financial_feed_list = []\n", + "for feed_url in financial_feeds[:2]:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " financial_feed_list.append(feed_data)\n", + " print(f\"✓ Ingested financial feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Market data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Financial feeds: {len(financial_feed_list)}\")\n", + "print(f\" Streaming sources: {len(stream_sources)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Market Entities and Build Temporal Knowledge Graph\n", + "\n", + "Extract market entities and build temporal knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "market_entities = []\n", + "market_relationships = []\n", + "\n", + "# Extract from market data\n", + "if parsed_data and parsed_data.data:\n", + " for market_entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(market_entry, dict):\n", + " symbol = market_entry.get(\"symbol\", \"\")\n", + " \n", + " market_entities.append({\n", + " \"id\": symbol,\n", + " \"type\": \"Stock\",\n", + " \"name\": symbol,\n", + " \"properties\": {\n", + " \"price\": market_entry.get(\"price\", 0),\n", + " \"volume\": market_entry.get(\"volume\", 0),\n", + " \"timestamp\": market_entry.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Price events\n", + " if market_entry.get(\"price\", 0) > 0:\n", + " market_relationships.append({\n", + " \"source\": symbol,\n", + " \"target\": f\"{symbol}_price_{market_entry.get('timestamp', '')}\",\n", + " \"type\": \"has_price\",\n", + " \"properties\": {\n", + " \"price\": market_entry.get(\"price\", 0),\n", + " \"timestamp\": market_entry.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "market_kg = builder.build(market_entities, market_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(market_kg)\n", + "\n", + "print(f\"Extracted {len(market_entities)} market entities\")\n", + "print(f\"Extracted {len(market_relationships)} relationships\")\n", + "print(f\"Built temporal market knowledge graph with {len(market_kg.get('entities', []))} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Analyze Market Patterns\n", + "\n", + "Analyze market patterns using temporal analysis and graph analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "start_time = (datetime.now() - timedelta(hours=1)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=market_kg,\n", + " query=\"Find market movements in the last hour\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " market_kg,\n", + " pattern_type=\"trend\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "centrality_scores = centrality_calculator.calculate_centrality(market_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(market_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(market_kg)\n", + "\n", + "print(f\"Temporal query returned {len(temporal_results.get('entities', []))} entities\")\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"Central stocks: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n", + "print(f\"Communities: {len(communities)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Predict Market Trends\n", + "\n", + "Predict market trends using inference engine.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Market trend prediction rules\n", + "inference_engine.add_rule(\"IF volume > 1000000 AND price_change > 0 THEN bullish_signal\")\n", + "inference_engine.add_rule(\"IF volume > 1000000 AND price_change < 0 THEN bearish_signal\")\n", + "inference_engine.add_rule(\"IF multiple stocks show same pattern THEN market_trend\")\n", + "\n", + "# Add facts from market data\n", + "if parsed_data and parsed_data.data:\n", + " for market_entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(market_entry, dict):\n", + " inference_engine.add_fact({\n", + " \"symbol\": market_entry.get(\"symbol\", \"\"),\n", + " \"volume\": market_entry.get(\"volume\", 0),\n", + " \"price\": market_entry.get(\"price\", 0)\n", + " })\n", + "\n", + "trend_predictions = inference_engine.forward_chain()\n", + "\n", + "print(f\"Generated {len(trend_predictions)} market trend predictions\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate market analysis reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(market_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(market_kg, os.path.join(temp_dir, \"market_kg.json\"))\n", + "csv_exporter.export_entities(market_entities, os.path.join(temp_dir, \"market_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(market_kg, os.path.join(temp_dir, \"market_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Market data analysis identified {len(trend_predictions)} trend predictions from {len(market_entities)} entities\",\n", + " \"stocks_analyzed\": len(market_entities),\n", + " \"patterns\": len(temporal_patterns),\n", + " \"predictions\": len(trend_predictions),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(market_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(market_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(market_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated market analysis report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Stream Market Data → Parse → Extract → Build Temporal KG → Analyze Patterns → Predict Trends → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/trading/News_Sentiment_Analysis.ipynb b/docs/cookbook/use_cases/trading/News_Sentiment_Analysis.ipynb new file mode 100644 index 00000000..608f40a6 --- /dev/null +++ b/docs/cookbook/use_cases/trading/News_Sentiment_Analysis.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# News Sentiment Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete news sentiment analysis pipeline for trading: ingest financial news from multiple sources (RSS feeds, news APIs, web sources), extract entities, build news knowledge graph, analyze sentiment using embeddings, and generate trading signals.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, FeedIngestor, DBIngestor, FileIngestor\n", + "- **Parsing**: HTMLParser, JSONParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Ingest News → Parse → Extract Entities → Build News KG → Generate Embeddings → Analyze Sentiment → Generate Trading Signals → Export → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Financial News from Multiple Sources\n", + "\n", + "Ingest financial news from RSS feeds, news APIs, and web sources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, FeedIngestor, DBIngestor, FileIngestor\n", + "from semantica.parse import HTMLParser, JSONParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "db_ingestor = DBIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "html_parser = HTMLParser()\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "# Real financial news feed URLs\n", + "financial_feeds = [\n", + " \"https://feeds.reuters.com/reuters/businessNews\", # Reuters Business\n", + " \"https://feeds.reuters.com/reuters/topNews\", # Reuters Top News\n", + " \"https://rss.cnn.com/rss/money_latest.rss\", # CNN Money\n", + " \"https://feeds.bloomberg.com/markets/news.rss\", # Bloomberg Markets\n", + " \"https://www.ft.com/?format=rss\" # Financial Times\n", + "]\n", + "\n", + "# Real news API endpoints\n", + "news_apis = [\n", + " \"https://newsapi.org/v2/everything?q=finance&apiKey=demo\", # NewsAPI (requires API key)\n", + " \"https://api.github.com/repos/financial-news/aggregator\" # Financial news aggregator\n", + "]\n", + "\n", + "# Real database connection for news data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/news_db\"\n", + "db_query = \"SELECT article_id, title, content, sentiment, published_date FROM financial_news WHERE published_date > NOW() - INTERVAL '24 hours' ORDER BY published_date DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample financial news data\n", + "news_file = os.path.join(temp_dir, \"financial_news.json\")\n", + "news_data = {\n", + " \"articles\": [\n", + " {\n", + " \"title\": \"Apple Reports Strong Q4 Earnings\",\n", + " \"content\": \"Apple Inc. reported strong fourth quarter earnings, beating analyst expectations with record revenue.\",\n", + " \"sentiment\": \"positive\",\n", + " \"published_date\": (datetime.now() - timedelta(hours=2)).isoformat(),\n", + " \"symbols\": [\"AAPL\"]\n", + " },\n", + " {\n", + " \"title\": \"Market Volatility Concerns Rise\",\n", + " \"content\": \"Financial markets show increased volatility amid economic uncertainty and geopolitical tensions.\",\n", + " \"sentiment\": \"negative\",\n", + " \"published_date\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"symbols\": [\"SPY\", \"QQQ\"]\n", + " }\n", + " ]\n", + "}\n", + "\n", + "with open(news_file, 'w') as f:\n", + " json.dump(news_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(news_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(news_file)\n", + "\n", + "# Ingest from financial feeds\n", + "financial_feed_list = []\n", + "for feed_url in financial_feeds[:3]: # Process first 3 feeds\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " financial_feed_list.append(feed_data)\n", + " print(f\"✓ Ingested financial feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " print(f\" Items: {len(feed_data.items) if hasattr(feed_data, 'items') else 0}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" News files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Financial feeds: {len(financial_feed_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract News Entities and Build Knowledge Graph\n", + "\n", + "Extract entities from news articles and build knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "news_entities = []\n", + "news_relationships = []\n", + "all_news_texts = []\n", + "\n", + "# Extract from news data\n", + "if parsed_data and parsed_data.data:\n", + " articles = parsed_data.data.get(\"articles\", []) if isinstance(parsed_data.data, dict) else parsed_data.data if isinstance(parsed_data.data, list) else []\n", + " \n", + " for article in articles:\n", + " if isinstance(article, dict):\n", + " article_text = f\"{article.get('title', '')} {article.get('content', '')}\"\n", + " all_news_texts.append(article_text)\n", + " \n", + " news_entities.append({\n", + " \"id\": article.get(\"title\", \"\"),\n", + " \"type\": \"News_Article\",\n", + " \"name\": article.get(\"title\", \"\"),\n", + " \"properties\": {\n", + " \"sentiment\": article.get(\"sentiment\", \"\"),\n", + " \"published_date\": article.get(\"published_date\", \"\")\n", + " }\n", + " })\n", + " \n", + " # Symbols mentioned\n", + " for symbol in article.get(\"symbols\", []):\n", + " news_entities.append({\n", + " \"id\": symbol,\n", + " \"type\": \"Stock\",\n", + " \"name\": symbol,\n", + " \"properties\": {}\n", + " })\n", + " news_relationships.append({\n", + " \"source\": article.get(\"title\", \"\"),\n", + " \"target\": symbol,\n", + " \"type\": \"mentions\",\n", + " \"properties\": {\n", + " \"sentiment\": article.get(\"sentiment\", \"\")\n", + " }\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "\n", + "news_kg = builder.build(news_entities, news_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(news_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(news_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(news_kg)\n", + "\n", + "print(f\"Extracted {len(news_entities)} news entities\")\n", + "print(f\"Extracted {len(news_relationships)} relationships\")\n", + "print(f\"Collected {len(all_news_texts)} news articles\")\n", + "print(f\"Built news knowledge graph with {len(news_kg.get('entities', []))} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Generate Embeddings and Analyze Sentiment\n", + "\n", + "Generate embeddings from news articles and analyze sentiment.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "\n", + "embeddings = embedding_generator.generate(all_news_texts)\n", + "\n", + "# Analyze sentiment from embeddings and article properties\n", + "sentiment_scores = []\n", + "for i, article in enumerate(parsed_data.data.get(\"articles\", []) if parsed_data and parsed_data.data and isinstance(parsed_data.data, dict) else []):\n", + " if isinstance(article, dict):\n", + " sentiment = article.get(\"sentiment\", \"neutral\")\n", + " sentiment_value = 1.0 if sentiment == \"positive\" else -1.0 if sentiment == \"negative\" else 0.0\n", + " \n", + " sentiment_scores.append({\n", + " \"article\": article.get(\"title\", \"\"),\n", + " \"sentiment\": sentiment,\n", + " \"score\": sentiment_value,\n", + " \"symbols\": article.get(\"symbols\", [])\n", + " })\n", + "\n", + "print(f\"Generated embeddings for {len(all_news_texts)} news articles\")\n", + "print(f\"Analyzed sentiment for {len(sentiment_scores)} articles\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Generate Trading Signals\n", + "\n", + "Generate trading signals based on sentiment analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "\n", + "# Trading signal generation rules\n", + "inference_engine.add_rule(\"IF sentiment is positive AND multiple articles mention symbol THEN buy_signal\")\n", + "inference_engine.add_rule(\"IF sentiment is negative AND multiple articles mention symbol THEN sell_signal\")\n", + "inference_engine.add_rule(\"IF sentiment score > 0.5 THEN strong_positive_signal\")\n", + "\n", + "# Generate trading signals\n", + "trading_signals = []\n", + "for sentiment_data in sentiment_scores:\n", + " symbol_sentiment = {}\n", + " for symbol in sentiment_data.get(\"symbols\", []):\n", + " if symbol not in symbol_sentiment:\n", + " symbol_sentiment[symbol] = []\n", + " symbol_sentiment[symbol].append(sentiment_data.get(\"score\", 0))\n", + " \n", + " for symbol, scores in symbol_sentiment.items():\n", + " avg_sentiment = sum(scores) / len(scores) if scores else 0\n", + " signal_type = \"buy\" if avg_sentiment > 0.3 else \"sell\" if avg_sentiment < -0.3 else \"hold\"\n", + " \n", + " trading_signals.append({\n", + " \"symbol\": symbol,\n", + " \"signal\": signal_type,\n", + " \"sentiment_score\": avg_sentiment,\n", + " \"confidence\": abs(avg_sentiment),\n", + " \"timestamp\": datetime.now().isoformat()\n", + " })\n", + " \n", + " inference_engine.add_fact({\n", + " \"symbol\": symbol,\n", + " \"sentiment\": sentiment_data.get(\"sentiment\", \"\"),\n", + " \"score\": avg_sentiment\n", + " })\n", + "\n", + "signal_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Generated {len(trading_signals)} trading signals\")\n", + "print(f\"Inferred {len(signal_insights)} signal patterns\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate sentiment analysis reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(news_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(news_kg, os.path.join(temp_dir, \"news_kg.json\"))\n", + "csv_exporter.export_entities(news_entities, os.path.join(temp_dir, \"news_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(news_kg, os.path.join(temp_dir, \"news_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"News sentiment analysis identified {len(trading_signals)} trading signals from {len(news_entities)} entities\",\n", + " \"articles_analyzed\": len([e for e in news_entities if e.get(\"type\") == \"News_Article\"]),\n", + " \"signals\": len(trading_signals),\n", + " \"buy_signals\": len([s for s in trading_signals if s.get(\"signal\") == \"buy\"]),\n", + " \"sell_signals\": len([s for s in trading_signals if s.get(\"signal\") == \"sell\"]),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(news_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(news_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(news_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated sentiment analysis report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Ingest News → Parse → Extract → Build KG → Embeddings → Sentiment Analysis → Trading Signals → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/trading/Real_Time_Market_Data.ipynb b/docs/cookbook/use_cases/trading/Real_Time_Market_Data.ipynb new file mode 100644 index 00000000..b83938ae --- /dev/null +++ b/docs/cookbook/use_cases/trading/Real_Time_Market_Data.ipynb @@ -0,0 +1,566 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Real-Time Market Data Integration Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates how to integrate Python/FastMCP MCP servers as data sources for real-time market data ingestion. Connect to real-time market data MCP servers via URL, ingest live market data, and build a temporal trading knowledge graph.\n", + "\n", + "**IMPORTANT**: This implementation supports ONLY Python-based MCP servers and FastMCP servers. Users can bring their own Python/FastMCP MCP servers via URL connections.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: MCPIngestor, ingest_mcp, StreamIngestor\n", + "- **Parsing**: MCPParser, JSONParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Connect to Real-Time Market MCP Server → Ingest Live Market Data via MCP → Parse MCP Responses → Extract Trading Entities → Build Temporal Trading KG → Analyze Market Patterns → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Connect to Real-Time Market Data MCP Server\n", + "\n", + "Connect to a Python/FastMCP MCP server that provides real-time market data via URL. The MCP server can expose resources (market feeds) and tools (real-time price queries, market data streams).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import MCPIngestor, ingest_mcp, StreamIngestor\n", + "from semantica.parse import MCPParser, JSONParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Initialize MCP ingestor\n", + "mcp_ingestor = MCPIngestor()\n", + "\n", + "# Connect to real-time market data MCP server via URL\n", + "# Replace with your actual MCP server URL\n", + "# Example: http://localhost:8000/mcp or https://api.example.com/market-mcp\n", + "market_mcp_url = \"http://localhost:8000/mcp\"\n", + "\n", + "try:\n", + " # Connect to MCP server with authentication (if required)\n", + " mcp_ingestor.connect(\n", + " \"market_server\",\n", + " url=market_mcp_url,\n", + " headers={\n", + " \"Authorization\": \"Bearer your_token\",\n", + " \"X-API-Key\": \"your_api_key\"\n", + " } if \"api.example.com\" in market_mcp_url else {}\n", + " )\n", + " print(f\"✓ Connected to real-time market data MCP server at {market_mcp_url}\")\n", + " \n", + " # List available resources (market feeds, price streams)\n", + " resources = mcp_ingestor.list_available_resources(\"market_server\")\n", + " print(f\"\\n📊 Available Resources ({len(resources)}):\")\n", + " for resource in resources[:5]: # Show first 5\n", + " print(f\" - {resource.uri}: {resource.name}\")\n", + " if resource.description:\n", + " print(f\" {resource.description[:80]}...\")\n", + " \n", + " # List available tools (real-time queries, market data streams)\n", + " tools = mcp_ingestor.list_available_tools(\"market_server\")\n", + " print(f\"\\n🔧 Available Tools ({len(tools)}):\")\n", + " for tool in tools[:5]: # Show first 5\n", + " print(f\" - {tool.name}: {tool.description or 'No description'}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Connection failed: {e}\")\n", + " print(\"Note: This example uses a placeholder URL. Replace with your actual MCP server URL.\")\n", + " print(\"For testing, you can use a mock MCP server or skip connection and use sample data below.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest Real-Time Market Data from MCP Server\n", + "\n", + "Ingest live market data using both resource-based and tool-based methods, including streaming data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize parsers\n", + "mcp_parser = MCPParser()\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "market_data = []\n", + "\n", + "# Method 1: Resource-based ingestion\n", + "# Ingest from MCP resources (market feeds)\n", + "try:\n", + " # Example: Ingest real-time market feed resource\n", + " market_feeds = mcp_ingestor.ingest_resources(\n", + " \"market_server\",\n", + " resource_uris=[\"resource://market/real_time\", \"resource://market/stream\"]\n", + " )\n", + " \n", + " for item in market_feeds:\n", + " market_data.append(item)\n", + " print(f\"✓ Ingested resource: {item.resource_uri}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Resource ingestion: {e}\")\n", + "\n", + "# Method 2: Tool-based ingestion\n", + "# Call MCP tools to retrieve real-time data dynamically\n", + "try:\n", + " # Example: Get real-time stock prices\n", + " real_time_prices = mcp_ingestor.ingest_tool_output(\n", + " \"market_server\",\n", + " tool_name=\"get_real_time_prices\",\n", + " arguments={\n", + " \"symbols\": [\"AAPL\", \"MSFT\", \"GOOGL\", \"TSLA\", \"AMZN\"],\n", + " \"interval\": \"1min\"\n", + " }\n", + " )\n", + " \n", + " if real_time_prices:\n", + " market_data.append(real_time_prices)\n", + " print(f\"✓ Retrieved real-time prices via tool\")\n", + " \n", + " # Example: Get market depth data\n", + " market_depth = mcp_ingestor.ingest_tool_output(\n", + " \"market_server\",\n", + " tool_name=\"get_market_depth\",\n", + " arguments={\n", + " \"symbol\": \"AAPL\",\n", + " \"levels\": 5\n", + " }\n", + " )\n", + " \n", + " if market_depth:\n", + " market_data.append(market_depth)\n", + " print(f\"✓ Retrieved market depth via tool\")\n", + " \n", + "except Exception as e:\n", + " print(f\"⚠ Tool-based ingestion: {e}\")\n", + " print(\"Note: Using sample data for demonstration\")\n", + "\n", + "# Sample real-time market data (if MCP server is not available)\n", + "if not market_data:\n", + " print(\"\\n📝 Using sample real-time market data for demonstration:\")\n", + " sample_data = {\n", + " \"real_time_prices\": [\n", + " {\n", + " \"symbol\": \"AAPL\",\n", + " \"price\": 175.50,\n", + " \"change\": 2.30,\n", + " \"change_percent\": 1.33,\n", + " \"volume\": 45000000,\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"bid\": 175.48,\n", + " \"ask\": 175.52,\n", + " \"spread\": 0.04\n", + " },\n", + " {\n", + " \"symbol\": \"MSFT\",\n", + " \"price\": 380.25,\n", + " \"change\": -1.50,\n", + " \"change_percent\": -0.39,\n", + " \"volume\": 28000000,\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"bid\": 380.20,\n", + " \"ask\": 380.30,\n", + " \"spread\": 0.10\n", + " },\n", + " {\n", + " \"symbol\": \"GOOGL\",\n", + " \"price\": 142.80,\n", + " \"change\": 3.20,\n", + " \"change_percent\": 2.29,\n", + " \"volume\": 32000000,\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"bid\": 142.75,\n", + " \"ask\": 142.85,\n", + " \"spread\": 0.10\n", + " },\n", + " {\n", + " \"symbol\": \"TSLA\",\n", + " \"price\": 245.60,\n", + " \"change\": 5.40,\n", + " \"change_percent\": 2.25,\n", + " \"volume\": 55000000,\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"bid\": 245.55,\n", + " \"ask\": 245.65,\n", + " \"spread\": 0.10\n", + " },\n", + " {\n", + " \"symbol\": \"AMZN\",\n", + " \"price\": 155.30,\n", + " \"change\": 1.20,\n", + " \"change_percent\": 0.78,\n", + " \"volume\": 42000000,\n", + " \"timestamp\": datetime.now().isoformat(),\n", + " \"bid\": 155.25,\n", + " \"ask\": 155.35,\n", + " \"spread\": 0.10\n", + " }\n", + " ]\n", + " }\n", + " market_data.append(sample_data)\n", + " print(f\" Loaded {len(sample_data['real_time_prices'])} real-time price records\")\n", + "\n", + "print(f\"\\n📊 Total market data items ingested: {len(market_data)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Parse Real-Time Market Data\n", + "\n", + "Parse the real-time market data received from MCP server responses.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parsed_market_data = []\n", + "\n", + "# Parse MCP responses\n", + "for data_item in market_data:\n", + " try:\n", + " # Parse MCP response (handles JSON, text, binary)\n", + " if isinstance(data_item, dict):\n", + " parsed_item = data_item\n", + " else:\n", + " parsed_item = mcp_parser.parse_response(data_item, response_type=\"json\")\n", + " \n", + " parsed_market_data.append(parsed_item)\n", + " \n", + " except Exception as e:\n", + " print(f\"⚠ Parsing error: {e}\")\n", + "\n", + "# Extract real-time prices\n", + "real_time_prices = []\n", + "\n", + "for item in parsed_market_data:\n", + " if isinstance(item, dict):\n", + " if \"real_time_prices\" in item:\n", + " real_time_prices.extend(item[\"real_time_prices\"])\n", + " elif \"symbol\" in item and \"price\" in item:\n", + " real_time_prices.append(item)\n", + "\n", + "print(f\"✓ Parsed {len(parsed_market_data)} data items\")\n", + "print(f\"✓ Extracted {len(real_time_prices)} real-time price records\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Extract Trading Entities and Relationships\n", + "\n", + "Extract trading entities (stocks, prices, market events) and relationships from real-time MCP data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "trading_entities = []\n", + "trading_relationships = []\n", + "\n", + "# Extract from real-time prices\n", + "for price_data in real_time_prices:\n", + " if isinstance(price_data, dict):\n", + " symbol = price_data.get(\"symbol\", \"\")\n", + " timestamp = price_data.get(\"timestamp\", \"\")\n", + " \n", + " # Stock entity with real-time price\n", + " trading_entities.append({\n", + " \"id\": f\"{symbol}_{timestamp}\",\n", + " \"type\": \"PricePoint\",\n", + " \"name\": f\"{symbol} @ {timestamp}\",\n", + " \"properties\": {\n", + " \"symbol\": symbol,\n", + " \"price\": price_data.get(\"price\", 0),\n", + " \"change\": price_data.get(\"change\", 0),\n", + " \"change_percent\": price_data.get(\"change_percent\", 0),\n", + " \"volume\": price_data.get(\"volume\", 0),\n", + " \"bid\": price_data.get(\"bid\", 0),\n", + " \"ask\": price_data.get(\"ask\", 0),\n", + " \"spread\": price_data.get(\"spread\", 0),\n", + " \"timestamp\": timestamp\n", + " }\n", + " })\n", + " \n", + " # Stock entity\n", + " trading_entities.append({\n", + " \"id\": symbol,\n", + " \"type\": \"Stock\",\n", + " \"name\": symbol,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " # PricePoint-Stock relationship\n", + " trading_relationships.append({\n", + " \"source\": f\"{symbol}_{timestamp}\",\n", + " \"target\": symbol,\n", + " \"type\": \"price_for\",\n", + " \"properties\": {\"timestamp\": timestamp}\n", + " })\n", + " \n", + " # Market event detection\n", + " if price_data.get(\"change_percent\", 0) > 2:\n", + " event_id = f\"event_{symbol}_{timestamp}\"\n", + " trading_entities.append({\n", + " \"id\": event_id,\n", + " \"type\": \"MarketEvent\",\n", + " \"name\": \"Price Surge\",\n", + " \"properties\": {\n", + " \"event_type\": \"surge\",\n", + " \"magnitude\": price_data.get(\"change_percent\", 0),\n", + " \"timestamp\": timestamp\n", + " }\n", + " })\n", + " trading_relationships.append({\n", + " \"source\": event_id,\n", + " \"target\": symbol,\n", + " \"type\": \"affects\",\n", + " \"properties\": {\"timestamp\": timestamp}\n", + " })\n", + "\n", + "# Remove duplicates\n", + "seen_entities = set()\n", + "unique_entities = []\n", + "for entity in trading_entities:\n", + " entity_key = (entity[\"id\"], entity[\"type\"])\n", + " if entity_key not in seen_entities:\n", + " seen_entities.add(entity_key)\n", + " unique_entities.append(entity)\n", + "\n", + "trading_entities = unique_entities\n", + "\n", + "print(f\"✓ Extracted {len(trading_entities)} trading entities\")\n", + "print(f\" - Price Points: {len([e for e in trading_entities if e['type'] == 'PricePoint'])}\")\n", + "print(f\" - Stocks: {len([e for e in trading_entities if e['type'] == 'Stock'])}\")\n", + "print(f\" - Market Events: {len([e for e in trading_entities if e['type'] == 'MarketEvent'])}\")\n", + "print(f\"✓ Extracted {len(trading_relationships)} relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Build Temporal Trading Knowledge Graph\n", + "\n", + "Build a temporal knowledge graph from the extracted trading entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "# Build temporal knowledge graph\n", + "trading_kg = builder.build(trading_entities, trading_relationships)\n", + "\n", + "# Analyze graph structure\n", + "metrics = graph_analyzer.compute_metrics(trading_kg)\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "# Calculate graph metrics\n", + "centrality_scores = centrality_calculator.calculate_centrality(trading_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(trading_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(trading_kg)\n", + "\n", + "# Detect temporal patterns\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " trading_kg,\n", + " pattern_type=\"trend\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "print(f\"✓ Built temporal trading knowledge graph\")\n", + "print(f\" Entities: {len(trading_kg.get('entities', []))}\")\n", + "print(f\" Relationships: {len(trading_kg.get('relationships', []))}\")\n", + "print(f\" Graph density: {metrics.get('density', 0):.3f}\")\n", + "print(f\" Communities detected: {len(communities)}\")\n", + "print(f\" Temporal patterns: {len(temporal_patterns)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Analyze Market Patterns\n", + "\n", + "Analyze real-time market patterns using temporal queries and pattern detection.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Temporal analysis\n", + "start_time = (datetime.now() - timedelta(hours=1)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=trading_kg,\n", + " query=\"Find price movements in the last hour\",\n", + " start_time=start_time,\n", + " end_time=end_time\n", + ")\n", + "\n", + "# Inference engine for trading rules\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Trading analysis rules\n", + "inference_engine.add_rule(\"IF change_percent > 2 AND volume > 40000000 THEN strong_momentum\")\n", + "inference_engine.add_rule(\"IF change_percent < -1 AND volume > 50000000 THEN selling_pressure\")\n", + "inference_engine.add_rule(\"IF spread < 0.05 AND volume > 30000000 THEN high_liquidity\")\n", + "\n", + "# Add facts from real-time price data\n", + "for price_data in real_time_prices:\n", + " if isinstance(price_data, dict):\n", + " inference_engine.add_fact({\n", + " \"symbol\": price_data.get(\"symbol\", \"\"),\n", + " \"change_percent\": price_data.get(\"change_percent\", 0),\n", + " \"volume\": price_data.get(\"volume\", 0),\n", + " \"spread\": price_data.get(\"spread\", 0)\n", + " })\n", + "\n", + "# Generate trading insights\n", + "trading_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"✓ Temporal analysis completed\")\n", + "print(f\" Temporal entities: {len(temporal_results.get('entities', []))}\")\n", + "print(f\" Trading insights: {len(trading_insights)}\")\n", + "\n", + "# Display insights\n", + "for insight in trading_insights[:3]:\n", + " print(f\" - {insight}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Export and Visualize\n", + "\n", + "Export the trading knowledge graph and generate visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "import os\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export knowledge graph\n", + "json_exporter.export_knowledge_graph(trading_kg, os.path.join(temp_dir, \"trading_kg.json\"))\n", + "csv_exporter.export_entities(trading_entities, os.path.join(temp_dir, \"trading_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(trading_kg, os.path.join(temp_dir, \"trading_kg.rdf\"))\n", + "\n", + "# Generate report\n", + "report_data = {\n", + " \"summary\": f\"Real-time market data integration from MCP server identified {len(trading_insights)} insights\",\n", + " \"price_points\": len([e for e in trading_entities if e['type'] == 'PricePoint']),\n", + " \"stocks\": len([e for e in trading_entities if e['type'] == 'Stock']),\n", + " \"market_events\": len([e for e in trading_entities if e['type'] == 'MarketEvent']),\n", + " \"temporal_patterns\": len(temporal_patterns),\n", + " \"insights\": len(trading_insights)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "print(\"✓ Exported trading knowledge graph\")\n", + "print(f\" JSON: {os.path.join(temp_dir, 'trading_kg.json')}\")\n", + "print(f\" CSV: {os.path.join(temp_dir, 'trading_entities.csv')}\")\n", + "print(f\" RDF: {os.path.join(temp_dir, 'trading_kg.rdf')}\")\n", + "print(f\"✓ Generated report ({len(report)} characters)\")\n", + "\n", + "# Visualize\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(trading_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(trading_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(trading_kg, output=\"interactive\")\n", + "\n", + "print(\"✓ Generated visualizations for trading knowledge graph\")\n", + "\n", + "# Cleanup: Disconnect from MCP server\n", + "try:\n", + " mcp_ingestor.disconnect(\"market_server\")\n", + " print(\"\\n✓ Disconnected from MCP server\")\n", + "except:\n", + " pass\n", + "\n", + "print(f\"\\n✅ Pipeline complete: MCP Server → Ingest Real-Time Data → Parse → Extract → Build Temporal KG → Analyze Patterns → Export → Visualize\")\n", + "print(f\"📊 Total modules used: 20+\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/trading/Real_Time_Monitoring.ipynb b/docs/cookbook/use_cases/trading/Real_Time_Monitoring.ipynb new file mode 100644 index 00000000..1dda969d --- /dev/null +++ b/docs/cookbook/use_cases/trading/Real_Time_Monitoring.ipynb @@ -0,0 +1,394 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Real-Time Trading Monitoring Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete real-time trading monitoring pipeline: stream trading data from multiple sources (trading platforms, market data streams, databases), build temporal knowledge graph, monitor positions in real-time, detect anomalies, and generate alerts.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: StreamIngestor, FileIngestor, DBIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, AutomatedFixer\n", + "- **Export**: JSONExporter, CSVExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Real-Time Trading Streams → Parse → Extract Entities → Build Temporal KG → Monitor Positions → Detect Anomalies → Generate Alerts → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Stream Trading Data from Multiple Sources\n", + "\n", + "Stream trading data from trading platforms, market data streams, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import StreamIngestor, FileIngestor, DBIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor, AutomatedFixer\n", + "from semantica.export import JSONExporter, CSVExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "import time\n", + "from datetime import datetime, timedelta\n", + "from collections import deque\n", + "\n", + "stream_ingestor = StreamIngestor()\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real streaming sources for trading data\n", + "stream_sources = [\n", + " {\n", + " \"type\": \"kafka\",\n", + " \"topic\": \"trading_data\",\n", + " \"bootstrap_servers\": [\"localhost:9092\"],\n", + " \"consumer_config\": {\"group_id\": \"trading_monitor\"}\n", + " },\n", + " {\n", + " \"type\": \"rabbitmq\",\n", + " \"queue\": \"trading_events\",\n", + " \"connection_url\": \"amqp://user:password@localhost:5672/\"\n", + " }\n", + "]\n", + "\n", + "# Real market data APIs\n", + "market_apis = [\n", + " \"https://api.polygon.io/v2/aggs/ticker/AAPL/range/1/minute/2024-01-15/2024-01-15\", # Polygon.io\n", + " \"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol=AAPL&interval=1min&apikey=demo\" # Alpha Vantage\n", + "]\n", + "\n", + "# Real database connection for trading positions\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/trading_db\"\n", + "db_query = \"SELECT position_id, symbol, quantity, entry_price, current_price, timestamp FROM positions WHERE timestamp > NOW() - INTERVAL '1 hour' ORDER BY timestamp DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample real-time trading data\n", + "trading_stream_file = os.path.join(temp_dir, \"trading_stream.json\")\n", + "trading_stream = [\n", + " {\n", + " \"position_id\": \"POS-001\",\n", + " \"symbol\": \"AAPL\",\n", + " \"quantity\": 100,\n", + " \"entry_price\": 175.00,\n", + " \"current_price\": 175.50,\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=5)).isoformat()\n", + " },\n", + " {\n", + " \"position_id\": \"POS-002\",\n", + " \"symbol\": \"MSFT\",\n", + " \"quantity\": 50,\n", + " \"entry_price\": 380.00,\n", + " \"current_price\": 380.25,\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=4)).isoformat()\n", + " },\n", + " {\n", + " \"position_id\": \"POS-003\",\n", + " \"symbol\": \"GOOGL\",\n", + " \"quantity\": 75,\n", + " \"entry_price\": 142.00,\n", + " \"current_price\": 142.80,\n", + " \"timestamp\": (datetime.now() - timedelta(minutes=3)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(trading_stream_file, 'w') as f:\n", + " json.dump(trading_stream, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(trading_stream_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(trading_stream_file)\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Trading stream files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Streaming sources: {len(stream_sources)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Trading Entities and Build Temporal Knowledge Graph\n", + "\n", + "Extract trading entities and build temporal knowledge graph for real-time monitoring.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "trading_entities = []\n", + "trading_relationships = []\n", + "\n", + "# Extract from trading stream data\n", + "if parsed_data and parsed_data.data:\n", + " for position in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(position, dict):\n", + " position_id = position.get(\"position_id\", \"\")\n", + " symbol = position.get(\"symbol\", \"\")\n", + " \n", + " trading_entities.append({\n", + " \"id\": position_id,\n", + " \"type\": \"Position\",\n", + " \"name\": position_id,\n", + " \"properties\": {\n", + " \"quantity\": position.get(\"quantity\", 0),\n", + " \"entry_price\": position.get(\"entry_price\", 0),\n", + " \"current_price\": position.get(\"current_price\", 0),\n", + " \"timestamp\": position.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + " \n", + " trading_entities.append({\n", + " \"id\": symbol,\n", + " \"type\": \"Stock\",\n", + " \"name\": symbol,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " trading_relationships.append({\n", + " \"source\": position_id,\n", + " \"target\": symbol,\n", + " \"type\": \"holds\",\n", + " \"properties\": {\n", + " \"quantity\": position.get(\"quantity\", 0),\n", + " \"timestamp\": position.get(\"timestamp\", \"\")\n", + " }\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "trading_kg = builder.build(trading_entities, trading_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(trading_kg)\n", + "\n", + "print(f\"Extracted {len(trading_entities)} trading entities\")\n", + "print(f\"Extracted {len(trading_relationships)} relationships\")\n", + "print(f\"Built temporal trading knowledge graph with {len(trading_kg.get('entities', []))} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Monitor Positions in Real-Time\n", + "\n", + "Monitor trading positions using temporal queries.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "\n", + "current_time = datetime.now().isoformat()\n", + "start_time = (datetime.now() - timedelta(minutes=10)).isoformat()\n", + "\n", + "# Query current positions\n", + "current_positions = temporal_query.query_time_range(\n", + " graph=trading_kg,\n", + " query=\"Find current positions\",\n", + " start_time=start_time,\n", + " end_time=current_time\n", + ")\n", + "\n", + "# Monitor position changes\n", + "position_changes = []\n", + "if parsed_data and parsed_data.data:\n", + " for position in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(position, dict):\n", + " price_change = position.get(\"current_price\", 0) - position.get(\"entry_price\", 0)\n", + " price_change_percent = (price_change / position.get(\"entry_price\", 1)) * 100 if position.get(\"entry_price\", 0) > 0 else 0\n", + " \n", + " position_changes.append({\n", + " \"position_id\": position.get(\"position_id\", \"\"),\n", + " \"symbol\": position.get(\"symbol\", \"\"),\n", + " \"price_change\": price_change,\n", + " \"price_change_percent\": price_change_percent,\n", + " \"timestamp\": position.get(\"timestamp\", \"\")\n", + " })\n", + "\n", + "centrality_scores = centrality_calculator.calculate_centrality(trading_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(trading_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(trading_kg)\n", + "\n", + "print(f\"Monitoring {len(current_positions.get('entities', []))} current positions\")\n", + "print(f\"Position changes tracked: {len(position_changes)}\")\n", + "print(f\"Central positions: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Detect Anomalies and Generate Alerts\n", + "\n", + "Detect anomalies in trading positions and generate alerts.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Anomaly detection rules\n", + "inference_engine.add_rule(\"IF price_change_percent > 5 AND quantity > 100 THEN large_gain_alert\")\n", + "inference_engine.add_rule(\"IF price_change_percent < -5 AND quantity > 100 THEN large_loss_alert\")\n", + "inference_engine.add_rule(\"IF price_change_percent > 10 THEN extreme_movement_alert\")\n", + "\n", + "# Detect anomalies\n", + "anomalies = []\n", + "alerts = []\n", + "\n", + "for position_change in position_changes:\n", + " anomaly_score = 0\n", + " reasons = []\n", + " \n", + " if abs(position_change.get(\"price_change_percent\", 0)) > 5:\n", + " anomaly_score += 3\n", + " reasons.append(\"Significant price movement\")\n", + " \n", + " if position_change.get(\"price_change_percent\", 0) > 10:\n", + " anomaly_score += 5\n", + " reasons.append(\"Extreme price movement\")\n", + " \n", + " if anomaly_score >= 3:\n", + " anomaly = {\n", + " \"position_id\": position_change.get(\"position_id\", \"\"),\n", + " \"symbol\": position_change.get(\"symbol\", \"\"),\n", + " \"severity\": \"high\" if anomaly_score >= 5 else \"medium\",\n", + " \"score\": anomaly_score,\n", + " \"reasons\": reasons,\n", + " \"timestamp\": position_change.get(\"timestamp\", \"\")\n", + " }\n", + " anomalies.append(anomaly)\n", + " \n", + " alert = {\n", + " \"alert_id\": f\"alert_{position_change.get('position_id', '')}_{int(time.time())}\",\n", + " \"type\": \"trading_anomaly\",\n", + " \"severity\": anomaly[\"severity\"],\n", + " \"position\": position_change.get(\"position_id\", \"\"),\n", + " \"symbol\": position_change.get(\"symbol\", \"\"),\n", + " \"message\": f\"Anomaly detected: {', '.join(reasons)}\",\n", + " \"timestamp\": position_change.get(\"timestamp\", \"\")\n", + " }\n", + " alerts.append(alert)\n", + " \n", + " inference_engine.add_fact({\n", + " \"position_id\": position_change.get(\"position_id\", \"\"),\n", + " \"price_change_percent\": position_change.get(\"price_change_percent\", 0),\n", + " \"quantity\": position_change.get(\"quantity\", 0) if \"quantity\" in position_change else 0\n", + " })\n", + "\n", + "detected_anomalies = inference_engine.forward_chain()\n", + "\n", + "print(f\"Detected {len(anomalies)} trading anomalies\")\n", + "print(f\"Generated {len(alerts)} alerts\")\n", + "print(f\"Inferred {len(detected_anomalies)} anomaly patterns\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate real-time monitoring reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(trading_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(trading_kg, os.path.join(temp_dir, \"trading_kg.json\"))\n", + "csv_exporter.export_entities(trading_entities, os.path.join(temp_dir, \"trading_entities.csv\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Real-time monitoring detected {len(anomalies)} anomalies and generated {len(alerts)} alerts\",\n", + " \"positions_monitored\": len([e for e in trading_entities if e.get(\"type\") == \"Position\"]),\n", + " \"anomalies\": len(anomalies),\n", + " \"alerts\": len(alerts),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(trading_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(trading_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(trading_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated real-time monitoring report and visualizations\")\n", + "print(f\"Real-time monitoring active\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Real-Time Streams → Parse → Extract → Build Temporal KG → Monitor Positions → Detect Anomalies → Alerts → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/trading/Risk_Assessment.ipynb b/docs/cookbook/use_cases/trading/Risk_Assessment.ipynb new file mode 100644 index 00000000..d1760b26 --- /dev/null +++ b/docs/cookbook/use_cases/trading/Risk_Assessment.ipynb @@ -0,0 +1,399 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Risk Assessment Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete risk assessment pipeline for trading: ingest risk data from multiple sources (portfolio data, market risk metrics, historical data), extract risk entities, build risk knowledge graph, analyze risk relationships, and assess portfolio risk.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: FileIngestor, DBIngestor, WebIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Risk Data Sources → Parse → Extract Risk Entities → Build Risk KG → Analyze Risk Relationships → Assess Portfolio Risk → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Risk Data from Multiple Sources\n", + "\n", + "Ingest risk data from portfolio databases, market risk metrics, and historical data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import FileIngestor, DBIngestor, WebIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.conflicts import ConflictDetector\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "file_ingestor = FileIngestor()\n", + "db_ingestor = DBIngestor()\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real risk data sources\n", + "risk_apis = [\n", + " \"https://api.polygon.io/v2/aggs/ticker/AAPL/range/1/day/2024-01-01/2024-01-31\", # Polygon.io\n", + " \"https://www.alphavantage.co/query?function=OVERVIEW&symbol=AAPL&apikey=demo\" # Alpha Vantage\n", + "]\n", + "\n", + "financial_feeds = [\n", + " \"https://feeds.reuters.com/reuters/businessNews\",\n", + " \"https://rss.cnn.com/rss/money_latest.rss\"\n", + "]\n", + "\n", + "# Real database connection for portfolio risk data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/portfolio_db\"\n", + "db_query = \"SELECT portfolio_id, symbol, quantity, value, risk_score, volatility FROM portfolio_positions WHERE last_updated > NOW() - INTERVAL '1 day' ORDER BY risk_score DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample portfolio risk data\n", + "risk_data_file = os.path.join(temp_dir, \"portfolio_risk.json\")\n", + "portfolio_risk_data = {\n", + " \"portfolio_id\": \"PORT-001\",\n", + " \"positions\": [\n", + " {\n", + " \"symbol\": \"AAPL\",\n", + " \"quantity\": 100,\n", + " \"value\": 17550.00,\n", + " \"risk_score\": 0.15,\n", + " \"volatility\": 0.20,\n", + " \"beta\": 1.2\n", + " },\n", + " {\n", + " \"symbol\": \"MSFT\",\n", + " \"quantity\": 50,\n", + " \"value\": 19012.50,\n", + " \"risk_score\": 0.12,\n", + " \"volatility\": 0.18,\n", + " \"beta\": 0.9\n", + " },\n", + " {\n", + " \"symbol\": \"GOOGL\",\n", + " \"quantity\": 75,\n", + " \"value\": 10710.00,\n", + " \"risk_score\": 0.18,\n", + " \"volatility\": 0.25,\n", + " \"beta\": 1.1\n", + " }\n", + " ],\n", + " \"total_value\": 47272.50,\n", + " \"portfolio_risk_score\": 0.15\n", + "}\n", + "\n", + "with open(risk_data_file, 'w') as f:\n", + " json.dump(portfolio_risk_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(risk_data_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(risk_data_file)\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Risk data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Risk Entities and Build Risk Knowledge Graph\n", + "\n", + "Extract risk entities and build risk knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "risk_entities = []\n", + "risk_relationships = []\n", + "\n", + "# Extract from portfolio risk data\n", + "if parsed_data and parsed_data.data:\n", + " portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " \n", + " if isinstance(portfolio, dict):\n", + " portfolio_id = portfolio.get(\"portfolio_id\", \"\")\n", + " \n", + " risk_entities.append({\n", + " \"id\": portfolio_id,\n", + " \"type\": \"Portfolio\",\n", + " \"name\": portfolio_id,\n", + " \"properties\": {\n", + " \"total_value\": portfolio.get(\"total_value\", 0),\n", + " \"portfolio_risk_score\": portfolio.get(\"portfolio_risk_score\", 0)\n", + " }\n", + " })\n", + " \n", + " # Positions and risk metrics\n", + " for position in portfolio.get(\"positions\", []):\n", + " if isinstance(position, dict):\n", + " symbol = position.get(\"symbol\", \"\")\n", + " \n", + " risk_entities.append({\n", + " \"id\": symbol,\n", + " \"type\": \"Stock\",\n", + " \"name\": symbol,\n", + " \"properties\": {\n", + " \"quantity\": position.get(\"quantity\", 0),\n", + " \"value\": position.get(\"value\", 0),\n", + " \"risk_score\": position.get(\"risk_score\", 0),\n", + " \"volatility\": position.get(\"volatility\", 0),\n", + " \"beta\": position.get(\"beta\", 0)\n", + " }\n", + " })\n", + " \n", + " risk_relationships.append({\n", + " \"source\": portfolio_id,\n", + " \"target\": symbol,\n", + " \"type\": \"contains\",\n", + " \"properties\": {\n", + " \"quantity\": position.get(\"quantity\", 0),\n", + " \"value\": position.get(\"value\", 0)\n", + " }\n", + " })\n", + " \n", + " risk_relationships.append({\n", + " \"source\": symbol,\n", + " \"target\": f\"{symbol}_risk\",\n", + " \"type\": \"has_risk\",\n", + " \"properties\": {\n", + " \"risk_score\": position.get(\"risk_score\", 0),\n", + " \"volatility\": position.get(\"volatility\", 0)\n", + " }\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "graph_analyzer = GraphAnalyzer()\n", + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "\n", + "risk_kg = builder.build(risk_entities, risk_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(risk_kg)\n", + "centrality_scores = centrality_calculator.calculate_centrality(risk_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(risk_kg)\n", + "\n", + "print(f\"Extracted {len(risk_entities)} risk entities\")\n", + "print(f\"Extracted {len(risk_relationships)} risk relationships\")\n", + "print(f\"Built risk knowledge graph with {len(risk_kg.get('entities', []))} entities\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Analyze Risk Relationships\n", + "\n", + "Analyze risk relationships using graph analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "\n", + "connectivity = connectivity_analyzer.analyze_connectivity(risk_kg)\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " risk_kg,\n", + " pattern_type=\"risk\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "# Analyze risk concentration\n", + "risk_concentration = {}\n", + "if parsed_data and parsed_data.data:\n", + " portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " if isinstance(portfolio, dict):\n", + " total_value = portfolio.get(\"total_value\", 1)\n", + " for position in portfolio.get(\"positions\", []):\n", + " if isinstance(position, dict):\n", + " symbol = position.get(\"symbol\", \"\")\n", + " value = position.get(\"value\", 0)\n", + " concentration = (value / total_value) * 100 if total_value > 0 else 0\n", + " risk_concentration[symbol] = {\n", + " \"concentration\": concentration,\n", + " \"risk_score\": position.get(\"risk_score\", 0),\n", + " \"value\": value\n", + " }\n", + "\n", + "print(f\"Risk relationships analyzed\")\n", + "print(f\" Connected components: {len(connectivity.get('components', []))}\")\n", + "print(f\" Temporal patterns: {len(temporal_patterns)}\")\n", + "print(f\" Risk concentrations: {len(risk_concentration)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Assess Portfolio Risk\n", + "\n", + "Assess portfolio risk using inference engine.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Portfolio risk assessment rules\n", + "inference_engine.add_rule(\"IF risk_score > 0.2 AND concentration > 20 THEN high_risk_position\")\n", + "inference_engine.add_rule(\"IF volatility > 0.3 AND beta > 1.5 THEN high_volatility_risk\")\n", + "inference_engine.add_rule(\"IF portfolio_risk_score > 0.2 THEN high_portfolio_risk\")\n", + "\n", + "# Assess portfolio risk\n", + "portfolio_risk_assessment = {}\n", + "if parsed_data and parsed_data.data:\n", + " portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + " if isinstance(portfolio, dict):\n", + " portfolio_risk_score = portfolio.get(\"portfolio_risk_score\", 0)\n", + " \n", + " # Calculate weighted risk\n", + " total_risk = 0\n", + " total_value = portfolio.get(\"total_value\", 1)\n", + " for position in portfolio.get(\"positions\", []):\n", + " if isinstance(position, dict):\n", + " position_risk = position.get(\"risk_score\", 0) * (position.get(\"value\", 0) / total_value) if total_value > 0 else 0\n", + " total_risk += position_risk\n", + " \n", + " inference_engine.add_fact({\n", + " \"symbol\": position.get(\"symbol\", \"\"),\n", + " \"risk_score\": position.get(\"risk_score\", 0),\n", + " \"volatility\": position.get(\"volatility\", 0),\n", + " \"beta\": position.get(\"beta\", 0),\n", + " \"concentration\": risk_concentration.get(position.get(\"symbol\", \"\"), {}).get(\"concentration\", 0)\n", + " })\n", + " \n", + " portfolio_risk_assessment = {\n", + " \"portfolio_id\": portfolio.get(\"portfolio_id\", \"\"),\n", + " \"overall_risk_score\": portfolio_risk_score,\n", + " \"weighted_risk\": total_risk,\n", + " \"risk_level\": \"high\" if portfolio_risk_score > 0.2 else \"medium\" if portfolio_risk_score > 0.1 else \"low\",\n", + " \"positions_count\": len(portfolio.get(\"positions\", []))\n", + " }\n", + " \n", + " inference_engine.add_fact({\n", + " \"portfolio_id\": portfolio.get(\"portfolio_id\", \"\"),\n", + " \"portfolio_risk_score\": portfolio_risk_score\n", + " })\n", + "\n", + "risk_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Portfolio risk assessment complete\")\n", + "print(f\" Overall risk score: {portfolio_risk_assessment.get('overall_risk_score', 0):.3f}\")\n", + "print(f\" Risk level: {portfolio_risk_assessment.get('risk_level', 'unknown')}\")\n", + "print(f\" Generated {len(risk_insights)} risk insights\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Reports and Visualize\n", + "\n", + "Generate risk assessment reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(risk_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(risk_kg, os.path.join(temp_dir, \"risk_kg.json\"))\n", + "csv_exporter.export_entities(risk_entities, os.path.join(temp_dir, \"risk_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(risk_kg, os.path.join(temp_dir, \"risk_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Risk assessment identified {len(risk_insights)} risk insights for portfolio {portfolio_risk_assessment.get('portfolio_id', '')}\",\n", + " \"portfolio_risk_score\": portfolio_risk_assessment.get('overall_risk_score', 0),\n", + " \"risk_level\": portfolio_risk_assessment.get('risk_level', 'unknown'),\n", + " \"positions_analyzed\": len([e for e in risk_entities if e.get(\"type\") == \"Stock\"]),\n", + " \"insights\": len(risk_insights),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(risk_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(risk_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(risk_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated risk assessment report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Risk Data → Parse → Extract → Build Risk KG → Analyze Relationships → Assess Portfolio Risk → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/cookbook/use_cases/trading/Strategy_Backtesting.ipynb b/docs/cookbook/use_cases/trading/Strategy_Backtesting.ipynb new file mode 100644 index 00000000..1c890ac3 --- /dev/null +++ b/docs/cookbook/use_cases/trading/Strategy_Backtesting.ipynb @@ -0,0 +1,412 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Strategy Backtesting Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete strategy backtesting pipeline: ingest historical market data from multiple sources (databases, market data APIs, historical feeds), build temporal knowledge graph, test trading strategies on historical data, and analyze performance metrics.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: DBIngestor, FileIngestor, WebIngestor, FeedIngestor\n", + "- **Parsing**: JSONParser, CSVParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor\n", + "- **Export**: JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Historical Data → Parse → Extract Entities → Build Temporal KG → Test Strategies → Analyze Performance → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Historical Market Data\n", + "\n", + "Ingest historical market data from databases, market data APIs, and historical feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import DBIngestor, FileIngestor, WebIngestor, FeedIngestor\n", + "from semantica.parse import JSONParser, CSVParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "db_ingestor = DBIngestor()\n", + "file_ingestor = FileIngestor()\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "csv_parser = CSVParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real historical market data APIs\n", + "historical_market_apis = [\n", + " \"https://api.polygon.io/v2/aggs/ticker/AAPL/range/1/day/2023-01-01/2024-01-01\", # Polygon.io historical\n", + " \"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=AAPL&apikey=demo\", # Alpha Vantage historical\n", + " \"https://api.github.com/repos/ranaroussi/yfinance\" # Yahoo Finance historical data\n", + "]\n", + "\n", + "# Real financial news feeds for historical context\n", + "historical_feeds = [\n", + " \"https://feeds.reuters.com/reuters/businessNews\",\n", + " \"https://rss.cnn.com/rss/money_latest.rss\",\n", + " \"https://feeds.bloomberg.com/markets/news.rss\"\n", + "]\n", + "\n", + "# Real database connection for historical market data\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/historical_market_db\"\n", + "db_query = \"SELECT symbol, date, open, high, low, close, volume FROM historical_prices WHERE date >= '2023-01-01' AND date <= '2024-01-01' ORDER BY date DESC\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample historical market data (simulating real historical data structure)\n", + "historical_data_file = os.path.join(temp_dir, \"historical_data.json\")\n", + "historical_data = [\n", + " {\"symbol\": \"AAPL\", \"date\": \"2023-01-15\", \"open\": 150.00, \"high\": 152.00, \"low\": 149.50, \"close\": 151.50, \"volume\": 50000000},\n", + " {\"symbol\": \"AAPL\", \"date\": \"2023-01-16\", \"open\": 151.50, \"high\": 153.00, \"low\": 151.00, \"close\": 152.75, \"volume\": 52000000},\n", + " {\"symbol\": \"MSFT\", \"date\": \"2023-01-15\", \"open\": 350.00, \"high\": 352.00, \"low\": 349.50, \"close\": 351.25, \"volume\": 30000000},\n", + " {\"symbol\": \"MSFT\", \"date\": \"2023-01-16\", \"open\": 351.25, \"high\": 353.50, \"low\": 350.75, \"close\": 352.50, \"volume\": 31000000}\n", + "]\n", + "\n", + "with open(historical_data_file, 'w') as f:\n", + " json.dump(historical_data, f, indent=2)\n", + "\n", + "file_objects = file_ingestor.ingest_file(historical_data_file, read_content=True)\n", + "parsed_data = structured_parser.parse_json(historical_data_file)\n", + "\n", + "# Ingest from historical market APIs\n", + "historical_api_list = []\n", + "for api_url in historical_market_apis[:1]:\n", + " try:\n", + " api_content = web_ingestor.ingest_url(api_url)\n", + " if api_content:\n", + " historical_api_list.append(api_content)\n", + " print(f\"✓ Ingested historical market API: {api_content.url if hasattr(api_content, 'url') else api_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Historical API ingestion for {api_url}: {str(e)[:100]}\")\n", + "\n", + "# Ingest from historical news feeds\n", + "historical_feed_list = []\n", + "for feed_url in historical_feeds:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " historical_feed_list.append(feed_data)\n", + " print(f\"✓ Ingested historical feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "print(f\"\\n📊 Historical Data Ingestion Summary:\")\n", + "print(f\" Historical data files: {len([file_objects]) if file_objects else 0}\")\n", + "print(f\" Historical market APIs: {len(historical_api_list)}\")\n", + "print(f\" Historical feeds: {len(historical_feed_list)}\")\n", + "print(f\" Database sources: 1\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Market Entities and Build Temporal Knowledge Graph\n", + "\n", + "Extract market entities from historical data and build temporal knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "historical_entities = []\n", + "historical_relationships = []\n", + "\n", + "# Extract from historical data\n", + "if parsed_data and parsed_data.data:\n", + " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + " if isinstance(entry, dict):\n", + " symbol = entry.get(\"symbol\", \"\")\n", + " date = entry.get(\"date\", \"\")\n", + " \n", + " historical_entities.append({\n", + " \"id\": f\"{symbol}_{date}\",\n", + " \"type\": \"Historical_Price\",\n", + " \"name\": f\"{symbol} on {date}\",\n", + " \"properties\": {\n", + " \"symbol\": symbol,\n", + " \"date\": date,\n", + " \"open\": entry.get(\"open\", 0),\n", + " \"high\": entry.get(\"high\", 0),\n", + " \"low\": entry.get(\"low\", 0),\n", + " \"close\": entry.get(\"close\", 0),\n", + " \"volume\": entry.get(\"volume\", 0)\n", + " }\n", + " })\n", + " \n", + " historical_entities.append({\n", + " \"id\": symbol,\n", + " \"type\": \"Stock\",\n", + " \"name\": symbol,\n", + " \"properties\": {}\n", + " })\n", + " \n", + " historical_relationships.append({\n", + " \"source\": symbol,\n", + " \"target\": f\"{symbol}_{date}\",\n", + " \"type\": \"has_price_on\",\n", + " \"properties\": {\"date\": date}\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "temporal_query = TemporalGraphQuery()\n", + "temporal_pattern_detector = TemporalPatternDetector()\n", + "graph_analyzer = GraphAnalyzer()\n", + "\n", + "historical_kg = builder.build(historical_entities, historical_relationships)\n", + "\n", + "metrics = graph_analyzer.compute_metrics(historical_kg)\n", + "\n", + "print(f\"Extracted {len(historical_entities)} historical entities\")\n", + "print(f\"Extracted {len(historical_relationships)} relationships\")\n", + "print(f\"Built temporal knowledge graph with {len(historical_kg.get('entities', []))} entities\")\n", + "print(f\"Graph density: {metrics.get('density', 0):.3f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Test Trading Strategies\n", + "\n", + "Test trading strategies on historical data using temporal analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define trading strategies\n", + "strategies = [\n", + " {\n", + " \"name\": \"Moving Average Crossover\",\n", + " \"entry_rule\": \"IF close > moving_average_20 THEN buy\",\n", + " \"exit_rule\": \"IF close < moving_average_20 THEN sell\"\n", + " },\n", + " {\n", + " \"name\": \"Momentum Strategy\",\n", + " \"entry_rule\": \"IF price_change > 2% AND volume > average_volume THEN buy\",\n", + " \"exit_rule\": \"IF price_change < -1% THEN sell\"\n", + " }\n", + "]\n", + "\n", + "# Backtest strategies\n", + "backtest_results = []\n", + "for strategy in strategies:\n", + " trades = []\n", + " positions = {}\n", + " \n", + " if parsed_data and parsed_data.data:\n", + " sorted_data = sorted(parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data], \n", + " key=lambda x: x.get(\"date\", \"\"))\n", + " \n", + " for entry in sorted_data:\n", + " if isinstance(entry, dict):\n", + " symbol = entry.get(\"symbol\", \"\")\n", + " close_price = entry.get(\"close\", 0)\n", + " date = entry.get(\"date\", \"\")\n", + " \n", + " # Simple strategy logic (moving average simulation)\n", + " if symbol not in positions:\n", + " # Entry signal\n", + " if close_price > 150: # Simplified entry condition\n", + " positions[symbol] = {\n", + " \"entry_price\": close_price,\n", + " \"entry_date\": date,\n", + " \"quantity\": 100\n", + " }\n", + " else:\n", + " # Exit signal\n", + " if close_price > positions[symbol][\"entry_price\"] * 1.02: # 2% profit target\n", + " trades.append({\n", + " \"symbol\": symbol,\n", + " \"entry_price\": positions[symbol][\"entry_price\"],\n", + " \"exit_price\": close_price,\n", + " \"entry_date\": positions[symbol][\"entry_date\"],\n", + " \"exit_date\": date,\n", + " \"profit\": (close_price - positions[symbol][\"entry_price\"]) * positions[symbol][\"quantity\"],\n", + " \"return_pct\": ((close_price - positions[symbol][\"entry_price\"]) / positions[symbol][\"entry_price\"]) * 100\n", + " })\n", + " del positions[symbol]\n", + " \n", + " total_profit = sum(t[\"profit\"] for t in trades)\n", + " total_return = sum(t[\"return_pct\"] for t in trades) / len(trades) if trades else 0\n", + " \n", + " backtest_results.append({\n", + " \"strategy\": strategy[\"name\"],\n", + " \"trades\": len(trades),\n", + " \"total_profit\": total_profit,\n", + " \"average_return\": total_return,\n", + " \"win_rate\": len([t for t in trades if t[\"profit\"] > 0]) / len(trades) if trades else 0\n", + " })\n", + "\n", + "print(f\"Backtested {len(strategies)} trading strategies\")\n", + "for result in backtest_results:\n", + " print(f\" Strategy: {result['strategy']} - Trades: {result['trades']}, Profit: ${result['total_profit']:.2f}, Avg Return: {result['average_return']:.2f}%\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze Performance Metrics\n", + "\n", + "Analyze strategy performance using graph analytics and inference.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "centrality_calculator = CentralityCalculator()\n", + "community_detector = CommunityDetector()\n", + "connectivity_analyzer = ConnectivityAnalyzer()\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "explanation_generator = ExplanationGenerator()\n", + "\n", + "# Analyze graph structure\n", + "centrality_scores = centrality_calculator.calculate_centrality(historical_kg, measure=\"degree\")\n", + "communities = community_detector.detect_communities(historical_kg)\n", + "connectivity = connectivity_analyzer.analyze_connectivity(historical_kg)\n", + "\n", + "# Temporal pattern detection\n", + "start_date = \"2023-01-01\"\n", + "end_date = \"2024-01-01\"\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " graph=historical_kg,\n", + " query=\"Find price movements in backtest period\",\n", + " start_time=start_date,\n", + " end_time=end_date\n", + ")\n", + "\n", + "temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(\n", + " historical_kg,\n", + " pattern_type=\"trend\",\n", + " min_frequency=1\n", + ")\n", + "\n", + "# Performance inference rules\n", + "inference_engine.add_rule(\"IF average_return > 5% AND win_rate > 0.6 THEN profitable_strategy\")\n", + "inference_engine.add_rule(\"IF total_profit > 1000 AND trades > 10 THEN successful_backtest\")\n", + "\n", + "for result in backtest_results:\n", + " inference_engine.add_fact({\n", + " \"strategy\": result[\"strategy\"],\n", + " \"average_return\": result[\"average_return\"],\n", + " \"win_rate\": result[\"win_rate\"],\n", + " \"total_profit\": result[\"total_profit\"],\n", + " \"trades\": result[\"trades\"]\n", + " })\n", + "\n", + "performance_insights = inference_engine.forward_chain()\n", + "\n", + "print(f\"Performance analysis complete\")\n", + "print(f\" Temporal patterns: {len(temporal_patterns)}\")\n", + "print(f\" Central stocks: {len([e for e, score in centrality_scores.items() if score > 0])}\")\n", + "print(f\" Communities: {len(communities)}\")\n", + "print(f\" Performance insights: {len(performance_insights)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Backtest Reports and Visualize\n", + "\n", + "Generate comprehensive backtest reports and visualize results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "json_exporter = JSONExporter()\n", + "csv_exporter = CSVExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "quality_score = quality_assessor.assess_overall_quality(historical_kg)\n", + "\n", + "json_exporter.export_knowledge_graph(historical_kg, os.path.join(temp_dir, \"backtest_kg.json\"))\n", + "csv_exporter.export_entities(historical_entities, os.path.join(temp_dir, \"historical_entities.csv\"))\n", + "rdf_exporter.export_knowledge_graph(historical_kg, os.path.join(temp_dir, \"backtest_kg.rdf\"))\n", + "\n", + "report_data = {\n", + " \"summary\": f\"Strategy backtesting analyzed {len(backtest_results)} strategies on {len(historical_entities)} historical data points\",\n", + " \"strategies_tested\": len(backtest_results),\n", + " \"total_trades\": sum(r[\"trades\"] for r in backtest_results),\n", + " \"best_strategy\": max(backtest_results, key=lambda x: x[\"total_profit\"])[\"strategy\"] if backtest_results else \"N/A\",\n", + " \"patterns_detected\": len(temporal_patterns),\n", + " \"quality_score\": quality_score.get('overall_score', 0)\n", + "}\n", + "\n", + "report = report_generator.generate_report(report_data, format=\"markdown\")\n", + "\n", + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "kg_viz = kg_visualizer.visualize_network(historical_kg, output=\"interactive\")\n", + "temporal_viz = temporal_visualizer.visualize_timeline(historical_kg, output=\"interactive\")\n", + "analytics_viz = analytics_visualizer.visualize_analytics(historical_kg, output=\"interactive\")\n", + "\n", + "print(\"Generated backtest report and visualizations\")\n", + "print(f\"Total modules used: 20+\")\n", + "print(f\"Pipeline complete: Historical Data → Parse → Extract → Build Temporal KG → Test Strategies → Analyze Performance → Reports → Visualize\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/css/custom.css b/docs/css/custom.css index 2944a80a..b2f4fbf5 100644 --- a/docs/css/custom.css +++ b/docs/css/custom.css @@ -8,6 +8,10 @@ html { /* Cream background */ :root { --md-default-bg-color: #FAF7F2; + --md-primary-fg-color: #4A4F2F; + --md-primary-fg-color--light: #8C9464; + --md-primary-fg-color--dark: #3A3E25; + --md-accent-fg-color: #8C9464; } [data-md-color-scheme="slate"] { diff --git a/docs/reference/core.md b/docs/reference/core.md new file mode 100644 index 00000000..9acf3f49 --- /dev/null +++ b/docs/reference/core.md @@ -0,0 +1,3 @@ +# Core + +::: semantica.core diff --git a/docs/reference/embeddings.md b/docs/reference/embeddings.md new file mode 100644 index 00000000..37c51c7d --- /dev/null +++ b/docs/reference/embeddings.md @@ -0,0 +1,3 @@ +# Embeddings + +::: semantica.embeddings diff --git a/docs/reference/export.md b/docs/reference/export.md new file mode 100644 index 00000000..d1abba81 --- /dev/null +++ b/docs/reference/export.md @@ -0,0 +1,3 @@ +# Export + +::: semantica.export diff --git a/docs/reference/ingest.md b/docs/reference/ingest.md new file mode 100644 index 00000000..5071e5dd --- /dev/null +++ b/docs/reference/ingest.md @@ -0,0 +1,3 @@ +# Ingest + +::: semantica.ingest diff --git a/docs/reference/kg.md b/docs/reference/kg.md new file mode 100644 index 00000000..506f4d2f --- /dev/null +++ b/docs/reference/kg.md @@ -0,0 +1,3 @@ +# Kg + +::: semantica.kg diff --git a/docs/reference/normalize.md b/docs/reference/normalize.md new file mode 100644 index 00000000..a7cef528 --- /dev/null +++ b/docs/reference/normalize.md @@ -0,0 +1,3 @@ +# Normalize + +::: semantica.normalize diff --git a/docs/reference/ontology.md b/docs/reference/ontology.md new file mode 100644 index 00000000..29d2dfee --- /dev/null +++ b/docs/reference/ontology.md @@ -0,0 +1,3 @@ +# Ontology + +::: semantica.ontology diff --git a/docs/reference/parse.md b/docs/reference/parse.md new file mode 100644 index 00000000..91a864d5 --- /dev/null +++ b/docs/reference/parse.md @@ -0,0 +1,3 @@ +# Parse + +::: semantica.parse diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md new file mode 100644 index 00000000..1c3d970e --- /dev/null +++ b/docs/reference/pipeline.md @@ -0,0 +1,3 @@ +# Pipeline + +::: semantica.pipeline diff --git a/docs/reference/reasoning.md b/docs/reference/reasoning.md new file mode 100644 index 00000000..23fb4d70 --- /dev/null +++ b/docs/reference/reasoning.md @@ -0,0 +1,3 @@ +# Reasoning + +::: semantica.reasoning diff --git a/docs/reference/semantic_extract.md b/docs/reference/semantic_extract.md new file mode 100644 index 00000000..503dbabd --- /dev/null +++ b/docs/reference/semantic_extract.md @@ -0,0 +1,3 @@ +# Semantic Extract + +::: semantica.semantic_extract diff --git a/docs/reference/triple_store.md b/docs/reference/triple_store.md new file mode 100644 index 00000000..bd34b4b1 --- /dev/null +++ b/docs/reference/triple_store.md @@ -0,0 +1,3 @@ +# Triple Store + +::: semantica.triple_store diff --git a/docs/reference/utils.md b/docs/reference/utils.md new file mode 100644 index 00000000..38aabf2d --- /dev/null +++ b/docs/reference/utils.md @@ -0,0 +1,3 @@ +# Utils + +::: semantica.utils diff --git a/docs/reference/vector_store.md b/docs/reference/vector_store.md new file mode 100644 index 00000000..9f982c62 --- /dev/null +++ b/docs/reference/vector_store.md @@ -0,0 +1,3 @@ +# Vector Store + +::: semantica.vector_store diff --git a/docs/reference/visualization.md b/docs/reference/visualization.md new file mode 100644 index 00000000..872c9937 --- /dev/null +++ b/docs/reference/visualization.md @@ -0,0 +1,3 @@ +# Visualization + +::: semantica.visualization diff --git a/mkdocs.yml b/mkdocs.yml index e0b8c0f7..2c4e7656 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,15 +14,15 @@ theme: palette: # Light mode - scheme: default - primary: "#4A4F2F" - accent: "#8C9464" + primary: custom + accent: custom toggle: icon: material/brightness-7 name: Switch to dark mode # Dark mode - scheme: slate - primary: "#4A4F2F" - accent: "#8C9464" + primary: custom + accent: custom toggle: icon: material/brightness-4 name: Switch to light mode @@ -76,6 +76,16 @@ plugins: lang: en - minify: minify_html: true + - mkdocstrings: + handlers: + python: + options: + docstring_style: google + show_source: true + show_root_heading: true + show_category_heading: true + - mkdocs-jupyter: + include_source: true # Custom CSS extra_css: @@ -91,11 +101,43 @@ nav: - Guides: - concepts.md - examples.md - - cookbook.md - learning-more.md - - Advanced: - - deep-dive.md - - api.md + - Cookbook: + - Introduction: + - cookbook/introduction/Welcome_to_Semantica.ipynb + - cookbook/introduction/Configuration_Basics.ipynb + - cookbook/introduction/Your_First_Knowledge_Graph.ipynb + - cookbook/introduction/Data_Ingestion.ipynb + - cookbook/introduction/Document_Parsing.ipynb + - cookbook/introduction/Data_Normalization.ipynb + - cookbook/introduction/Entity_Extraction.ipynb + - cookbook/introduction/Relation_Extraction.ipynb + - cookbook/introduction/Embedding_Generation.ipynb + - cookbook/introduction/Vector_Store.ipynb + - cookbook/introduction/Ontology.ipynb + - cookbook/introduction/Conflict_Detection.ipynb + - cookbook/introduction/Deduplication.ipynb + - cookbook/introduction/Building_Knowledge_Graphs.ipynb + - cookbook/introduction/Graph_Analytics.ipynb + - cookbook/introduction/Graph_Quality.ipynb + - cookbook/introduction/Visualization.ipynb + - cookbook/introduction/Export.ipynb + - Reference: + - Core: reference/core.md + - Ingest: reference/ingest.md + - Parse: reference/parse.md + - Normalize: reference/normalize.md + - Semantic Extract: reference/semantic_extract.md + - Knowledge Graph: reference/kg.md + - Embeddings: reference/embeddings.md + - Vector Store: reference/vector_store.md + - Triple Store: reference/triple_store.md + - Ontology: reference/ontology.md + - Reasoning: reference/reasoning.md + - Pipeline: reference/pipeline.md + - Export: reference/export.md + - Visualization: reference/visualization.md + - Utils: reference/utils.md - Resources: - community-projects.md - citation.md diff --git a/requirements-docs.txt b/requirements-docs.txt index ac18083e..b2b4b5de 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -3,5 +3,7 @@ mkdocs-material>=9.4.0 mkdocs-minify-plugin>=0.7.0 mkdocs-mermaid2-plugin>=1.0.0 pymdown-extensions>=10.0 -materialx>=2.2.0 + +mkdocstrings[python]>=0.24.0 +mkdocs-jupyter>=0.24.0 diff --git a/setup_docs.py b/setup_docs.py new file mode 100644 index 00000000..62cba39e --- /dev/null +++ b/setup_docs.py @@ -0,0 +1,46 @@ +import os +import shutil + +# Create directories +os.makedirs("docs/reference", exist_ok=True) +os.makedirs("docs/cookbook", exist_ok=True) + +# Modules to generate docs for +modules = { + "core": "semantica.core", + "ingest": "semantica.ingest", + "parse": "semantica.parse", + "normalize": "semantica.normalize", + "semantic_extract": "semantica.semantic_extract", + "kg": "semantica.kg", + "embeddings": "semantica.embeddings", + "vector_store": "semantica.vector_store", + "triple_store": "semantica.triple_store", + "ontology": "semantica.ontology", + "reasoning": "semantica.reasoning", + "pipeline": "semantica.pipeline", + "export": "semantica.export", + "visualization": "semantica.visualization", + "utils": "semantica.utils" +} + +# Generate reference markdown files +for name, package in modules.items(): + content = f"# {name.replace('_', ' ').title()}\n\n::: {package}\n" + with open(f"docs/reference/{name}.md", "w") as f: + f.write(content) + print(f"Created docs/reference/{name}.md") + +# Copy cookbook directory +if os.path.exists("cookbook"): + if os.path.exists("docs/cookbook"): + shutil.rmtree("docs/cookbook") + shutil.copytree("cookbook", "docs/cookbook") + print("Copied cookbook to docs/cookbook") + +# Remove old files +files_to_remove = ["docs/MODULES_DOCUMENTATION.md", "docs/cookbook.md", "docs/api.md"] +for f in files_to_remove: + if os.path.exists(f): + os.remove(f) + print(f"Removed {f}")