mengzhx/semantica
1
0
Fork 0
mirror of https://github.com/semantica-agi/semantica.git synced 2026-08-29 04:26:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(kg): Remove ConflictDetector and Deduplicator from kg module

Browse Source 
- Remove ConflictDetector and Deduplicator from semantica.kg module
- Update all imports to use dedicated semantica.conflicts and semantica.deduplication modules
- Update all cookbook notebooks to use class-based API instead of convenience functions
- Fix calculate_centrality calls to use specific methods (calculate_degree_centrality, calculate_betweenness_centrality)
- Update detect_communities and analyze_connectivity calls to pass graph parameter
- Update documentation (kg_usage.md, docs/reference/kg.md) to reflect changes
- Remove conflict and deduplicate task types from method registry
This commit is contained in:
KaifAhmad1
2025-12-06 15:57:48 +05:30
parent 6dd9837aa6
commit d258ef6880
22 changed files with 398 additions and 1203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cookbook/advanced/02_Advanced_Graph_Analytics.ipynb
+10 -4
View File
@@ -45,7 +45,8 @@
"metadata": {},
"outputs": [],
"source": [
"from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector, ConnectivityAnalyzer, GraphValidator, Deduplicator\n",
"from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector, ConnectivityAnalyzer, GraphValidator\n",
"from semantica.deduplication import DuplicateDetector, EntityMerger, MergeStrategy\n",
"\n",
"builder = GraphBuilder()\n",
"analyzer = GraphAnalyzer()\n",
@@ -159,13 +160,18 @@
"outputs": [],
"source": [
"graph_validator = GraphValidator()\n",
"deduplicator = Deduplicator()\n",
"\n",
"validation_result = graph_validator.validate(kg)\n",
"deduplicated_kg = deduplicator.deduplicate(kg)\n",
"\n",
"print(f\"Graph validation: {validation_result.get('valid', False)}\")\n",
"print(f\"Deduplicated entities: {len(deduplicated_kg.get('entities', []))}\")\n"
"print(f\"Issues found: {len(validation_result.get('issues', []))}\")\n",
"\n",
"# For deduplication, use semantica.deduplication module:\n",
"# from semantica.deduplication import DuplicateDetector, EntityMerger, MergeStrategy\n",
"# detector = DuplicateDetector(similarity_threshold=0.8)\n",
"# duplicate_groups = detector.detect_duplicate_groups(kg.get('entities', []))\n",
"# merger = EntityMerger()\n",
"# merge_operations = merger.merge_duplicates(kg.get('entities', []), strategy=MergeStrategy.KEEP_MOST_COMPLETE)\n"
]
},
{
cookbook/use_cases/blockchain/02_Transaction_Network_Analysis.ipynb
+1 -1
View File
@@ -62,7 +62,7 @@
"from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n",
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
"from semantica.kg_qa import KGQualityAssessor\n",
"from semantica.kg import ConflictDetector\n",
"from semantica.conflicts import ConflictDetector\n",
"from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n",
"from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n",
"import tempfile\n",
cookbook/use_cases/cybersecurity/01_Anomaly_Detection_Real_Time.ipynb
+3 -2
View File
@@ -292,7 +292,8 @@
"community_detector = CommunityDetector()\n",
"connectivity_analyzer = ConnectivityAnalyzer()\n",
"\n",
"centrality_scores = centrality_calculator.calculate_centrality(temporal_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(temporal_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(temporal_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(temporal_kg)\n",
"\n",
@@ -482,4 +483,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/cybersecurity/02_Incident_Analysis.ipynb
+3 -2
View File
@@ -302,7 +302,8 @@
"# Analyze graph structure\n",
"metrics = graph_analyzer.compute_metrics(incident_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(incident_kg)\n",
"centrality_scores = centrality_calculator.calculate_centrality(incident_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(incident_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"\n",
"print(f\"Built incident knowledge graph\")\n",
"print(f\" Entities: {len(incident_kg.get('entities', []))}\")\n",
@@ -462,4 +463,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/finance/01_Financial_Data_Integration.ipynb
+3 -2
View File
@@ -379,7 +379,8 @@
"connectivity_analyzer = ConnectivityAnalyzer()\n",
"\n",
"# Calculate graph metrics\n",
"centrality_scores = centrality_calculator.calculate_centrality(financial_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(financial_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(financial_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(financial_kg)\n",
"\n",
@@ -517,4 +518,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb
+3 -2
View File
@@ -275,7 +275,8 @@
"\n",
"# Analyze graph structure\n",
"metrics = graph_analyzer.compute_metrics(financial_kg)\n",
"centrality_scores = centrality_calculator.calculate_centrality(financial_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(financial_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(financial_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(financial_kg)\n",
"\n",
@@ -424,4 +425,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/healthcare/02_Disease_Network_Analysis.ipynb
+3 -2
View File
@@ -294,7 +294,8 @@
"temporal_pattern_detector = TemporalPatternDetector()\n",
"\n",
"metrics = graph_analyzer.compute_metrics(disease_kg)\n",
"centrality_scores = centrality_calculator.calculate_centrality(disease_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(disease_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(disease_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(disease_kg)\n",
"\n",
@@ -413,4 +414,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/healthcare/03_Drug_Interactions_Analysis.ipynb
+3 -2
View File
@@ -260,7 +260,8 @@
"drug_kg = builder.build(drug_entities, drug_relationships)\n",
"\n",
"metrics = graph_analyzer.compute_metrics(drug_kg)\n",
"centrality_scores = centrality_calculator.calculate_centrality(drug_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(drug_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(drug_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(drug_kg)\n",
"\n",
@@ -423,4 +424,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/renewable_energy/01_Energy_Market_Analysis.ipynb
+3 -2
View File
@@ -311,7 +311,8 @@
" pricing_analysis[energy_type][\"avg_price\"] = sum(data[\"prices\"]) / len(data[\"prices\"])\n",
" pricing_analysis[energy_type][\"total_volume\"] = sum(data[\"volumes\"])\n",
"\n",
"centrality_scores = centrality_calculator.calculate_centrality(energy_market_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(market_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(energy_market_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(energy_market_kg)\n",
"\n",
@@ -438,4 +439,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/renewable_energy/03_Grid_Management.ipynb
+3 -2
View File
@@ -326,7 +326,8 @@
" min_frequency=1\n",
")\n",
"\n",
"centrality_scores = centrality_calculator.calculate_centrality(grid_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(grid_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(grid_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(grid_kg)\n",
"\n",
@@ -466,4 +467,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/supply_chain/01_Supply_Chain_Data_Integration.ipynb
+3 -2
View File
@@ -418,7 +418,8 @@
"connectivity = connectivity_analyzer.analyze_connectivity(supply_chain_kg)\n",
"\n",
"# Calculate graph metrics\n",
"centrality_scores = centrality_calculator.calculate_centrality(supply_chain_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(supply_chain_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(supply_chain_kg)\n",
"\n",
"print(f\" Entities: {len(supply_chain_kg.get('entities', []))}\")\n",
@@ -565,4 +566,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb
+3 -2
View File
@@ -257,7 +257,8 @@
" min_frequency=1\n",
")\n",
"\n",
"centrality_scores = centrality_calculator.calculate_centrality(market_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(market_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(market_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(market_kg)\n",
"\n",
@@ -363,4 +364,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb
+3 -2
View File
@@ -265,7 +265,8 @@
" \"timestamp\": position.get(\"timestamp\", \"\")\n",
" })\n",
"\n",
"centrality_scores = centrality_calculator.calculate_centrality(trading_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(trading_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(trading_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(trading_kg)\n",
"\n",
@@ -405,4 +406,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb
+3 -2
View File
@@ -316,7 +316,8 @@
"explanation_generator = ExplanationGenerator()\n",
"\n",
"# Analyze graph structure\n",
"centrality_scores = centrality_calculator.calculate_centrality(historical_kg, measure=\"degree\")\n",
"centrality_result = centrality_calculator.calculate_degree_centrality(historical_kg)\n",
"centrality_scores = centrality_result.get('centrality', {})\n",
"communities = community_detector.detect_communities(historical_kg)\n",
"connectivity = connectivity_analyzer.analyze_connectivity(historical_kg)\n",
"\n",
@@ -417,4 +418,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
docs/reference/kg.md
+19 -23
View File
@@ -24,13 +24,7 @@
---
Deduplicate entities using fuzzy matching and semantic similarity
- :material-alert-decagram:{ .lg .middle } **Conflict Detection**
---
Detect and resolve contradicting facts from multiple sources
Resolve entities using fuzzy matching and semantic similarity
- :material-chart-network:{ .lg .middle } **Graph Analytics**
@@ -48,10 +42,14 @@
!!! tip "When to Use"
- **KG Building**: The primary module for assembling a KG from extracted data
- **Data Cleaning**: Merging duplicates and resolving conflicts
- **Entity Resolution**: Resolving and merging similar entities
- **Analysis**: Understanding the structure and importance of nodes
- **Time-Series**: Modeling how the graph evolves over time
!!! note "Related Modules"
- **Conflict Detection**: Use `semantica.conflicts` module for conflict detection and resolution
- **Deduplication**: Use `semantica.deduplication` module for advanced deduplication
---
## ⚙️ Algorithms Used
@@ -83,7 +81,7 @@ Constructs the KG from raw data.
| Method | Description |
|--------|-------------|
| `build(sources)` | Build graph from inputs |
| `merge_entities()` | Run deduplication |
| `merge_entities()` | Merge duplicate entities during building |
**Example:**
@@ -118,25 +116,19 @@ Queries time-aware graphs.
---
## Convenience Functions
## Using Classes
```python
<<<<<<< HEAD
from semantica.kg import GraphBuilder, analyze_graph
from semantica.kg import GraphBuilder, GraphAnalyzer
# Build using GraphBuilder
builder = GraphBuilder(resolve_conflicts=True)
builder = GraphBuilder(merge_entities=True)
kg = builder.build(sources)
=======
from semantica.kg import build, analyze_graph
# Build
kg = build(sources, resolve_conflicts=True)
>>>>>>> origin/main
# Analyze
stats = analyze_graph(kg)
print(f"Communities: {stats['communities']}")
analyzer = GraphAnalyzer()
stats = analyzer.analyze_graph(kg)
print(f"Communities: {stats.get('communities', [])}")
```
---
@@ -148,7 +140,7 @@ print(f"Communities: {stats['communities']}")
```bash
export KG_MERGE_STRATEGY=fuzzy
export KG_TEMPORAL_GRANULARITY=day
export KG_CONFLICT_RESOLUTION=confidence
export KG_RESOLUTION_STRATEGY=fuzzy
```
### YAML Configuration
@@ -191,10 +183,12 @@ print(f"New nodes since 2020: {len(diff.nodes)}")
## Best Practices
1. **Clean Data First**: Use `EntityResolver` aggressively to prevent "entity explosion" (too many duplicate nodes).
1. **Clean Data First**: Use `EntityResolver` to resolve similar entities and prevent "entity explosion" (too many duplicate nodes).
2. **Use Provenance**: Always track sources (`track_history=True`) to debug where bad data came from.
3. **Temporal Granularity**: Choose the right granularity (Day vs Second) to balance performance and precision.
4. **Validate**: Run `GraphValidator` after building to ensure structural integrity.
5. **Deduplication**: Use `semantica.deduplication` module for advanced deduplication needs.
6. **Conflict Resolution**: Use `semantica.conflicts` module for conflict detection and resolution.
---
@@ -203,3 +197,5 @@ print(f"New nodes since 2020: {len(diff.nodes)}")
- [Graph Store Module](graph_store.md) - Persistence layer
- [Semantic Extract Module](semantic_extract.md) - Data source
- [Visualization Module](visualization.md) - Visualizing the KG
- [Conflicts Module](conflicts.md) - Conflict detection and resolution
- [Deduplication Module](deduplication.md) - Advanced deduplication
fix_remaining_notebooks.py
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
"""Fix remaining notebooks with calculate_centrality calls"""
import json
import os
# List of notebooks to fix
notebooks = [
"cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb",
"cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb",
"cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb",
"cookbook/use_cases/healthcare/02_Disease_Network_Analysis.ipynb",
"cookbook/use_cases/healthcare/03_Drug_Interactions_Analysis.ipynb",
"cookbook/use_cases/renewable_energy/01_Energy_Market_Analysis.ipynb",
"cookbook/use_cases/renewable_energy/03_Grid_Management.ipynb",
"cookbook/use_cases/finance/01_Financial_Data_Integration.ipynb",
"cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb",
"cookbook/use_cases/cybersecurity/01_Anomaly_Detection_Real_Time.ipynb",
"cookbook/use_cases/cybersecurity/02_Incident_Analysis.ipynb",
"cookbook/use_cases/supply_chain/01_Supply_Chain_Data_Integration.ipynb",
]
old_pattern = 'calculate_centrality('
new_code = [
'centrality_result = centrality_calculator.calculate_degree_centrality(',
"centrality_scores = centrality_result.get('centrality', {})"
]
for notebook_path in notebooks:
if not os.path.exists(notebook_path):
print(f"Skipping {notebook_path} - file not found")
continue
print(f"Processing {notebook_path}...")
with open(notebook_path, 'r', encoding='utf-8') as f:
nb = json.load(f)
modified = False
for cell in nb['cells']:
if cell.get('cell_type') == 'code':
source = cell.get('source', [])
if isinstance(source, list):
# Check if any line contains the pattern
for i, line in enumerate(source):
if old_pattern in line and 'measure="degree"' in line:
# Extract the graph variable name
if 'market_kg' in line:
graph_var = 'market_kg'
elif 'trading_kg' in line:
graph_var = 'trading_kg'
elif 'historical_kg' in line:
graph_var = 'historical_kg'
elif 'disease_kg' in line:
graph_var = 'disease_kg'
elif 'drug_kg' in line:
graph_var = 'drug_kg'
elif 'energy_market_kg' in line:
graph_var = 'energy_market_kg'
elif 'grid_kg' in line:
graph_var = 'grid_kg'
elif 'financial_kg' in line:
graph_var = 'financial_kg'
elif 'temporal_kg' in line:
graph_var = 'temporal_kg'
elif 'incident_kg' in line:
graph_var = 'incident_kg'
elif 'supply_chain_kg' in line:
graph_var = 'supply_chain_kg'
else:
print(f" Warning: Could not identify graph variable in line: {line}")
continue
# Replace the line
new_lines = [
f"centrality_result = centrality_calculator.calculate_degree_centrality({graph_var})\n",
f"centrality_scores = centrality_result.get('centrality', {{}})\n"
]
cell['source'][i:i+1] = new_lines
modified = True
print(f" Fixed calculate_centrality call with {graph_var}")
break
if modified:
with open(notebook_path, 'w', encoding='utf-8') as f:
json.dump(nb, f, indent=1, ensure_ascii=False)
print(f" ✓ Updated {notebook_path}")
else:
print(f" - No changes needed in {notebook_path}")
print("\nDone!")
semantica/kg/__init__.py
+19 -143
View File
@@ -3,16 +3,16 @@ Knowledge Graph Management Module
This module provides comprehensive knowledge graph construction and management capabilities
for the Semantica framework, including temporal knowledge graph support for time-aware
knowledge representation, graph analytics, entity resolution, conflict detection, and
provenance tracking.
knowledge representation, graph analytics, entity resolution, and provenance tracking.
Note: For conflict detection and resolution, use the semantica.conflicts module.
For deduplication, use the semantica.deduplication module.
Algorithms Used:
Knowledge Graph Construction:
- Graph Building: Entity-relationship graph construction from multiple sources
- Entity Resolution: Fuzzy string matching, exact matching, semantic similarity matching for duplicate detection
- Conflict Detection: Value conflict detection (same entity with different property values), relationship conflict detection
- Conflict Resolution: Highest confidence strategy, source-based resolution
- Temporal Graph Support: Time-aware edge creation with valid_from/valid_until timestamps
- Temporal Granularity: Time normalization (second, minute, hour, day, week, month, year)
- Entity Merging: Property aggregation, metadata merging, provenance tracking
@@ -39,12 +39,6 @@ Entity Resolution:
- Entity Merging: Property conflict resolution, metadata aggregation
- ID Normalization: Canonical ID assignment for merged entities
Conflict Detection:
- Value Conflict Detection: Property value comparison, unique value set extraction
- Relationship Conflict Detection: Relationship property comparison, conflict identification
- Source Tracking: Multi-source conflict tracking, provenance-based conflict resolution
- Conflict Categorization: Value conflicts vs relationship conflicts
Graph Validation:
- Entity Validation: Required field checking (ID, type), unique ID verification
- Relationship Validation: Source/target reference validation, required field checking
@@ -60,11 +54,6 @@ Temporal Operations:
- Temporal Path Finding: BFS with temporal validity constraints
- Version Management: Snapshot creation, version comparison, timestamp-based versioning
Deduplication:
- Duplicate Group Detection: Similarity-based clustering, threshold-based grouping
- Entity Merging: Property aggregation strategies, metadata merging
- Provenance Tracking: Source tracking for merged entities, lineage maintenance
Provenance Tracking:
- Source Tracking: Multi-source entity tracking, timestamp recording
- Lineage Retrieval: Complete provenance history reconstruction
@@ -79,8 +68,7 @@ Seed Management:
Key Features:
- Knowledge graph construction from multiple sources
- Temporal knowledge graph support with time-aware edges
- Entity resolution and deduplication
- Conflict detection and resolution
- Entity resolution
- Comprehensive graph analytics (centrality, communities, connectivity)
- Graph validation and consistency checking
- Temporal queries and pattern detection
@@ -95,75 +83,44 @@ Main Classes:
- TemporalGraphQuery: Time-aware graph querying
- TemporalPatternDetector: Temporal pattern detection
- TemporalVersionManager: Temporal versioning and snapshots
- ConflictDetector: Conflict detection and resolution
- ProvenanceTracker: Provenance tracking and management
- CentralityCalculator: Centrality measures calculation
- CommunityDetector: Community detection
- ConnectivityAnalyzer: Connectivity analysis
- Deduplicator: Graph deduplication
- GraphValidator: Graph validation
- SeedManager: Seed data management
- MethodRegistry: Registry for custom KG methods
- KGConfig: Configuration manager for KG module
Convenience Functions:
- build: Build knowledge graph from sources
- build_kg: Knowledge graph building wrapper
- analyze_graph: Graph analysis wrapper
- resolve_entities: Entity resolution wrapper
- validate_graph: Graph validation wrapper
- detect_conflicts: Conflict detection wrapper
- calculate_centrality: Centrality calculation wrapper
- detect_communities: Community detection wrapper
- analyze_connectivity: Connectivity analysis wrapper
- deduplicate_graph: Deduplication wrapper
- query_temporal: Temporal query wrapper
- get_kg_method: Get KG method by name
- list_available_methods: List registered methods
Global Instances:
- method_registry: Global MethodRegistry instance for registering custom methods
- kg_config: Global KGConfig instance for configuration management
Example Usage:
>>> from semantica.kg import build, build_kg, analyze_graph, calculate_centrality
>>> # Using convenience function
>>> kg = build(sources=[{"entities": [...], "relationships": [...]}])
>>> # Using method functions
>>> kg = build_kg(sources, method="default")
>>> analysis = analyze_graph(kg, method="default")
>>> centrality = calculate_centrality(kg, method="degree")
>>> # Using classes directly
>>> from semantica.kg import GraphBuilder
>>> builder = GraphBuilder(merge_entities=True, resolve_conflicts=True)
>>> graph = builder.build(sources)
>>> from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator
>>> # Build knowledge graph
>>> builder = GraphBuilder(merge_entities=True)
>>> kg = builder.build(sources=[{"entities": [...], "relationships": [...]}])
>>> # Analyze graph
>>> analyzer = GraphAnalyzer()
>>> analysis = analyzer.analyze_graph(kg)
>>> # Calculate centrality
>>> centrality_calc = CentralityCalculator()
>>> degree_centrality = centrality_calc.calculate_degree_centrality(kg)
Author: Semantica Contributors
License: MIT
"""
from typing import Any, Dict, List, Optional, Union
from .centrality_calculator import CentralityCalculator
from .community_detector import CommunityDetector
from .config import KGConfig, kg_config
from .conflict_detector import ConflictDetector
from .connectivity_analyzer import ConnectivityAnalyzer
from .deduplicator import Deduplicator
from .entity_resolver import EntityResolver
from .graph_analyzer import GraphAnalyzer
from .graph_builder import GraphBuilder
from .graph_validator import GraphValidator
from .methods import (
analyze_connectivity,
analyze_graph,
build_kg,
calculate_centrality,
deduplicate_graph,
detect_communities,
detect_conflicts,
get_kg_method,
list_available_methods,
query_temporal,
resolve_entities,
validate_graph,
)
from .provenance_tracker import ProvenanceTracker
from .registry import MethodRegistry, method_registry
from .seed_manager import SeedManager
@@ -181,96 +138,15 @@ __all__ = [
"TemporalGraphQuery",
"TemporalPatternDetector",
"TemporalVersionManager",
"ConflictDetector",
"ProvenanceTracker",
"CentralityCalculator",
"CommunityDetector",
"ConnectivityAnalyzer",
"Deduplicator",
"GraphValidator",
"SeedManager",
# Registry and Methods
# Registry and Configuration
"MethodRegistry",
"method_registry",
"build_kg",
"analyze_graph",
"resolve_entities",
"validate_graph",
"detect_conflicts",
"calculate_centrality",
"detect_communities",
"analyze_connectivity",
"deduplicate_graph",
"query_temporal",
"get_kg_method",
"list_available_methods",
# Configuration
"KGConfig",
"kg_config",
]
def build(
sources: Union[List[Any], Any],
merge_entities: bool = True,
entity_resolution_strategy: str = "fuzzy",
resolve_conflicts: bool = True,
enable_temporal: bool = False,
temporal_granularity: str = "day",
track_history: bool = False,
version_snapshots: bool = False,
entity_resolver: Optional[EntityResolver] = None,
**options,
) -> Dict[str, Any]:
"""
Build knowledge graph from sources (module-level convenience function).
This is a user-friendly wrapper around GraphBuilder.build() that creates
a GraphBuilder instance and builds the knowledge graph.
Args:
sources: List of sources (documents, entities, relationships, or dicts with entities/relationships)
merge_entities: Whether to merge duplicate entities (default: True)
entity_resolution_strategy: Strategy for entity resolution - "fuzzy", "exact", "semantic" (default: "fuzzy")
resolve_conflicts: Whether to resolve conflicts (default: True)
enable_temporal: Enable temporal knowledge graph features (default: False)
temporal_granularity: Time granularity - "second", "minute", "hour", "day", etc. (default: "day")
track_history: Track historical changes (default: False)
version_snapshots: Create version snapshots (default: False)
entity_resolver: Optional EntityResolver instance (default: None, creates one if needed)
**options: Additional build options
Returns:
Dictionary containing:
- entities: List of entities
- relationships: List of relationships
- metadata: Graph metadata including counts and timestamps
Examples:
>>> import semantica
>>> result = semantica.kg.build(
... sources=[{"entities": [...], "relationships": [...]}],
... merge_entities=True,
... resolve_conflicts=True
... )
>>> print(f"Built graph with {result['metadata']['num_entities']} entities")
"""
# Normalize sources to list
if not isinstance(sources, list):
sources = [sources]
# Create GraphBuilder instance
graph_builder = GraphBuilder(
merge_entities=merge_entities,
entity_resolution_strategy=entity_resolution_strategy,
resolve_conflicts=resolve_conflicts,
enable_temporal=enable_temporal,
temporal_granularity=temporal_granularity,
track_history=track_history,
version_snapshots=version_snapshots,
**options,
)
# Build knowledge graph
graph = graph_builder.build(sources, entity_resolver=entity_resolver, **options)
return graph
semantica/kg/conflict_detector.py
-304
View File
@@ -1,304 +0,0 @@
"""
Conflict Detection Module
This module provides comprehensive conflict identification and resolution
capabilities for the Semantica framework, enabling detection of inconsistencies
in knowledge graphs.
Key Features:
- Value conflict detection (conflicting property values for same entity)
- Relationship conflict detection (conflicting relationship properties)
- Conflict resolution with multiple strategies
- Source tracking for conflicts
- Conflict categorization and prioritization
Main Classes:
- ConflictDetector: Main conflict detection and resolution engine
Example Usage:
>>> from semantica.kg import ConflictDetector
>>> detector = ConflictDetector()
>>> conflicts = detector.detect_conflicts(knowledge_graph)
>>> resolution = detector.resolve_conflicts(conflicts)
Author: Semantica Contributors
License: MIT
"""
from typing import Any, Dict, List, Optional
from ..conflicts.conflict_detector import Conflict
from ..conflicts.conflict_detector import ConflictDetector as BaseConflictDetector
from ..conflicts.conflict_resolver import ConflictResolver
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
class ConflictDetector:
"""
Conflict detection and resolution engine.
This class identifies conflicts and inconsistencies in knowledge graphs,
including value conflicts (same entity with different property values)
and relationship conflicts. Provides conflict resolution capabilities.
Features:
- Entity property value conflict detection
- Relationship property conflict detection
- Conflict resolution with configurable strategies
- Source tracking for conflict origins
Example Usage:
>>> detector = ConflictDetector()
>>> conflicts = detector.detect_conflicts(knowledge_graph)
>>> resolution = detector.resolve_conflicts(conflicts)
"""
def __init__(self, **config):
"""
Initialize conflict detector.
Sets up the detector with base conflict detection and resolution
components from the conflicts module.
Args:
**config: Configuration options:
- detection: Configuration for conflict detection (optional)
- resolution: Configuration for conflict resolution (optional)
"""
self.logger = get_logger("conflict_detector")
self.config = config
# Initialize progress tracker
self.progress_tracker = get_progress_tracker()
# Initialize conflict detection components
self.base_detector = BaseConflictDetector(**config.get("detection", {}))
self.resolver = ConflictResolver(**config.get("resolution", {}))
self.logger.debug("Conflict detector initialized")
def detect_conflicts(self, knowledge_graph: Any) -> List[Dict[str, Any]]:
"""
Detect conflicts in knowledge graph.
This method identifies conflicts in the knowledge graph, including:
- Value conflicts: Same entity with different values for the same property
- Relationship conflicts: Same relationship with conflicting properties
Args:
knowledge_graph: Knowledge graph instance (object with entities/relationships
attributes, or dict with "entities" and "relationships" keys)
Returns:
list: List of conflict dictionaries, each containing:
- entity_id or relationship: Identifier of conflicted element
- property: Property name with conflict
- conflicting_values: List of conflicting values
- type: Conflict type ("value_conflict" or "relationship_conflict")
- sources: List of source identifiers for conflicting values
"""
self.logger.info("Detecting conflicts in knowledge graph")
# Extract entities and relationships from graph
entities = []
relationships = []
if hasattr(knowledge_graph, "entities"):
entities = knowledge_graph.entities
elif hasattr(knowledge_graph, "get_entities"):
entities = knowledge_graph.get_entities()
elif isinstance(knowledge_graph, dict):
entities = knowledge_graph.get("entities", [])
relationships = knowledge_graph.get("relationships", [])
if hasattr(knowledge_graph, "relationships"):
relationships = knowledge_graph.relationships
elif hasattr(knowledge_graph, "get_relationships"):
relationships = knowledge_graph.get_relationships()
# Track conflict detection
tracking_id = self.progress_tracker.start_tracking(
file=None,
module="kg",
submodule="ConflictDetector",
message="Detecting conflicts",
)
try:
conflicts = []
self.progress_tracker.update_tracking(
tracking_id, message="Detecting value conflicts..."
)
# Detect value conflicts
entity_properties = {}
for entity in entities:
entity_id = entity.get("id") or entity.get("entity_id")
if not entity_id:
continue
for prop_name, prop_value in entity.items():
if prop_name in ["id", "entity_id", "type", "source"]:
continue
if entity_id not in entity_properties:
entity_properties[entity_id] = {}
if prop_name not in entity_properties[entity_id]:
entity_properties[entity_id][prop_name] = []
entity_properties[entity_id][prop_name].append(
{"value": prop_value, "entity": entity}
)
# Check for conflicts
for entity_id, properties in entity_properties.items():
for prop_name, values in properties.items():
unique_values = {
str(v["value"]) for v in values if v["value"] is not None
}
if len(unique_values) > 1:
conflicts.append(
{
"entity_id": entity_id,
"property": prop_name,
"conflicting_values": list(unique_values),
"type": "value_conflict",
"sources": [
v["entity"].get("source", "unknown") for v in values
],
}
)
self.progress_tracker.update_tracking(
tracking_id, message="Detecting relationship conflicts..."
)
# Detect relationship conflicts
relationship_map = {}
for rel in relationships:
source = rel.get("source") or rel.get("subject")
target = rel.get("target") or rel.get("object")
rel_type = rel.get("type") or rel.get("predicate")
key = f"{source}::{rel_type}::{target}"
if key not in relationship_map:
relationship_map[key] = []
relationship_map[key].append(rel)
# Check for relationship conflicts
for key, rels in relationship_map.items():
if len(rels) > 1:
# Check for conflicting properties
properties = {}
for rel in rels:
for prop_name, prop_value in rel.items():
if prop_name in [
"source",
"target",
"subject",
"object",
"type",
"predicate",
]:
continue
if prop_name not in properties:
properties[prop_name] = []
properties[prop_name].append(prop_value)
for prop_name, values in properties.items():
unique_values = {str(v) for v in values if v is not None}
if len(unique_values) > 1:
conflicts.append(
{
"relationship": key,
"property": prop_name,
"conflicting_values": list(unique_values),
"type": "relationship_conflict",
"sources": [
rel.get("source", "unknown") for rel in rels
],
}
)
self.logger.info(f"Detected {len(conflicts)} conflicts")
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
message=f"Detected {len(conflicts)} conflicts",
)
return conflicts
except Exception as e:
self.progress_tracker.stop_tracking(
tracking_id, status="failed", message=str(e)
)
raise
def resolve_conflicts(
self, conflicts: List[Dict[str, Any]], strategy: str = "highest_confidence"
) -> Dict[str, Any]:
"""
Resolve detected conflicts.
This method attempts to resolve conflicts using the configured conflict
resolver with the specified strategy. Converts conflict dictionaries
to Conflict objects for resolution.
Args:
conflicts: List of conflict dictionaries from detect_conflicts()
strategy: Resolution strategy to use (default: "highest_confidence")
Returns:
dict: Resolution results containing:
- resolved: List of successfully resolved conflicts with resolutions
- unresolved: List of conflicts that could not be resolved
- total: Total number of conflicts
- resolved_count: Number of resolved conflicts
- unresolved_count: Number of unresolved conflicts
"""
self.logger.info(f"Resolving {len(conflicts)} conflicts")
resolved = []
unresolved = []
for conflict in conflicts:
try:
# Convert to Conflict object for resolver
conflict_obj = Conflict(
conflict_id=conflict.get("entity_id")
or conflict.get("relationship", "unknown"),
conflict_type=conflict.get("type", "value_conflict"),
entity_id=conflict.get("entity_id"),
property_name=conflict.get("property"),
conflicting_values=conflict.get("conflicting_values", []),
sources=[{"source": s} for s in conflict.get("sources", [])],
)
# Resolve conflict
resolution = self.resolver.resolve_conflict(
conflict_obj, strategy=strategy
)
if resolution.resolved:
resolved.append(
{
"conflict": conflict,
"resolution": resolution.resolved_value,
"strategy": resolution.resolution_strategy,
}
)
else:
unresolved.append(conflict)
except Exception as e:
self.logger.error(f"Error resolving conflict: {e}")
unresolved.append(conflict)
return {
"resolved": resolved,
"unresolved": unresolved,
"total": len(conflicts),
"resolved_count": len(resolved),
"unresolved_count": len(unresolved),
}
semantica/kg/deduplicator.py
-175
View File
@@ -1,175 +0,0 @@
"""
Deduplication Module
This module provides comprehensive duplicate detection and merging capabilities
for the Semantica framework, enabling identification and resolution of duplicate
entities and relationships in knowledge graphs.
Key Features:
- Duplicate entity detection using similarity metrics
- Duplicate group identification
- Entity merging with configurable strategies
- Relationship deduplication
- Provenance tracking for merged entities
Main Classes:
- Deduplicator: Main deduplication engine
Example Usage:
>>> from semantica.kg import Deduplicator
>>> deduplicator = Deduplicator()
>>> duplicate_groups = deduplicator.find_duplicates(entities)
>>> merged_entities = deduplicator.merge_duplicates(duplicate_groups)
Author: Semantica Contributors
License: MIT
"""
from typing import Any, Dict, List, Optional
from ..deduplication.duplicate_detector import DuplicateDetector, DuplicateGroup
from ..deduplication.entity_merger import EntityMerger
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
class Deduplicator:
"""
Deduplication engine.
This class provides duplicate detection and merging capabilities for
knowledge graphs, using the deduplication module's duplicate detector
and entity merger components.
Features:
- Duplicate entity detection with similarity metrics
- Duplicate group identification
- Entity merging with configurable strategies
- Provenance tracking for merged entities
Example Usage:
>>> deduplicator = Deduplicator()
>>> duplicate_groups = deduplicator.find_duplicates(entities)
>>> merged_entities = deduplicator.merge_duplicates(duplicate_groups)
"""
def __init__(self, **config):
"""
Initialize deduplicator.
Sets up the deduplicator with duplicate detector and entity merger
components from the deduplication module.
Args:
**config: Configuration options:
- detection: Configuration for duplicate detection (optional)
- merger: Configuration for entity merging (optional)
"""
self.logger = get_logger("deduplicator")
self.config = config
# Initialize progress tracker
self.progress_tracker = get_progress_tracker()
# Initialize deduplication components
self.duplicate_detector = DuplicateDetector(**config.get("detection", {}))
self.entity_merger = EntityMerger(**config.get("merger", {}))
self.logger.debug("Deduplicator initialized")
def find_duplicates(
self, entities: List[Dict[str, Any]]
) -> List[List[Dict[str, Any]]]:
"""
Find duplicate entities.
This method detects duplicate entities using similarity metrics and
groups them into duplicate groups. Uses the duplicate detector from
the deduplication module.
Args:
entities: List of entity dictionaries to check for duplicates
Returns:
list: List of duplicate groups, where each group is a list of
duplicate entity dictionaries (groups with 2+ entities)
"""
self.logger.info(f"Finding duplicates in {len(entities)} entities")
# Track deduplication
tracking_id = self.progress_tracker.start_tracking(
file=None,
module="kg",
submodule="Deduplicator",
message="Finding duplicates",
)
try:
# Detect duplicate groups
duplicate_groups = self.duplicate_detector.detect_duplicate_groups(
entities, **self.config
)
# Convert to list of lists
result = []
for group in duplicate_groups:
if len(group.entities) >= 2:
result.append(group.entities)
self.logger.info(f"Found {len(result)} duplicate groups")
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
message=f"Found {len(result)} duplicate groups",
)
return result
except Exception as e:
self.progress_tracker.stop_tracking(
tracking_id, status="failed", message=str(e)
)
raise
def merge_duplicates(
self, duplicate_groups: List[List[Dict[str, Any]]]
) -> List[Dict[str, Any]]:
"""
Merge duplicate entities.
This method merges groups of duplicate entities using the entity merger
from the deduplication module. Each group is merged into a single entity
with merged properties and provenance tracking.
Args:
duplicate_groups: List of duplicate groups (each group is a list
of duplicate entity dictionaries)
Returns:
list: List of merged entity dictionaries (one per duplicate group)
"""
self.logger.info(f"Merging {len(duplicate_groups)} duplicate groups")
merged_entities = []
processed_ids = set()
for group in duplicate_groups:
if len(group) < 2:
continue
# Merge the group
merge_operations = self.entity_merger.merge_duplicates(group, **self.config)
for operation in merge_operations:
merged_entity = operation.merged_entity
merged_entities.append(merged_entity)
# Mark source entities as processed
for source_entity in operation.source_entities:
entity_id = source_entity.get("id") or source_entity.get(
"entity_id"
)
if entity_id:
processed_ids.add(entity_id)
self.logger.info(f"Merged to {len(merged_entities)} entities")
return merged_entities
semantica/kg/kg_usage.md
+212 -399
View File
File diff suppressed because it is too large Load Diff
semantica/kg/methods.py
+3 -123
View File
@@ -133,15 +133,16 @@ Main Functions:
- analyze_graph: Graph analysis wrapper
- resolve_entities: Entity resolution wrapper
- validate_graph: Graph validation wrapper
- detect_conflicts: Conflict detection wrapper
- calculate_centrality: Centrality calculation wrapper
- detect_communities: Community detection wrapper
- analyze_connectivity: Connectivity analysis wrapper
- deduplicate_graph: Deduplication wrapper
- query_temporal: Temporal query wrapper
- get_kg_method: Get KG method by name
- list_available_methods: List registered methods
Note: Conflict detection and deduplication have been moved to dedicated modules.
Use semantica.conflicts for conflict detection and semantica.deduplication for deduplication.
Example Usage:
>>> from semantica.kg.methods import build_kg, analyze_graph, calculate_centrality
>>> kg = build_kg(sources, method="default")
@@ -156,9 +157,7 @@ from ..utils.logging import get_logger
from .centrality_calculator import CentralityCalculator
from .community_detector import CommunityDetector
from .config import kg_config
from .conflict_detector import ConflictDetector
from .connectivity_analyzer import ConnectivityAnalyzer
from .deduplicator import Deduplicator
from .entity_resolver import EntityResolver
from .graph_analyzer import GraphAnalyzer
from .graph_builder import GraphBuilder
@@ -365,53 +364,6 @@ def validate_graph(graph: Dict[str, Any], method: str = "default", **kwargs) ->
raise
def detect_conflicts(
graph: Dict[str, Any], method: str = "default", **kwargs
) -> List[Dict[str, Any]]:
"""
Detect conflicts in knowledge graph (convenience function).
This is a user-friendly wrapper that detects conflicts using the specified method.
Args:
graph: Knowledge graph to analyze
method: Detection method (default: "default")
- "default": Comprehensive conflict detection
- "value": Value conflict detection only
- "relationship": Relationship conflict detection only
**kwargs: Additional options passed to ConflictDetector
Returns:
List of conflict dictionaries
Examples:
>>> from semantica.kg.methods import detect_conflicts
>>> conflicts = detect_conflicts(kg, method="default")
>>> value_conflicts = detect_conflicts(kg, method="value")
"""
# Check for custom method in registry
custom_method = method_registry.get("conflict", method)
if custom_method:
try:
return custom_method(graph, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
try:
# Get config
config = kg_config.get_method_config("conflict")
config.update(kwargs)
detector = ConflictDetector(**config)
return detector.detect_conflicts(graph)
except Exception as e:
logger.error(f"Failed to detect conflicts: {e}")
raise
def calculate_centrality(
graph: Dict[str, Any], method: str = "degree", **kwargs
) -> Dict[str, Any]:
@@ -582,78 +534,6 @@ def analyze_connectivity(
raise
def deduplicate_graph(
graph: Dict[str, Any], method: str = "default", **kwargs
) -> Dict[str, Any]:
"""
Deduplicate knowledge graph (convenience function).
This is a user-friendly wrapper that deduplicates a knowledge graph using the specified method.
Args:
graph: Knowledge graph to deduplicate
method: Deduplication method (default: "default")
- "default": Default deduplication
- "entities": Entity deduplication only
- "relationships": Relationship deduplication only
**kwargs: Additional options passed to Deduplicator
Returns:
Dictionary containing deduplicated graph
Examples:
>>> from semantica.kg.methods import deduplicate_graph
>>> deduplicated = deduplicate_graph(kg, method="default")
>>> entity_dedup = deduplicate_graph(kg, method="entities")
"""
# Check for custom method in registry
custom_method = method_registry.get("deduplicate", method)
if custom_method:
try:
return custom_method(graph, **kwargs)
except Exception as e:
logger.warning(
f"Custom method {method} failed: {e}, falling back to default"
)
try:
# Get config
config = kg_config.get_method_config("deduplicate")
config.update(kwargs)
deduplicator = Deduplicator(**config)
entities = graph.get("entities", [])
relationships = graph.get("relationships", [])
if method == "entities" or method == "default":
duplicate_groups = deduplicator.find_duplicates(entities)
merged_entities = deduplicator.merge_duplicates(duplicate_groups)
# Add non-duplicate entities (entities not in any duplicate group)
processed_ids = set()
for group in duplicate_groups:
for entity in group:
entity_id = entity.get("id") or entity.get("entity_id")
if entity_id:
processed_ids.add(entity_id)
# Add entities that weren't part of any duplicate group
for entity in entities:
entity_id = entity.get("id") or entity.get("entity_id")
if entity_id and entity_id not in processed_ids:
merged_entities.append(entity)
entities = merged_entities
return {
"entities": entities,
"relationships": relationships,
"metadata": graph.get("metadata", {}),
}
except Exception as e:
logger.error(f"Failed to deduplicate graph: {e}")
raise
def query_temporal(
graph: Dict[str, Any], query: str = "", method: str = "time_point", **kwargs
) -> Dict[str, Any]:
semantica/kg/registry.py
+6 -7
View File
@@ -10,13 +10,14 @@ Supported Registration Types:
* "analyze": Graph analysis methods
* "resolve": Entity resolution methods
* "validate": Graph validation methods
* "conflict": Conflict detection methods
* "centrality": Centrality calculation methods
* "community": Community detection methods
* "connectivity": Connectivity analysis methods
* "deduplicate": Deduplication methods
* "temporal": Temporal query methods
Note: Conflict detection and deduplication have been moved to dedicated modules.
Use semantica.conflicts for conflict detection and semantica.deduplication for deduplication.
Algorithms Used:
- Registry Pattern: Dictionary-based registration and lookup
- Dynamic Registration: Runtime function registration
@@ -26,7 +27,7 @@ Algorithms Used:
Key Features:
- Method registry for custom KG methods
- Task-based method organization (build, analyze, resolve, validate, conflict, centrality, community, connectivity, deduplicate, temporal)
- Task-based method organization (build, analyze, resolve, validate, centrality, community, connectivity, temporal)
- Dynamic registration and unregistration
- Easy discovery of available methods
- Support for community-contributed extensions
@@ -54,11 +55,9 @@ class MethodRegistry:
"analyze": {},
"resolve": {},
"validate": {},
"conflict": {},
"centrality": {},
"community": {},
"connectivity": {},
"deduplicate": {},
"temporal": {},
}
@@ -68,7 +67,7 @@ class MethodRegistry:
Register a custom KG method.
Args:
task: Task type ("build", "analyze", "resolve", "validate", "conflict", "centrality", "community", "connectivity", "deduplicate", "temporal")
task: Task type ("build", "analyze", "resolve", "validate", "centrality", "community", "connectivity", "temporal")
name: Method name
method_func: Method function
"""
@@ -82,7 +81,7 @@ class MethodRegistry:
Get method by task and name.
Args:
task: Task type ("build", "analyze", "resolve", "validate", "conflict", "centrality", "community", "connectivity", "deduplicate", "temporal")
task: Task type ("build", "analyze", "resolve", "validate", "centrality", "community", "connectivity", "temporal")
name: Method name
Returns: