mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
refactor(kg): Remove ConflictDetector and Deduplicator from kg module
- Remove ConflictDetector and Deduplicator from semantica.kg module - Update all imports to use semantica.conflicts and semantica.deduplication - Update all notebooks to use class-based API (no convenience functions) - Fix method signatures: pass graph parameter to methods instead of constructor - Update calculate_centrality calls to use specific methods (calculate_degree_centrality, etc.) - Fix detect_communities and analyze_connectivity return value handling - Update all documentation (kg_usage.md, docs/reference/kg.md) - Remove conflict_detector.py and deduplicator.py from kg module - Update registry.py to remove conflict and deduplicate task types
This commit is contained in:
KaifAhmad1
@@ -89,8 +89,10 @@
|
||||
"source": [
|
||||
"centrality_calculator = CentralityCalculator()\n",
|
||||
"\n",
|
||||
"degree_centrality = centrality_calculator.calculate_centrality(kg, measure=\"degree\")\n",
|
||||
"betweenness_centrality = centrality_calculator.calculate_centrality(kg, measure=\"betweenness\")\n",
|
||||
"degree_centrality_result = centrality_calculator.calculate_degree_centrality(kg)\n",
|
||||
"degree_centrality = degree_centrality_result.get('centrality', {})\n",
|
||||
"betweenness_centrality_result = centrality_calculator.calculate_betweenness_centrality(kg)\n",
|
||||
"betweenness_centrality = betweenness_centrality_result.get('centrality', {})\n",
|
||||
"\n",
|
||||
"print(f\"Degree centrality: {len(degree_centrality)} entities\")\n",
|
||||
"print(f\"Betweenness centrality: {len(betweenness_centrality)} entities\")\n"
|
||||
|
||||
@@ -46,7 +46,8 @@
|
||||
"source": [
|
||||
"from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor, FeedIngestor\n",
|
||||
"from semantica.parse import DocumentParser, StructuredDataParser\n",
|
||||
"from semantica.kg import GraphBuilder, EntityResolver, ConflictDetector, ProvenanceTracker\n",
|
||||
"from semantica.kg import GraphBuilder, EntityResolver, ProvenanceTracker\n",
|
||||
"from semantica.conflicts import ConflictDetector\n",
|
||||
"import tempfile\n",
|
||||
"import os\n",
|
||||
"import json\n",
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to build knowledge graphs from entities and relationships using Semantica's graph building modules. You'll learn to use `GraphBuilder`, `EntityResolver`, `GraphValidator`, and `Deduplicator`.\n",
|
||||
"This notebook demonstrates how to build knowledge graphs from entities and relationships using Semantica's graph building modules. You'll learn to use `GraphBuilder`, `EntityResolver`, and `GraphValidator`.\n",
|
||||
"\n",
|
||||
"**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/kg/)\n",
|
||||
"\n",
|
||||
@@ -19,7 +19,7 @@
|
||||
"- Use `GraphBuilder` to construct knowledge graphs\n",
|
||||
"- Use `EntityResolver` to resolve entity conflicts\n",
|
||||
"- Use `GraphValidator` to validate graph structure\n",
|
||||
"- Use `Deduplicator` to remove duplicate entities\n",
|
||||
"**Note**: For deduplication, use the `semantica.deduplication` module.\n",
|
||||
"\n",
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
@@ -145,14 +145,23 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import Deduplicator\n",
|
||||
"from semantica.deduplication import DuplicateDetector, EntityMerger, MergeStrategy\n",
|
||||
"\n",
|
||||
"deduplicator = Deduplicator()\n",
|
||||
"# Detect duplicates\n",
|
||||
"detector = DuplicateDetector(similarity_threshold=0.8)\n",
|
||||
"duplicate_groups = detector.detect_duplicate_groups(knowledge_graph.get('entities', []))\n",
|
||||
"\n",
|
||||
"deduplicated_graph = deduplicator.deduplicate(knowledge_graph)\n",
|
||||
"# Merge duplicates\n",
|
||||
"merger = EntityMerger()\n",
|
||||
"merge_operations = merger.merge_duplicates(\n",
|
||||
" knowledge_graph.get('entities', []),\n",
|
||||
" strategy=MergeStrategy.KEEP_MOST_COMPLETE\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"deduplicated_entities = [op.merged_entity for op in merge_operations]\n",
|
||||
"\n",
|
||||
"print(f\"Original entities: {len(knowledge_graph.get('entities', []))}\")\n",
|
||||
"print(f\"Deduplicated entities: {len(deduplicated_graph.get('entities', []))}\")\n"
|
||||
"print(f\"Deduplicated entities: {len(deduplicated_entities)}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -166,7 +175,7 @@
|
||||
"- **GraphBuilder**: Construct knowledge graphs from entities and relationships\n",
|
||||
"- **EntityResolver**: Resolve entity conflicts and duplicates\n",
|
||||
"- **GraphValidator**: Validate graph structure and quality\n",
|
||||
"- **Deduplicator**: Remove duplicate entities\n",
|
||||
"- **Deduplication**: Use `semantica.deduplication` module for removing duplicate entities\n",
|
||||
"\n",
|
||||
"Next: Learn how to analyze graphs in the Graph_Analytics notebook.\n"
|
||||
]
|
||||
|
||||
@@ -91,7 +91,8 @@
|
||||
"\n",
|
||||
"centrality_calculator = CentralityCalculator()\n",
|
||||
"\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(kg, measure=\"degree\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_degree_centrality(kg)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"\n",
|
||||
"print(f\"Centrality scores:\")\n",
|
||||
"for entity_id, score in list(centrality_scores.items())[:5]:\n",
|
||||
|
||||
@@ -521,27 +521,29 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"graph_analyzer = GraphAnalyzer(knowledge_graph)\n",
|
||||
"centrality_calculator = CentralityCalculator(knowledge_graph)\n",
|
||||
"community_detector = CommunityDetector(knowledge_graph)\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n",
|
||||
"graph_analyzer = GraphAnalyzer()\n",
|
||||
"centrality_calculator = CentralityCalculator()\n",
|
||||
"community_detector = CommunityDetector()\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer()\n",
|
||||
"temporal_query = TemporalGraphQuery(knowledge_graph)\n",
|
||||
"inference_engine = InferenceEngine()\n",
|
||||
"rule_manager = RuleManager()\n",
|
||||
"\n",
|
||||
"# Compute graph metrics\n",
|
||||
"graph_metrics = graph_analyzer.compute_metrics()\n",
|
||||
"graph_metrics = graph_analyzer.compute_metrics(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Calculate centrality\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_betweenness_centrality(knowledge_graph)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"top_central_targets = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n",
|
||||
"\n",
|
||||
"# Detect communities\n",
|
||||
"communities = community_detector.detect_communities()\n",
|
||||
"community_count = len(set(communities.values())) if communities else 0\n",
|
||||
"communities_result = community_detector.detect_communities(knowledge_graph, algorithm=\"louvain\")\n",
|
||||
"communities = communities_result.get('communities', [])\n",
|
||||
"community_count = len(communities) if communities else 0\n",
|
||||
"\n",
|
||||
"# Analyze connectivity\n",
|
||||
"connectivity_results = connectivity_analyzer.analyze_connectivity()\n",
|
||||
"connectivity_results = connectivity_analyzer.analyze_connectivity(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Define interaction prediction rules\n",
|
||||
"prediction_rules = [\n",
|
||||
|
||||
@@ -63,7 +63,7 @@
|
||||
"from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
"from semantica.kg_qa import KGQualityAssessor\n",
|
||||
"from semantica.kg import ConflictDetector\n",
|
||||
"from semantica.conflicts import ConflictDetector\n",
|
||||
"from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
"from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n",
|
||||
"import tempfile\n",
|
||||
@@ -460,26 +460,28 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"graph_analyzer = GraphAnalyzer(knowledge_graph)\n",
|
||||
"centrality_calculator = CentralityCalculator(knowledge_graph)\n",
|
||||
"community_detector = CommunityDetector(knowledge_graph)\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n",
|
||||
"graph_analyzer = GraphAnalyzer()\n",
|
||||
"centrality_calculator = CentralityCalculator()\n",
|
||||
"community_detector = CommunityDetector()\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer()\n",
|
||||
"temporal_query = TemporalGraphQuery(knowledge_graph)\n",
|
||||
"pattern_detector = TemporalPatternDetector(knowledge_graph)\n",
|
||||
"pattern_detector = TemporalPatternDetector()\n",
|
||||
"\n",
|
||||
"# Compute graph metrics\n",
|
||||
"graph_metrics = graph_analyzer.compute_metrics()\n",
|
||||
"graph_metrics = graph_analyzer.compute_metrics(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Calculate centrality\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_betweenness_centrality(knowledge_graph)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"top_central_genes = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n",
|
||||
"\n",
|
||||
"# Detect communities\n",
|
||||
"communities = community_detector.detect_communities()\n",
|
||||
"community_count = len(set(communities.values())) if communities else 0\n",
|
||||
"communities_result = community_detector.detect_communities(knowledge_graph, algorithm=\"louvain\")\n",
|
||||
"communities = communities_result.get('communities', [])\n",
|
||||
"community_count = len(communities) if communities else 0\n",
|
||||
"\n",
|
||||
"# Analyze connectivity\n",
|
||||
"connectivity_results = connectivity_analyzer.analyze_connectivity()\n",
|
||||
"connectivity_results = connectivity_analyzer.analyze_connectivity(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Detect temporal patterns\n",
|
||||
"start_time = (datetime.now() - timedelta(days=7)).isoformat()\n",
|
||||
@@ -491,6 +493,7 @@
|
||||
")\n",
|
||||
"\n",
|
||||
"temporal_patterns = pattern_detector.detect_temporal_patterns(\n",
|
||||
" knowledge_graph,\n",
|
||||
" relationship_types=[\"associated_with\"],\n",
|
||||
" time_window_hours=168\n",
|
||||
")\n",
|
||||
@@ -650,13 +653,13 @@
|
||||
"owl_exporter = OWLExporter()\n",
|
||||
"report_generator = ReportGenerator()\n",
|
||||
"kg_quality_assessor = KGQualityAssessor()\n",
|
||||
"conflict_detector = ConflictDetector(knowledge_graph)\n",
|
||||
"conflict_detector = ConflictDetector()\n",
|
||||
"\n",
|
||||
"# Assess graph quality\n",
|
||||
"quality_metrics = kg_quality_assessor.assess_quality(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Detect conflicts\n",
|
||||
"conflicts = conflict_detector.detect_conflicts()\n",
|
||||
"conflicts = conflict_detector.detect_conflicts(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Export knowledge graph\n",
|
||||
"kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"genomic_kg.json\"))\n",
|
||||
|
||||
@@ -392,29 +392,32 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"graph_analyzer = GraphAnalyzer(knowledge_graph)\n",
|
||||
"centrality_calculator = CentralityCalculator(knowledge_graph)\n",
|
||||
"community_detector = CommunityDetector(knowledge_graph)\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n",
|
||||
"graph_analyzer = GraphAnalyzer()\n",
|
||||
"centrality_calculator = CentralityCalculator()\n",
|
||||
"community_detector = CommunityDetector()\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer()\n",
|
||||
"temporal_query = TemporalGraphQuery(knowledge_graph)\n",
|
||||
"pattern_detector = TemporalPatternDetector(knowledge_graph)\n",
|
||||
"pattern_detector = TemporalPatternDetector()\n",
|
||||
"\n",
|
||||
"# Compute graph metrics\n",
|
||||
"graph_metrics = graph_analyzer.compute_metrics()\n",
|
||||
"graph_metrics = graph_analyzer.compute_metrics(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Calculate centrality\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_betweenness_centrality(knowledge_graph)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"top_central_protocols = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n",
|
||||
"\n",
|
||||
"# Detect communities\n",
|
||||
"communities = community_detector.detect_communities()\n",
|
||||
"community_count = len(set(communities.values())) if communities else 0\n",
|
||||
"communities_result = community_detector.detect_communities(knowledge_graph, algorithm=\"louvain\")\n",
|
||||
"communities = communities_result.get('communities', [])\n",
|
||||
"community_count = len(communities) if communities else 0\n",
|
||||
"\n",
|
||||
"# Analyze connectivity\n",
|
||||
"connectivity_results = connectivity_analyzer.analyze_connectivity()\n",
|
||||
"connectivity_results = connectivity_analyzer.analyze_connectivity(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Detect temporal patterns\n",
|
||||
"temporal_patterns = pattern_detector.detect_temporal_patterns(\n",
|
||||
" knowledge_graph,\n",
|
||||
" relationship_types=[\"has_pool\", \"has_token\"],\n",
|
||||
" time_window_hours=24\n",
|
||||
")\n",
|
||||
|
||||
@@ -448,11 +448,11 @@
|
||||
"aml_results = graph_store.execute_query(aml_query)\n",
|
||||
"\n",
|
||||
"temporal_query = TemporalGraphQuery(knowledge_graph)\n",
|
||||
"pattern_detector = TemporalPatternDetector(knowledge_graph)\n",
|
||||
"graph_analyzer = GraphAnalyzer(knowledge_graph)\n",
|
||||
"centrality_calculator = CentralityCalculator(knowledge_graph)\n",
|
||||
"community_detector = CommunityDetector(knowledge_graph)\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n",
|
||||
"pattern_detector = TemporalPatternDetector()\n",
|
||||
"graph_analyzer = GraphAnalyzer()\n",
|
||||
"centrality_calculator = CentralityCalculator()\n",
|
||||
"community_detector = CommunityDetector()\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer()\n",
|
||||
"\n",
|
||||
"# Query transactions in time range\n",
|
||||
"start_time = (datetime.now() - timedelta(hours=6)).isoformat()\n",
|
||||
@@ -466,20 +466,23 @@
|
||||
"\n",
|
||||
"# Detect temporal patterns\n",
|
||||
"temporal_patterns = pattern_detector.detect_temporal_patterns(\n",
|
||||
" knowledge_graph,\n",
|
||||
" relationship_types=[\"transfers_to\"],\n",
|
||||
" time_window_hours=6\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Calculate centrality to find key wallets\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_betweenness_centrality(knowledge_graph)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"top_central_wallets = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n",
|
||||
"\n",
|
||||
"# Detect communities (clustering)\n",
|
||||
"communities = community_detector.detect_communities()\n",
|
||||
"community_count = len(set(communities.values())) if communities else 0\n",
|
||||
"communities_result = community_detector.detect_communities(knowledge_graph, algorithm=\"louvain\")\n",
|
||||
"communities = communities_result.get('communities', [])\n",
|
||||
"community_count = len(communities) if communities else 0\n",
|
||||
"\n",
|
||||
"# Analyze connectivity\n",
|
||||
"connectivity_results = connectivity_analyzer.analyze_connectivity()\n",
|
||||
"connectivity_results = connectivity_analyzer.analyze_connectivity(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# AML Pattern Detection using Inference Engine\n",
|
||||
"inference_engine = InferenceEngine()\n",
|
||||
@@ -572,13 +575,13 @@
|
||||
"rdf_exporter = RDFExporter()\n",
|
||||
"report_generator = ReportGenerator()\n",
|
||||
"kg_quality_assessor = KGQualityAssessor()\n",
|
||||
"conflict_detector = ConflictDetector(knowledge_graph)\n",
|
||||
"conflict_detector = ConflictDetector()\n",
|
||||
"\n",
|
||||
"# Assess graph quality\n",
|
||||
"quality_metrics = kg_quality_assessor.assess_quality(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Detect conflicts\n",
|
||||
"conflicts = conflict_detector.detect_conflicts()\n",
|
||||
"conflicts = conflict_detector.detect_conflicts(knowledge_graph)\n",
|
||||
"\n",
|
||||
"# Generate alerts\n",
|
||||
"alerts = []\n",
|
||||
|
||||
@@ -59,7 +59,8 @@
|
||||
"from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer, ConnectivityAnalyzer\n",
|
||||
"from semantica.reasoning import InferenceEngine, ExplanationGenerator\n",
|
||||
"from semantica.kg_qa import KGQualityAssessor\n",
|
||||
"from semantica.kg import ProvenanceTracker, ConflictDetector\n",
|
||||
"from semantica.kg import ProvenanceTracker\n",
|
||||
"from semantica.conflicts import ConflictDetector\n",
|
||||
"from semantica.export import RDFExporter, ReportGenerator\n",
|
||||
"from semantica.visualization import AnalyticsVisualizer, TemporalVisualizer\n",
|
||||
"import tempfile\n",
|
||||
|
||||
@@ -264,7 +264,8 @@
|
||||
"impact_kg = builder.build(environmental_entities, environmental_relationships)\n",
|
||||
"\n",
|
||||
"metrics = graph_analyzer.compute_metrics(impact_kg)\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(impact_kg, measure=\"degree\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_degree_centrality(impact_kg)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"communities = community_detector.detect_communities(impact_kg)\n",
|
||||
"\n",
|
||||
"print(f\"Extracted {len(environmental_entities)} environmental entities\")\n",
|
||||
|
||||
@@ -228,7 +228,8 @@
|
||||
"resource_kg = builder.build(resource_entities, resource_relationships)\n",
|
||||
"\n",
|
||||
"metrics = graph_analyzer.compute_metrics(resource_kg)\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(resource_kg, measure=\"degree\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_degree_centrality(resource_kg)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"communities = community_detector.detect_communities(resource_kg)\n",
|
||||
"\n",
|
||||
"print(f\"Extracted {len(resource_entities)} resource entities\")\n",
|
||||
|
||||
@@ -211,7 +211,8 @@
|
||||
"news_kg = builder.build(news_entities, news_relationships)\n",
|
||||
"\n",
|
||||
"metrics = graph_analyzer.compute_metrics(news_kg)\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(news_kg, measure=\"degree\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_degree_centrality(news_kg)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"communities = community_detector.detect_communities(news_kg)\n",
|
||||
"\n",
|
||||
"print(f\"Extracted {len(news_entities)} news entities\")\n",
|
||||
|
||||
@@ -226,7 +226,8 @@
|
||||
"risk_kg = builder.build(risk_entities, risk_relationships)\n",
|
||||
"\n",
|
||||
"metrics = graph_analyzer.compute_metrics(risk_kg)\n",
|
||||
"centrality_scores = centrality_calculator.calculate_centrality(risk_kg, measure=\"degree\")\n",
|
||||
"centrality_result = centrality_calculator.calculate_degree_centrality(risk_kg)\n",
|
||||
"centrality_scores = centrality_result.get('centrality', {})\n",
|
||||
"communities = community_detector.detect_communities(risk_kg)\n",
|
||||
"\n",
|
||||
"print(f\"Extracted {len(risk_entities)} risk entities\")\n",
|
||||
|
||||
Reference in New Issue
Block a user