diff --git a/cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb b/cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb new file mode 100644 index 00000000..bab9b2d3 --- /dev/null +++ b/cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb @@ -0,0 +1,18 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb b/cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb new file mode 100644 index 00000000..28ab2822 --- /dev/null +++ b/cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb @@ -0,0 +1,713 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Drug Discovery Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete drug discovery pipeline: ingest drug and protein data from multiple sources (APIs, databases, feeds), extract compound and target entities, build drug-target knowledge graph, generate embeddings, perform similarity search, predict drug-target interactions, and identify targets.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, DBIngestor, FeedIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery\n", + "- **Embeddings**: EmbeddingGenerator, TextEmbedder\n", + "- **Vector Store**: VectorStore, HybridSearch\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Ontology**: OntologyGenerator, OntologyValidator\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Drug/Protein Data Sources → Parse → Extract Entities (compounds, targets, interactions) → Build Drug-Target KG → Generate Embeddings → Similarity Search → Predict Interactions → Target Identification → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Drug and Protein Data from Multiple Sources\n", + "\n", + "Ingest drug compound and protein target data from APIs, databases, and feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, DBIngestor, FeedIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery\n", + "from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n", + "from semantica.vector_store import VectorStore, HybridSearch\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.ontology import OntologyGenerator, OntologyValidator\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "# Real drug APIs\n", + "drug_apis = [\n", + " \"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/2244/JSON\", # PubChem API\n", + " \"https://www.ebi.ac.uk/chembl/api/data/molecule/CHEMBL25.json\", # ChEMBL API\n", + " \"https://go.drugbank.com/releases/latest\" # DrugBank API\n", + "]\n", + "\n", + "# Real protein APIs\n", + "protein_apis = [\n", + " \"https://www.uniprot.org/uniprot/P04637.json\", # UniProt API\n", + " \"https://www.rcsb.org/pdb/json/descriptors/1A2B\" # PDB API\n", + "]\n", + "\n", + "# Real interaction databases\n", + "interaction_databases = [\n", + " \"STRING\",\n", + " \"BioGRID\"\n", + "]\n", + "\n", + "# Real database connection for compound libraries\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/drug_discovery_db\"\n", + "db_query = \"SELECT compound_id, target_protein, interaction_type, binding_affinity, mechanism FROM drug_target_interactions WHERE binding_affinity < 100 LIMIT 1000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample drug-target data for local ingestion\n", + "drug_data_file = os.path.join(temp_dir, \"drug_targets.json\")\n", + "drug_data = [\n", + " {\n", + " \"compound_id\": \"CID2244\",\n", + " \"compound_name\": \"Aspirin\",\n", + " \"target_protein\": \"PTGS1\",\n", + " \"target_name\": \"Prostaglandin G/H synthase 1\",\n", + " \"interaction_type\": \"inhibitor\",\n", + " \"binding_affinity\": 5.2,\n", + " \"mechanism\": \"Irreversible inhibition\",\n", + " \"pathway\": \"Arachidonic acid metabolism\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat()\n", + " },\n", + " {\n", + " \"compound_id\": \"CID1983\",\n", + " \"compound_name\": \"Ibuprofen\",\n", + " \"target_protein\": \"PTGS2\",\n", + " \"target_name\": \"Prostaglandin G/H synthase 2\",\n", + " \"interaction_type\": \"inhibitor\",\n", + " \"binding_affinity\": 8.5,\n", + " \"mechanism\": \"Reversible inhibition\",\n", + " \"pathway\": \"Arachidonic acid metabolism\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=2)).isoformat()\n", + " },\n", + " {\n", + " \"compound_id\": \"CID1983\",\n", + " \"compound_name\": \"Ibuprofen\",\n", + " \"target_protein\": \"PTGS1\",\n", + " \"target_name\": \"Prostaglandin G/H synthase 1\",\n", + " \"interaction_type\": \"inhibitor\",\n", + " \"binding_affinity\": 12.3,\n", + " \"mechanism\": \"Reversible inhibition\",\n", + " \"pathway\": \"Arachidonic acid metabolism\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=2)).isoformat()\n", + " },\n", + " {\n", + " \"compound_id\": \"CID60823\",\n", + " \"compound_name\": \"Atorvastatin\",\n", + " \"target_protein\": \"HMGCR\",\n", + " \"target_name\": \"3-hydroxy-3-methylglutaryl-coenzyme A reductase\",\n", + " \"interaction_type\": \"inhibitor\",\n", + " \"binding_affinity\": 0.8,\n", + " \"mechanism\": \"Competitive inhibition\",\n", + " \"pathway\": \"Cholesterol biosynthesis\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=3)).isoformat()\n", + " },\n", + " {\n", + " \"compound_id\": \"CID54686970\",\n", + " \"compound_name\": \"Metformin\",\n", + " \"target_protein\": \"PRKAA1\",\n", + " \"target_name\": \"5'-AMP-activated protein kinase catalytic subunit alpha-1\",\n", + " \"interaction_type\": \"activator\",\n", + " \"binding_affinity\": 15.0,\n", + " \"mechanism\": \"Allosteric activation\",\n", + " \"pathway\": \"AMPK signaling\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=4)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(drug_data_file, 'w') as f:\n", + " json.dump(drug_data, f, indent=2)\n", + "\n", + "# Ingest from local file\n", + "file_data = file_ingestor.ingest_file(drug_data_file)\n", + "parsed_drug = structured_parser.parse_json(json.dumps(drug_data))\n", + "\n", + "# Ingest from drug APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(drug_apis[0]) # PubChem API\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"drug_target_interactions\",\n", + " limit=1000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " db_data = {\"data\": drug_data}\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Local drug-target interactions: {len(drug_data)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Drug APIs: {len(drug_apis)}\")\n", + "print(f\" Protein APIs: {len(protein_apis)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Drug and Target Entities\n", + "\n", + "Extract compounds, targets, and interactions from the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "all_drug_texts = []\n", + "all_interactions = []\n", + "\n", + "# Process parsed drug data\n", + "if parsed_drug and isinstance(parsed_drug, dict):\n", + " interactions = parsed_drug.get(\"data\", drug_data)\n", + " for interaction in interactions:\n", + " all_interactions.append(interaction)\n", + " interaction_text = f\"Compound {interaction.get('compound_name', '')} {interaction.get('interaction_type', '')} target {interaction.get('target_name', '')} with binding affinity {interaction.get('binding_affinity', 0)}\"\n", + " all_drug_texts.append(interaction_text)\n", + "\n", + "# Extract entities\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_triples = []\n", + "\n", + "for text in all_drug_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " triples = triple_extractor.extract(text)\n", + " all_triples.extend(triples)\n", + "\n", + "# Build structured entity list\n", + "compound_entities = []\n", + "target_entities = []\n", + "interaction_entities = []\n", + "\n", + "unique_compounds = {}\n", + "unique_targets = {}\n", + "\n", + "for interaction in all_interactions:\n", + " compound_id = interaction.get(\"compound_id\", \"\")\n", + " compound_name = interaction.get(\"compound_name\", \"\")\n", + " \n", + " if compound_id and compound_id not in unique_compounds:\n", + " compound_entity = {\n", + " \"id\": compound_id,\n", + " \"type\": \"Compound\",\n", + " \"properties\": {\n", + " \"compound_id\": compound_id,\n", + " \"name\": compound_name,\n", + " \"pathway\": interaction.get(\"pathway\", \"\")\n", + " }\n", + " }\n", + " compound_entities.append(compound_entity)\n", + " unique_compounds[compound_id] = compound_entity\n", + " \n", + " target_id = interaction.get(\"target_protein\", \"\")\n", + " target_name = interaction.get(\"target_name\", \"\")\n", + " \n", + " if target_id and target_id not in unique_targets:\n", + " target_entity = {\n", + " \"id\": target_id,\n", + " \"type\": \"Target\",\n", + " \"properties\": {\n", + " \"protein_id\": target_id,\n", + " \"name\": target_name,\n", + " \"pathway\": interaction.get(\"pathway\", \"\")\n", + " }\n", + " }\n", + " target_entities.append(target_entity)\n", + " unique_targets[target_id] = target_entity\n", + " \n", + " interaction_entity = {\n", + " \"id\": f\"{compound_id}_{target_id}\",\n", + " \"type\": \"Interaction\",\n", + " \"properties\": {\n", + " \"compound\": compound_id,\n", + " \"target\": target_id,\n", + " \"interaction_type\": interaction.get(\"interaction_type\", \"\"),\n", + " \"binding_affinity\": interaction.get(\"binding_affinity\", 0),\n", + " \"mechanism\": interaction.get(\"mechanism\", \"\"),\n", + " \"timestamp\": interaction.get(\"timestamp\", \"\")\n", + " }\n", + " }\n", + " interaction_entities.append(interaction_entity)\n", + "\n", + "print(f\"Extracted {len(compound_entities)} unique compounds\")\n", + "print(f\"Extracted {len(target_entities)} unique targets\")\n", + "print(f\"Extracted {len(interaction_entities)} interactions\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Extracted {len(all_triples)} triples\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Drug-Target Knowledge Graph\n", + "\n", + "Build a knowledge graph from extracted drug-target entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "# Add all entities\n", + "for compound in compound_entities:\n", + " builder.add_entity(\n", + " entity_id=compound[\"id\"],\n", + " entity_type=compound[\"type\"],\n", + " properties=compound.get(\"properties\", {})\n", + " )\n", + "\n", + "for target in target_entities:\n", + " builder.add_entity(\n", + " entity_id=target[\"id\"],\n", + " entity_type=target[\"type\"],\n", + " properties=target.get(\"properties\", {})\n", + " )\n", + "\n", + "for interaction in interaction_entities:\n", + " builder.add_entity(\n", + " entity_id=interaction[\"id\"],\n", + " entity_type=interaction[\"type\"],\n", + " properties=interaction.get(\"properties\", {})\n", + " )\n", + "\n", + "# Add relationships\n", + "relationships = []\n", + "for interaction in interaction_entities:\n", + " compound_id = interaction[\"properties\"].get(\"compound\", \"\")\n", + " target_id = interaction[\"properties\"].get(\"target\", \"\")\n", + " interaction_id = interaction[\"id\"]\n", + " interaction_type = interaction[\"properties\"].get(\"interaction_type\", \"\")\n", + " binding_affinity = interaction[\"properties\"].get(\"binding_affinity\", 0)\n", + " \n", + " # Compound-Target relationship\n", + " builder.add_relationship(\n", + " source_id=compound_id,\n", + " target_id=target_id,\n", + " relationship_type=interaction_type,\n", + " properties={\n", + " \"binding_affinity\": binding_affinity,\n", + " \"mechanism\": interaction[\"properties\"].get(\"mechanism\", \"\")\n", + " }\n", + " )\n", + " \n", + " # Interaction relationships\n", + " builder.add_relationship(\n", + " source_id=compound_id,\n", + " target_id=interaction_id,\n", + " relationship_type=\"has_interaction\",\n", + " properties={}\n", + " )\n", + " builder.add_relationship(\n", + " source_id=interaction_id,\n", + " target_id=target_id,\n", + " relationship_type=\"targets\",\n", + " properties={}\n", + " )\n", + " \n", + " relationships.append({\n", + " \"source\": compound_id,\n", + " \"target\": target_id,\n", + " \"type\": interaction_type,\n", + " \"binding_affinity\": binding_affinity\n", + " })\n", + "\n", + "knowledge_graph = builder.build()\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.nodes)} nodes\")\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.edges)} edges\")\n", + "print(f\"Added {len(relationships)} drug-target relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Generate Embeddings and Setup Vector Store\n", + "\n", + "Generate embeddings from compound and target descriptions and setup vector store for similarity search.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_generator = EmbeddingGenerator()\n", + "text_embedder = TextEmbedder()\n", + "vector_store = VectorStore()\n", + "hybrid_search = HybridSearch(vector_store, knowledge_graph)\n", + "\n", + "# Generate embeddings for compounds\n", + "compound_texts = []\n", + "compound_metadata = []\n", + "for compound in compound_entities:\n", + " compound_text = f\"{compound['properties'].get('name', '')} pathway {compound['properties'].get('pathway', '')}\"\n", + " compound_texts.append(compound_text)\n", + " compound_metadata.append({\n", + " \"id\": compound[\"id\"],\n", + " \"type\": \"compound\",\n", + " \"name\": compound[\"properties\"].get(\"name\", \"\")\n", + " })\n", + "\n", + "# Generate embeddings for targets\n", + "target_texts = []\n", + "target_metadata = []\n", + "for target in target_entities:\n", + " target_text = f\"{target['properties'].get('name', '')} pathway {target['properties'].get('pathway', '')}\"\n", + " target_texts.append(target_text)\n", + " target_metadata.append({\n", + " \"id\": target[\"id\"],\n", + " \"type\": \"target\",\n", + " \"name\": target[\"properties\"].get(\"name\", \"\")\n", + " })\n", + "\n", + "# Generate embeddings\n", + "all_texts = compound_texts + target_texts\n", + "all_metadata = compound_metadata + target_metadata\n", + "\n", + "embeddings = []\n", + "for text in all_texts:\n", + " embedding = text_embedder.embed(text)\n", + " embeddings.append(embedding)\n", + "\n", + "# Store in vector store\n", + "vector_store.store_vectors(embeddings, all_metadata)\n", + "\n", + "print(f\"Generated {len(embeddings)} embeddings\")\n", + "print(f\"Stored {len(compound_texts)} compound embeddings\")\n", + "print(f\"Stored {len(target_texts)} target embeddings\")\n", + "print(f\"Vector store ready for similarity search\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Predict Drug-Target Interactions\n", + "\n", + "Use hybrid search and inference to predict drug-target interactions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer(knowledge_graph)\n", + "centrality_calculator = CentralityCalculator(knowledge_graph)\n", + "community_detector = CommunityDetector(knowledge_graph)\n", + "connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n", + "temporal_query = TemporalGraphQuery(knowledge_graph)\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Compute graph metrics\n", + "graph_metrics = graph_analyzer.compute_metrics()\n", + "\n", + "# Calculate centrality\n", + "centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n", + "top_central_targets = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n", + "\n", + "# Detect communities\n", + "communities = community_detector.detect_communities()\n", + "community_count = len(set(communities.values())) if communities else 0\n", + "\n", + "# Analyze connectivity\n", + "connectivity_results = connectivity_analyzer.analyze_connectivity()\n", + "\n", + "# Define interaction prediction rules\n", + "prediction_rules = [\n", + " {\n", + " \"name\": \"high_affinity_interaction\",\n", + " \"condition\": \"binding_affinity < 10\",\n", + " \"action\": \"predict_strong_interaction\"\n", + " },\n", + " {\n", + " \"name\": \"inhibitor_interaction\",\n", + " \"condition\": \"interaction_type == 'inhibitor' AND binding_affinity < 5\",\n", + " \"action\": \"predict_potent_inhibitor\"\n", + " },\n", + " {\n", + " \"name\": \"target_identification\",\n", + " \"condition\": \"multiple_compounds_target_same_protein\",\n", + " \"action\": \"identify_druggable_target\"\n", + " }\n", + "]\n", + "\n", + "for rule in prediction_rules:\n", + " rule_manager.add_rule(rule[\"name\"], rule[\"condition\"], rule[\"action\"])\n", + "\n", + "# Predict interactions using similarity search\n", + "query_compound = \"Aspirin\"\n", + "query_embedding = text_embedder.embed(query_compound)\n", + "\n", + "# Hybrid search\n", + "search_results = hybrid_search.search(\n", + " query_embedding=query_embedding,\n", + " query_text=query_compound,\n", + " k=5,\n", + " use_graph_expansion=True\n", + ")\n", + "\n", + "# Predict interactions\n", + "predicted_interactions = []\n", + "for compound in compound_entities:\n", + " compound_id = compound[\"id\"]\n", + " compound_name = compound[\"properties\"].get(\"name\", \"\")\n", + " \n", + " # Find interactions for this compound\n", + " compound_interactions = [r for r in relationships if r[\"source\"] == compound_id]\n", + " \n", + " # Calculate interaction score\n", + " interaction_score = 0\n", + " for interaction in compound_interactions:\n", + " binding_affinity = interaction.get(\"binding_affinity\", 100)\n", + " if binding_affinity < 10:\n", + " interaction_score += 3\n", + " elif binding_affinity < 50:\n", + " interaction_score += 2\n", + " else:\n", + " interaction_score += 1\n", + " \n", + " predicted_interactions.append({\n", + " \"compound\": compound_name,\n", + " \"compound_id\": compound_id,\n", + " \"interaction_count\": len(compound_interactions),\n", + " \"interaction_score\": interaction_score,\n", + " \"targets\": [r[\"target\"] for r in compound_interactions]\n", + " })\n", + "\n", + "# Identify druggable targets\n", + "target_interaction_counts = {}\n", + "for rel in relationships:\n", + " target = rel[\"target\"]\n", + " target_interaction_counts[target] = target_interaction_counts.get(target, 0) + 1\n", + "\n", + "druggable_targets = []\n", + "for target_id, count in target_interaction_counts.items():\n", + " if count >= 2:\n", + " target_name = next((t[\"properties\"].get(\"name\", target_id) for t in target_entities if t[\"id\"] == target_id), target_id)\n", + " druggable_targets.append({\n", + " \"target\": target_name,\n", + " \"target_id\": target_id,\n", + " \"compound_count\": count,\n", + " \"description\": f\"Target {target_name} is targeted by {count} compounds, indicating druggability\"\n", + " })\n", + "\n", + "print(f\"Analyzed {len(compound_entities)} compounds\")\n", + "print(f\"Found {community_count} target communities\")\n", + "print(f\"Identified {len(druggable_targets)} druggable targets\")\n", + "print(f\"\\nTop 5 Central Targets:\")\n", + "for i, (target_id, centrality) in enumerate(top_central_targets[:5], 1):\n", + " target_name = next((t[\"properties\"].get(\"name\", target_id) for t in target_entities if t[\"id\"] == target_id), target_id)\n", + " print(f\" {i}. {target_name} (centrality: {centrality:.3f})\")\n", + "print(f\"\\nPredicted Interactions:\")\n", + "for pred in sorted(predicted_interactions, key=lambda x: x[\"interaction_score\"], reverse=True)[:5]:\n", + " print(f\" - {pred['compound']}: {pred['interaction_count']} interactions, Score: {pred['interaction_score']}\")\n", + "print(f\"\\nDruggable Targets:\")\n", + "for target in druggable_targets[:5]:\n", + " print(f\" - {target['target']}: {target['compound_count']} compounds\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ontology_generator = OntologyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Generate drug discovery ontology\n", + "drug_ontology = ontology_generator.generate_ontology(\n", + " knowledge_graph=knowledge_graph,\n", + " domain=\"DrugDiscovery\"\n", + ")\n", + "\n", + "# Validate ontology\n", + "validation_result = ontology_validator.validate_ontology(drug_ontology)\n", + "\n", + "# Export knowledge graph\n", + "kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"drug_target_kg.json\"))\n", + "kg_rdf = rdf_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"drug_target_kg.rdf\"))\n", + "\n", + "# Export ontology\n", + "ontology_owl = owl_exporter.export(drug_ontology, output_path=os.path.join(temp_dir, \"drug_ontology.owl\"))\n", + "\n", + "# Generate report\n", + "report_content = f\"\"\"\n", + "# Drug Discovery Pipeline Report\n", + "\n", + "## Executive Summary\n", + "- Total Compounds Analyzed: {len(compound_entities)}\n", + "- Total Targets: {len(target_entities)}\n", + "- Total Interactions: {len(interaction_entities)}\n", + "- Druggable Targets Identified: {len(druggable_targets)}\n", + "- High-Affinity Interactions: {len([r for r in relationships if r.get('binding_affinity', 100) < 10])}\n", + "\n", + "## Top Druggable Targets\n", + "\"\"\"\n", + "for i, target in enumerate(druggable_targets[:10], 1):\n", + " report_content += f\"\"\"\n", + "{i}. {target['target']}\n", + " - Compound Count: {target['compound_count']}\n", + " - Description: {target['description']}\n", + "\"\"\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Predicted Interactions\n", + "\"\"\"\n", + "for pred in sorted(predicted_interactions, key=lambda x: x[\"interaction_score\"], reverse=True)[:10]:\n", + " report_content += f\"\"\"\n", + "### {pred['compound']}\n", + "- Interaction Count: {pred['interaction_count']}\n", + "- Interaction Score: {pred['interaction_score']}\n", + "- Targets: {', '.join(pred['targets'][:5])}\n", + "\"\"\"\n", + "\n", + "report_path = os.path.join(temp_dir, \"drug_discovery_report.md\")\n", + "with open(report_path, 'w') as f:\n", + " f.write(report_content)\n", + "\n", + "print(f\"Generated drug discovery ontology with {len(drug_ontology.classes)} classes\")\n", + "print(f\"Ontology validation: {'Valid' if validation_result.valid else 'Invalid'}\")\n", + "print(f\" Errors: {len(validation_result.errors)}\")\n", + "print(f\" Warnings: {len(validation_result.warnings)}\")\n", + "print(f\"Exported knowledge graph to JSON and RDF\")\n", + "print(f\"Exported ontology to OWL\")\n", + "print(f\"Generated discovery report: {report_path}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Visualize Drug-Target Network\n", + "\n", + "Visualize the drug-target knowledge graph, ontology, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "kg_viz = kg_visualizer.visualize(\n", + " knowledge_graph,\n", + " layout=\"force_directed\",\n", + " highlight_nodes=[t[\"id\"] for t in druggable_targets[:5]],\n", + " node_size_by=\"centrality\"\n", + ")\n", + "\n", + "# Visualize ontology\n", + "ontology_viz = ontology_visualizer.visualize(\n", + " drug_ontology,\n", + " layout=\"hierarchical\"\n", + ")\n", + "\n", + "# Visualize analytics\n", + "analytics_viz = analytics_visualizer.visualize(\n", + " knowledge_graph,\n", + " metrics={\n", + " \"centrality\": dict(top_central_targets[:10]),\n", + " \"communities\": communities,\n", + " \"connectivity\": connectivity_results,\n", + " \"interaction_scores\": {p[\"compound_id\"]: p[\"interaction_score\"] for p in predicted_interactions}\n", + " }\n", + ")\n", + "\n", + "print(\"Generated visualizations:\")\n", + "print(\" - Knowledge Graph: Drug-target network with highlighted druggable targets\")\n", + "print(\" - Ontology Visualization: Drug discovery ontology hierarchy\")\n", + "print(\" - Analytics Visualization: Centrality, communities, connectivity, and interaction scores\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb b/cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb new file mode 100644 index 00000000..41c52085 --- /dev/null +++ b/cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb @@ -0,0 +1,775 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Genomic Variant Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete genomic variant analysis pipeline: ingest genomic data from multiple sources (APIs, databases, feeds), extract variant entities, build genomic knowledge graph, analyze disease associations, predict variant impact, and perform pathway analysis.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, DBIngestor, FeedIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser, DocumentParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Genomic Data Sources → Parse → Extract Entities (variants, genes, diseases, pathways) → Build Genomic KG → Analyze Associations → Predict Impact → Pathway Analysis → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest Genomic Data from Multiple Sources\n", + "\n", + "Ingest genomic variant data from APIs, databases, and research feeds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, DBIngestor, FeedIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, StructuredDataParser, DocumentParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.kg import ConflictDetector\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "document_parser = DocumentParser()\n", + "\n", + "# Real genomic APIs\n", + "genomic_apis = [\n", + " \"https://rest.ensembl.org/variation/human/rs699\", # Ensembl API\n", + " \"https://api.ncbi.nlm.nih.gov/variation/v0/variant/NC_000001.10:g.230710048A%3EG\", # NCBI Variation API\n", + " \"https://api.ncbi.nlm.nih.gov/variation/v0/beta/refsnp/699\" # ClinVar API\n", + "]\n", + "\n", + "# Real genomic databases\n", + "genomic_databases = [\n", + " \"dbSNP\",\n", + " \"ClinVar\",\n", + " \"COSMIC\"\n", + "]\n", + "\n", + "# Real research feeds\n", + "genomic_feeds = [\n", + " \"https://pubmed.ncbi.nlm.nih.gov/rss/search?term=genomic+variants\",\n", + " \"https://pubmed.ncbi.nlm.nih.gov/rss/search?term=genetic+variation\"\n", + "]\n", + "\n", + "# Real database connection for variant annotations\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/genomic_db\"\n", + "db_query = \"SELECT variant_id, gene_symbol, disease_name, clinical_significance, chromosome, position FROM variants WHERE clinical_significance IN ('Pathogenic', 'Likely Pathogenic') LIMIT 1000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample genomic variant data for local ingestion\n", + "genomic_data_file = os.path.join(temp_dir, \"genomic_variants.json\")\n", + "genomic_data = [\n", + " {\n", + " \"variant_id\": \"rs699\",\n", + " \"gene_symbol\": \"AGT\",\n", + " \"gene_name\": \"Angiotensinogen\",\n", + " \"disease_name\": \"Hypertension\",\n", + " \"clinical_significance\": \"Pathogenic\",\n", + " \"chromosome\": \"1\",\n", + " \"position\": 230710048,\n", + " \"ref_allele\": \"A\",\n", + " \"alt_allele\": \"G\",\n", + " \"pathway\": \"Renin-angiotensin system\",\n", + " \"impact\": \"High\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=1)).isoformat()\n", + " },\n", + " {\n", + " \"variant_id\": \"rs7412\",\n", + " \"gene_symbol\": \"APOE\",\n", + " \"gene_name\": \"Apolipoprotein E\",\n", + " \"disease_name\": \"Alzheimer's Disease\",\n", + " \"clinical_significance\": \"Pathogenic\",\n", + " \"chromosome\": \"19\",\n", + " \"position\": 44908822,\n", + " \"ref_allele\": \"C\",\n", + " \"alt_allele\": \"T\",\n", + " \"pathway\": \"Lipid metabolism\",\n", + " \"impact\": \"High\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=2)).isoformat()\n", + " },\n", + " {\n", + " \"variant_id\": \"rs1800566\",\n", + " \"gene_symbol\": \"NAT2\",\n", + " \"gene_name\": \"N-acetyltransferase 2\",\n", + " \"disease_name\": \"Drug Metabolism\",\n", + " \"clinical_significance\": \"Likely Pathogenic\",\n", + " \"chromosome\": \"8\",\n", + " \"position\": 18248728,\n", + " \"ref_allele\": \"G\",\n", + " \"alt_allele\": \"A\",\n", + " \"pathway\": \"Drug metabolism\",\n", + " \"impact\": \"Moderate\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=3)).isoformat()\n", + " },\n", + " {\n", + " \"variant_id\": \"rs1799853\",\n", + " \"gene_symbol\": \"CYP2C9\",\n", + " \"gene_name\": \"Cytochrome P450 2C9\",\n", + " \"disease_name\": \"Warfarin Sensitivity\",\n", + " \"clinical_significance\": \"Pathogenic\",\n", + " \"chromosome\": \"10\",\n", + " \"position\": 96741054,\n", + " \"ref_allele\": \"C\",\n", + " \"alt_allele\": \"T\",\n", + " \"pathway\": \"Drug metabolism\",\n", + " \"impact\": \"High\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=4)).isoformat()\n", + " },\n", + " {\n", + " \"variant_id\": \"rs1057910\",\n", + " \"gene_symbol\": \"CYP2C9\",\n", + " \"gene_name\": \"Cytochrome P450 2C9\",\n", + " \"disease_name\": \"Warfarin Sensitivity\",\n", + " \"clinical_significance\": \"Pathogenic\",\n", + " \"chromosome\": \"10\",\n", + " \"position\": 96741055,\n", + " \"ref_allele\": \"A\",\n", + " \"alt_allele\": \"C\",\n", + " \"pathway\": \"Drug metabolism\",\n", + " \"impact\": \"High\",\n", + " \"timestamp\": (datetime.now() - timedelta(days=5)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(genomic_data_file, 'w') as f:\n", + " json.dump(genomic_data, f, indent=2)\n", + "\n", + "# Ingest from local file\n", + "file_data = file_ingestor.ingest_file(genomic_data_file)\n", + "parsed_genomic = structured_parser.parse_json(json.dumps(genomic_data))\n", + "\n", + "# Ingest from genomic APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(genomic_apis[0]) # Ensembl API\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Ingest from genomic feeds\n", + "feed_data_list = []\n", + "for feed_url in genomic_feeds:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " feed_data_list.append(feed_data)\n", + " print(f\"✓ Ingested feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion failed for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"variants\",\n", + " limit=1000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " db_data = {\"data\": genomic_data}\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Local variants: {len(genomic_data)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Feeds ingested: {len(feed_data_list)}\")\n", + "print(f\" Web APIs: {len(genomic_apis)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Genomic Entities\n", + "\n", + "Extract variants, genes, diseases, and pathways from the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "triple_extractor = TripleExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "\n", + "all_genomic_texts = []\n", + "all_variants = []\n", + "\n", + "# Process parsed genomic data\n", + "if parsed_genomic and isinstance(parsed_genomic, dict):\n", + " variants = parsed_genomic.get(\"data\", genomic_data)\n", + " for variant in variants:\n", + " all_variants.append(variant)\n", + " variant_text = f\"Variant {variant.get('variant_id', '')} in gene {variant.get('gene_symbol', '')} associated with {variant.get('disease_name', '')} in pathway {variant.get('pathway', '')}\"\n", + " all_genomic_texts.append(variant_text)\n", + "\n", + "# Extract entities\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_triples = []\n", + "\n", + "for text in all_genomic_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " triples = triple_extractor.extract(text)\n", + " all_triples.extend(triples)\n", + "\n", + "# Build structured entity list\n", + "variant_entities = []\n", + "gene_entities = []\n", + "disease_entities = []\n", + "pathway_entities = []\n", + "\n", + "unique_genes = {}\n", + "unique_diseases = {}\n", + "unique_pathways = {}\n", + "\n", + "for variant in all_variants:\n", + " variant_entity = {\n", + " \"id\": variant.get(\"variant_id\", \"\"),\n", + " \"type\": \"Variant\",\n", + " \"properties\": {\n", + " \"variant_id\": variant.get(\"variant_id\", \"\"),\n", + " \"chromosome\": variant.get(\"chromosome\", \"\"),\n", + " \"position\": variant.get(\"position\", 0),\n", + " \"ref_allele\": variant.get(\"ref_allele\", \"\"),\n", + " \"alt_allele\": variant.get(\"alt_allele\", \"\"),\n", + " \"clinical_significance\": variant.get(\"clinical_significance\", \"\"),\n", + " \"impact\": variant.get(\"impact\", \"\"),\n", + " \"timestamp\": variant.get(\"timestamp\", \"\")\n", + " }\n", + " }\n", + " variant_entities.append(variant_entity)\n", + " \n", + " # Add gene entity\n", + " gene_symbol = variant.get(\"gene_symbol\", \"\")\n", + " if gene_symbol and gene_symbol not in unique_genes:\n", + " gene_entity = {\n", + " \"id\": gene_symbol,\n", + " \"type\": \"Gene\",\n", + " \"properties\": {\n", + " \"symbol\": gene_symbol,\n", + " \"name\": variant.get(\"gene_name\", \"\"),\n", + " \"chromosome\": variant.get(\"chromosome\", \"\")\n", + " }\n", + " }\n", + " gene_entities.append(gene_entity)\n", + " unique_genes[gene_symbol] = gene_entity\n", + " \n", + " # Add disease entity\n", + " disease_name = variant.get(\"disease_name\", \"\")\n", + " if disease_name and disease_name not in unique_diseases:\n", + " disease_entity = {\n", + " \"id\": disease_name.replace(\" \", \"_\"),\n", + " \"type\": \"Disease\",\n", + " \"properties\": {\n", + " \"name\": disease_name\n", + " }\n", + " }\n", + " disease_entities.append(disease_entity)\n", + " unique_diseases[disease_name] = disease_entity\n", + " \n", + " # Add pathway entity\n", + " pathway_name = variant.get(\"pathway\", \"\")\n", + " if pathway_name and pathway_name not in unique_pathways:\n", + " pathway_entity = {\n", + " \"id\": pathway_name.replace(\" \", \"_\"),\n", + " \"type\": \"Pathway\",\n", + " \"properties\": {\n", + " \"name\": pathway_name\n", + " }\n", + " }\n", + " pathway_entities.append(pathway_entity)\n", + " unique_pathways[pathway_name] = pathway_entity\n", + "\n", + "print(f\"Extracted {len(variant_entities)} variants\")\n", + "print(f\"Extracted {len(gene_entities)} unique genes\")\n", + "print(f\"Extracted {len(disease_entities)} unique diseases\")\n", + "print(f\"Extracted {len(pathway_entities)} unique pathways\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Extracted {len(all_triples)} triples\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Genomic Knowledge Graph\n", + "\n", + "Build a knowledge graph from extracted genomic entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "# Add all entities\n", + "for variant in variant_entities:\n", + " builder.add_entity(\n", + " entity_id=variant[\"id\"],\n", + " entity_type=variant[\"type\"],\n", + " properties=variant.get(\"properties\", {})\n", + " )\n", + "\n", + "for gene in gene_entities:\n", + " builder.add_entity(\n", + " entity_id=gene[\"id\"],\n", + " entity_type=gene[\"type\"],\n", + " properties=gene.get(\"properties\", {})\n", + " )\n", + "\n", + "for disease in disease_entities:\n", + " builder.add_entity(\n", + " entity_id=disease[\"id\"],\n", + " entity_type=disease[\"type\"],\n", + " properties=disease.get(\"properties\", {})\n", + " )\n", + "\n", + "for pathway in pathway_entities:\n", + " builder.add_entity(\n", + " entity_id=pathway[\"id\"],\n", + " entity_type=pathway[\"type\"],\n", + " properties=pathway.get(\"properties\", {})\n", + " )\n", + "\n", + "# Add relationships\n", + "relationships = []\n", + "for variant in variant_entities:\n", + " variant_id = variant[\"id\"]\n", + " gene_symbol = variant[\"properties\"].get(\"gene_symbol\", \"\")\n", + " disease_name = variant[\"properties\"].get(\"disease_name\", \"\").replace(\" \", \"_\")\n", + " pathway_name = variant[\"properties\"].get(\"pathway\", \"\").replace(\" \", \"_\")\n", + " \n", + " # Find corresponding entities\n", + " gene_entity = unique_genes.get(gene_symbol)\n", + " disease_entity = unique_diseases.get(disease_name.replace(\"_\", \" \"))\n", + " pathway_entity = unique_pathways.get(pathway_name.replace(\"_\", \" \"))\n", + " \n", + " if gene_entity:\n", + " builder.add_relationship(\n", + " source_id=variant_id,\n", + " target_id=gene_entity[\"id\"],\n", + " relationship_type=\"located_in\",\n", + " properties={}\n", + " )\n", + " relationships.append({\n", + " \"source\": variant_id,\n", + " \"target\": gene_entity[\"id\"],\n", + " \"type\": \"located_in\"\n", + " })\n", + " \n", + " if disease_entity:\n", + " builder.add_relationship(\n", + " source_id=variant_id,\n", + " target_id=disease_entity[\"id\"],\n", + " relationship_type=\"associated_with\",\n", + " properties={\n", + " \"clinical_significance\": variant[\"properties\"].get(\"clinical_significance\", \"\")\n", + " }\n", + " )\n", + " relationships.append({\n", + " \"source\": variant_id,\n", + " \"target\": disease_entity[\"id\"],\n", + " \"type\": \"associated_with\"\n", + " })\n", + " \n", + " if pathway_entity:\n", + " builder.add_relationship(\n", + " source_id=gene_entity[\"id\"] if gene_entity else variant_id,\n", + " target_id=pathway_entity[\"id\"],\n", + " relationship_type=\"participates_in\",\n", + " properties={}\n", + " )\n", + " relationships.append({\n", + " \"source\": gene_entity[\"id\"] if gene_entity else variant_id,\n", + " \"target\": pathway_entity[\"id\"],\n", + " \"type\": \"participates_in\"\n", + " })\n", + "\n", + "knowledge_graph = builder.build()\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.nodes)} nodes\")\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.edges)} edges\")\n", + "print(f\"Added {len(relationships)} genomic relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze Associations and Predict Impact\n", + "\n", + "Analyze variant-disease associations and predict variant impact on protein function.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer(knowledge_graph)\n", + "centrality_calculator = CentralityCalculator(knowledge_graph)\n", + "community_detector = CommunityDetector(knowledge_graph)\n", + "connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n", + "temporal_query = TemporalGraphQuery(knowledge_graph)\n", + "pattern_detector = TemporalPatternDetector(knowledge_graph)\n", + "\n", + "# Compute graph metrics\n", + "graph_metrics = graph_analyzer.compute_metrics()\n", + "\n", + "# Calculate centrality\n", + "centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n", + "top_central_genes = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n", + "\n", + "# Detect communities\n", + "communities = community_detector.detect_communities()\n", + "community_count = len(set(communities.values())) if communities else 0\n", + "\n", + "# Analyze connectivity\n", + "connectivity_results = connectivity_analyzer.analyze_connectivity()\n", + "\n", + "# Detect temporal patterns\n", + "start_time = (datetime.now() - timedelta(days=7)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "temporal_results = temporal_query.query_time_range(\n", + " start_time=start_time,\n", + " end_time=end_time,\n", + " relationship_types=[\"associated_with\", \"located_in\"]\n", + ")\n", + "\n", + "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", + " relationship_types=[\"associated_with\"],\n", + " time_window_hours=168\n", + ")\n", + "\n", + "# Impact Prediction using Inference Engine\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Define impact prediction rules\n", + "impact_rules = [\n", + " {\n", + " \"name\": \"high_impact_pathogenic\",\n", + " \"condition\": \"clinical_significance == 'Pathogenic' AND impact == 'High'\",\n", + " \"action\": \"predict_high_impact\"\n", + " },\n", + " {\n", + " \"name\": \"moderate_impact_likely_pathogenic\",\n", + " \"condition\": \"clinical_significance == 'Likely Pathogenic' AND impact == 'Moderate'\",\n", + " \"action\": \"predict_moderate_impact\"\n", + " },\n", + " {\n", + " \"name\": \"disease_association\",\n", + " \"condition\": \"associated_with_disease AND pathogenic\",\n", + " \"action\": \"predict_disease_risk\"\n", + " }\n", + "]\n", + "\n", + "for rule in impact_rules:\n", + " rule_manager.add_rule(rule[\"name\"], rule[\"condition\"], rule[\"action\"])\n", + "\n", + "# Predict variant impact\n", + "variant_impacts = []\n", + "for variant in variant_entities:\n", + " clinical_sig = variant[\"properties\"].get(\"clinical_significance\", \"\")\n", + " impact = variant[\"properties\"].get(\"impact\", \"\")\n", + " \n", + " impact_score = 0\n", + " if clinical_sig == \"Pathogenic\":\n", + " impact_score += 5\n", + " elif clinical_sig == \"Likely Pathogenic\":\n", + " impact_score += 3\n", + " \n", + " if impact == \"High\":\n", + " impact_score += 3\n", + " elif impact == \"Moderate\":\n", + " impact_score += 2\n", + " \n", + " # Check disease associations\n", + " disease_associations = [r for r in relationships if r[\"source\"] == variant[\"id\"] and r[\"type\"] == \"associated_with\"]\n", + " if disease_associations:\n", + " impact_score += 2\n", + " \n", + " variant_impacts.append({\n", + " \"variant\": variant[\"id\"],\n", + " \"impact_score\": min(impact_score, 10),\n", + " \"predicted_impact\": \"High\" if impact_score >= 7 else \"Moderate\" if impact_score >= 4 else \"Low\",\n", + " \"clinical_significance\": clinical_sig,\n", + " \"disease_associations\": len(disease_associations)\n", + " })\n", + "\n", + "print(f\"Analyzed {len(variant_entities)} variants\")\n", + "print(f\"Found {community_count} gene communities\")\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"\\nTop 5 Central Genes:\")\n", + "for i, (gene_id, centrality) in enumerate(top_central_genes[:5], 1):\n", + " print(f\" {i}. {gene_id} (centrality: {centrality:.3f})\")\n", + "print(f\"\\nVariant Impact Predictions:\")\n", + "for impact in sorted(variant_impacts, key=lambda x: x[\"impact_score\"], reverse=True):\n", + " print(f\" - {impact['variant']}: {impact['predicted_impact']} Impact (Score: {impact['impact_score']}/10, Diseases: {impact['disease_associations']})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Genomic Ontology and Pathway Analysis\n", + "\n", + "Generate genomic ontology and perform pathway analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "# Generate genomic ontology\n", + "genomic_ontology = ontology_generator.generate_ontology(\n", + " knowledge_graph=knowledge_graph,\n", + " domain=\"Genomics\"\n", + ")\n", + "\n", + "# Infer classes\n", + "classes = class_inferrer.infer_classes(knowledge_graph)\n", + "for cls in classes:\n", + " genomic_ontology.add_class(cls)\n", + "\n", + "# Generate properties\n", + "properties = property_generator.generate_properties(knowledge_graph)\n", + "for prop in properties:\n", + " genomic_ontology.add_property(prop)\n", + "\n", + "# Validate ontology\n", + "validation_result = ontology_validator.validate_ontology(genomic_ontology)\n", + "\n", + "# Pathway analysis\n", + "pathway_analysis = {}\n", + "for pathway in pathway_entities:\n", + " pathway_id = pathway[\"id\"]\n", + " pathway_name = pathway[\"properties\"].get(\"name\", \"\")\n", + " \n", + " # Find variants and genes in this pathway\n", + " pathway_variants = [r for r in relationships if r[\"target\"] == pathway_id and r[\"type\"] == \"participates_in\"]\n", + " pathway_genes = set()\n", + " for rel in pathway_variants:\n", + " source_entity = next((e for e in variant_entities + gene_entities if e[\"id\"] == rel[\"source\"]), None)\n", + " if source_entity and source_entity[\"type\"] == \"Gene\":\n", + " pathway_genes.add(source_entity[\"id\"])\n", + " \n", + " pathway_analysis[pathway_name] = {\n", + " \"variants\": len([r for r in pathway_variants if next((e for e in variant_entities if e[\"id\"] == r[\"source\"]), None)]),\n", + " \"genes\": len(pathway_genes),\n", + " \"diseases\": len([r for r in relationships if r[\"source\"] in [e[\"id\"] for e in variant_entities] and r[\"type\"] == \"associated_with\"])\n", + " }\n", + "\n", + "print(f\"Generated genomic ontology with {len(genomic_ontology.classes)} classes\")\n", + "print(f\"Ontology validation: {'Valid' if validation_result.valid else 'Invalid'}\")\n", + "print(f\" Errors: {len(validation_result.errors)}\")\n", + "print(f\" Warnings: {len(validation_result.warnings)}\")\n", + "print(f\"\\nPathway Analysis:\")\n", + "for pathway_name, analysis in pathway_analysis.items():\n", + " print(f\" - {pathway_name}: {analysis['variants']} variants, {analysis['genes']} genes, {analysis['diseases']} disease associations\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Generate Reports and Visualize\n", + "\n", + "Generate comprehensive genomic analysis reports and visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "kg_quality_assessor = KGQualityAssessor()\n", + "conflict_detector = ConflictDetector(knowledge_graph)\n", + "\n", + "# Assess graph quality\n", + "quality_metrics = kg_quality_assessor.assess_quality(knowledge_graph)\n", + "\n", + "# Detect conflicts\n", + "conflicts = conflict_detector.detect_conflicts()\n", + "\n", + "# Export knowledge graph\n", + "kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"genomic_kg.json\"))\n", + "kg_rdf = rdf_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"genomic_kg.rdf\"))\n", + "\n", + "# Export ontology\n", + "ontology_owl = owl_exporter.export(genomic_ontology, output_path=os.path.join(temp_dir, \"genomic_ontology.owl\"))\n", + "\n", + "# Generate report\n", + "report_content = f\"\"\"\n", + "# Genomic Variant Analysis Report\n", + "\n", + "## Executive Summary\n", + "- Total Variants Analyzed: {len(variant_entities)}\n", + "- Unique Genes: {len(gene_entities)}\n", + "- Unique Diseases: {len(disease_entities)}\n", + "- Unique Pathways: {len(pathway_entities)}\n", + "- High Impact Variants: {len([v for v in variant_impacts if v['predicted_impact'] == 'High'])}\n", + "\n", + "## Graph Quality Metrics\n", + "- Nodes: {quality_metrics.get('node_count', len(knowledge_graph.nodes))}\n", + "- Edges: {quality_metrics.get('edge_count', len(knowledge_graph.edges))}\n", + "- Completeness: {quality_metrics.get('completeness', 0):.2%}\n", + "- Consistency: {quality_metrics.get('consistency', 0):.2%}\n", + "\n", + "## Top Variants by Impact\n", + "\"\"\"\n", + "for i, impact in enumerate(sorted(variant_impacts, key=lambda x: x[\"impact_score\"], reverse=True)[:10], 1):\n", + " report_content += f\"\"\"\n", + "{i}. {impact['variant']}\n", + " - Predicted Impact: {impact['predicted_impact']}\n", + " - Impact Score: {impact['impact_score']}/10\n", + " - Clinical Significance: {impact['clinical_significance']}\n", + " - Disease Associations: {impact['disease_associations']}\n", + "\"\"\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Pathway Analysis\n", + "\"\"\"\n", + "for pathway_name, analysis in pathway_analysis.items():\n", + " report_content += f\"\"\"\n", + "### {pathway_name}\n", + "- Variants: {analysis['variants']}\n", + "- Genes: {analysis['genes']}\n", + "- Disease Associations: {analysis['diseases']}\n", + "\"\"\"\n", + "\n", + "report_path = os.path.join(temp_dir, \"genomic_analysis_report.md\")\n", + "with open(report_path, 'w') as f:\n", + " f.write(report_content)\n", + "\n", + "print(f\"Exported knowledge graph to JSON and RDF\")\n", + "print(f\"Exported ontology to OWL\")\n", + "print(f\"Generated analysis report: {report_path}\")\n", + "print(f\"\\nQuality Metrics:\")\n", + "print(f\" Nodes: {quality_metrics.get('node_count', len(knowledge_graph.nodes))}\")\n", + "print(f\" Edges: {quality_metrics.get('edge_count', len(knowledge_graph.edges))}\")\n", + "print(f\" Completeness: {quality_metrics.get('completeness', 0):.2%}\")\n", + "print(f\" Consistency: {quality_metrics.get('consistency', 0):.2%}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Visualize Genomic Network\n", + "\n", + "Visualize the genomic knowledge graph, ontology, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "kg_viz = kg_visualizer.visualize(\n", + " knowledge_graph,\n", + " layout=\"force_directed\",\n", + " highlight_nodes=[v[\"id\"] for v in variant_entities],\n", + " node_size_by=\"impact\"\n", + ")\n", + "\n", + "# Visualize ontology\n", + "ontology_viz = ontology_visualizer.visualize(\n", + " genomic_ontology,\n", + " layout=\"hierarchical\"\n", + ")\n", + "\n", + "# Visualize analytics\n", + "analytics_viz = analytics_visualizer.visualize(\n", + " knowledge_graph,\n", + " metrics={\n", + " \"centrality\": dict(top_central_genes[:10]),\n", + " \"communities\": communities,\n", + " \"connectivity\": connectivity_results,\n", + " \"impact_scores\": {v[\"variant\"]: v[\"impact_score\"] for v in variant_impacts}\n", + " }\n", + ")\n", + "\n", + "print(\"Generated visualizations:\")\n", + "print(\" - Knowledge Graph: Genomic variant network with impact-based sizing\")\n", + "print(\" - Ontology Visualization: Genomic ontology hierarchy\")\n", + "print(\" - Analytics Visualization: Centrality, communities, connectivity, and impact scores\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb b/cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb new file mode 100644 index 00000000..eb1e5bbf --- /dev/null +++ b/cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb @@ -0,0 +1,703 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# DeFi Protocol Intelligence Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete DeFi protocol intelligence pipeline: ingest DeFi data from multiple sources (APIs, feeds, databases), extract protocol entities, build DeFi knowledge graph, analyze relationships, assess risks, optimize yields, and generate reports.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: WebIngestor, FeedIngestor, DBIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, HTMLParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, SemanticAnalyzer, EventDetector\n", + "- **KG**: GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "- **Analytics**: ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**DeFi Data Sources → Parse → Extract Entities (protocols, pools, tokens, strategies) → Build DeFi KG → Analyze Relationships → Risk Assessment → Yield Optimization → Generate Reports → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Ingest DeFi Data from Multiple Sources\n", + "\n", + "Ingest DeFi protocol data from APIs, feeds, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import WebIngestor, FeedIngestor, DBIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, HTMLParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, SemanticAnalyzer, EventDetector\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", + "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n", + "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, OntologyVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "web_ingestor = WebIngestor()\n", + "feed_ingestor = FeedIngestor()\n", + "db_ingestor = DBIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "html_parser = HTMLParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real DeFi APIs\n", + "defi_apis = [\n", + " \"https://api.llama.fi/protocols\", # DeFiLlama API\n", + " \"https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2\", # The Graph - Uniswap\n", + " \"https://api.github.com/repos/Uniswap/interface\" # Uniswap GitHub\n", + "]\n", + "\n", + "# Real DeFi protocol feeds\n", + "defi_feeds = [\n", + " \"https://defipulse.com/blog/feed\", # DeFi Pulse\n", + " \"https://feeds.feedburner.com/TheDefiant\" # The Defiant\n", + "]\n", + "\n", + "# Real database connection for protocol metrics\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/defi_db\"\n", + "db_query = \"SELECT protocol_name, tvl, apy, token_address, pool_address, timestamp FROM defi_protocols WHERE timestamp > NOW() - INTERVAL '7 days' ORDER BY tvl DESC LIMIT 1000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample DeFi protocol data for local ingestion\n", + "defi_data_file = os.path.join(temp_dir, \"defi_protocols.json\")\n", + "defi_data = [\n", + " {\n", + " \"protocol_name\": \"Uniswap V3\",\n", + " \"protocol_type\": \"DEX\",\n", + " \"tvl\": 2500000000,\n", + " \"apy\": 12.5,\n", + " \"token_address\": \"0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984\",\n", + " \"pool_address\": \"0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8\",\n", + " \"token_symbol\": \"UNI\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat()\n", + " },\n", + " {\n", + " \"protocol_name\": \"Aave V3\",\n", + " \"protocol_type\": \"Lending\",\n", + " \"tvl\": 1800000000,\n", + " \"apy\": 8.3,\n", + " \"token_address\": \"0x7Fc66500c84A76Ad7e9c93437bFc5Ac33E2DDaE9\",\n", + " \"pool_address\": \"0x87870Bca3F3fD6335C3F4ce8392D69350B4fA4E2\",\n", + " \"token_symbol\": \"AAVE\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=2)).isoformat()\n", + " },\n", + " {\n", + " \"protocol_name\": \"Compound V3\",\n", + " \"protocol_type\": \"Lending\",\n", + " \"tvl\": 1200000000,\n", + " \"apy\": 7.8,\n", + " \"token_address\": \"0xc00e94Cb662C3520282E6f5717214004A7f26888\",\n", + " \"pool_address\": \"0xc3d688B667034EAD2F183C05b6e4B5e5B5b5b5b5\",\n", + " \"token_symbol\": \"COMP\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=3)).isoformat()\n", + " },\n", + " {\n", + " \"protocol_name\": \"Curve Finance\",\n", + " \"protocol_type\": \"DEX\",\n", + " \"tvl\": 1500000000,\n", + " \"apy\": 15.2,\n", + " \"token_address\": \"0xD533a949740bb3306d119CC777fa900bA034cd52\",\n", + " \"pool_address\": \"0xbEbc44782C7dB0a1A60Cb6fe97d0b483032FF1C7\",\n", + " \"token_symbol\": \"CRV\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=4)).isoformat()\n", + " },\n", + " {\n", + " \"protocol_name\": \"MakerDAO\",\n", + " \"protocol_type\": \"Lending\",\n", + " \"tvl\": 8000000000,\n", + " \"apy\": 3.5,\n", + " \"token_address\": \"0x9f8F72aA9304c8B593d555F12eF6589cC3A579A2\",\n", + " \"pool_address\": \"0x35D1b3F3D7966A1DFe207aa4514C12a2594E9c99\",\n", + " \"token_symbol\": \"MKR\",\n", + " \"chain\": \"Ethereum\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=5)).isoformat()\n", + " }\n", + "]\n", + "\n", + "with open(defi_data_file, 'w') as f:\n", + " json.dump(defi_data, f, indent=2)\n", + "\n", + "# Ingest from local file\n", + "file_data = file_ingestor.ingest_file(defi_data_file)\n", + "parsed_defi = structured_parser.parse_json(json.dumps(defi_data))\n", + "\n", + "# Ingest from DeFi APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(defi_apis[2]) # GitHub API\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Ingest from DeFi feeds\n", + "feed_data_list = []\n", + "for feed_url in defi_feeds:\n", + " try:\n", + " feed_data = feed_ingestor.ingest_feed(feed_url)\n", + " if feed_data:\n", + " feed_data_list.append(feed_data)\n", + " print(f\"✓ Ingested feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}\")\n", + " except Exception as e:\n", + " print(f\"⚠ Feed ingestion failed for {feed_url}: {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"defi_protocols\",\n", + " limit=1000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " db_data = {\"data\": defi_data}\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Local protocols: {len(defi_data)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Feeds ingested: {len(feed_data_list)}\")\n", + "print(f\" Web APIs: {len(defi_apis)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract DeFi Entities\n", + "\n", + "Extract protocols, pools, tokens, and strategies from the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "semantic_analyzer = SemanticAnalyzer()\n", + "event_detector = EventDetector()\n", + "\n", + "all_defi_texts = []\n", + "all_protocols = []\n", + "\n", + "# Process parsed DeFi data\n", + "if parsed_defi and isinstance(parsed_defi, dict):\n", + " protocols = parsed_defi.get(\"data\", defi_data)\n", + " for protocol in protocols:\n", + " all_protocols.append(protocol)\n", + " protocol_text = f\"Protocol {protocol.get('protocol_name', '')} type {protocol.get('protocol_type', '')} TVL {protocol.get('tvl', 0)} APY {protocol.get('apy', 0)} token {protocol.get('token_symbol', '')}\"\n", + " all_defi_texts.append(protocol_text)\n", + "\n", + "# Extract entities\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_events = []\n", + "\n", + "for text in all_defi_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " events = event_detector.detect_events(text)\n", + " all_events.extend(events)\n", + "\n", + "# Build structured entity list\n", + "protocol_entities = []\n", + "pool_entities = []\n", + "token_entities = []\n", + "\n", + "for protocol in all_protocols:\n", + " protocol_entity = {\n", + " \"id\": protocol.get(\"protocol_name\", \"\").replace(\" \", \"_\"),\n", + " \"type\": \"Protocol\",\n", + " \"properties\": {\n", + " \"name\": protocol.get(\"protocol_name\", \"\"),\n", + " \"type\": protocol.get(\"protocol_type\", \"\"),\n", + " \"tvl\": protocol.get(\"tvl\", 0),\n", + " \"apy\": protocol.get(\"apy\", 0),\n", + " \"chain\": protocol.get(\"chain\", \"\"),\n", + " \"timestamp\": protocol.get(\"timestamp\", \"\")\n", + " }\n", + " }\n", + " protocol_entities.append(protocol_entity)\n", + " \n", + " # Add pool entity\n", + " pool_entity = {\n", + " \"id\": protocol.get(\"pool_address\", \"\"),\n", + " \"type\": \"Pool\",\n", + " \"properties\": {\n", + " \"address\": protocol.get(\"pool_address\", \"\"),\n", + " \"protocol\": protocol.get(\"protocol_name\", \"\"),\n", + " \"tvl\": protocol.get(\"tvl\", 0),\n", + " \"apy\": protocol.get(\"apy\", 0)\n", + " }\n", + " }\n", + " pool_entities.append(pool_entity)\n", + " \n", + " # Add token entity\n", + " token_entity = {\n", + " \"id\": protocol.get(\"token_address\", \"\"),\n", + " \"type\": \"Token\",\n", + " \"properties\": {\n", + " \"address\": protocol.get(\"token_address\", \"\"),\n", + " \"symbol\": protocol.get(\"token_symbol\", \"\"),\n", + " \"protocol\": protocol.get(\"protocol_name\", \"\")\n", + " }\n", + " }\n", + " token_entities.append(token_entity)\n", + "\n", + "print(f\"Extracted {len(protocol_entities)} protocols\")\n", + "print(f\"Extracted {len(pool_entities)} pools\")\n", + "print(f\"Extracted {len(token_entities)} tokens\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Detected {len(all_events)} events\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build DeFi Knowledge Graph\n", + "\n", + "Build a knowledge graph from extracted DeFi entities and relationships.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "# Add all entities\n", + "for protocol in protocol_entities:\n", + " builder.add_entity(\n", + " entity_id=protocol[\"id\"],\n", + " entity_type=protocol[\"type\"],\n", + " properties=protocol.get(\"properties\", {})\n", + " )\n", + "\n", + "for pool in pool_entities:\n", + " builder.add_entity(\n", + " entity_id=pool[\"id\"],\n", + " entity_type=pool[\"type\"],\n", + " properties=pool.get(\"properties\", {})\n", + " )\n", + "\n", + "for token in token_entities:\n", + " builder.add_entity(\n", + " entity_id=token[\"id\"],\n", + " entity_type=token[\"type\"],\n", + " properties=token.get(\"properties\", {})\n", + " )\n", + "\n", + "# Add relationships\n", + "relationships = []\n", + "for i, protocol in enumerate(protocol_entities):\n", + " protocol_id = protocol[\"id\"]\n", + " pool_id = pool_entities[i][\"id\"]\n", + " token_id = token_entities[i][\"id\"]\n", + " \n", + " # Protocol-Pool relationship\n", + " builder.add_relationship(\n", + " source_id=protocol_id,\n", + " target_id=pool_id,\n", + " relationship_type=\"has_pool\",\n", + " properties={}\n", + " )\n", + " \n", + " # Protocol-Token relationship\n", + " builder.add_relationship(\n", + " source_id=protocol_id,\n", + " target_id=token_id,\n", + " relationship_type=\"has_token\",\n", + " properties={}\n", + " )\n", + " \n", + " # Pool-Token relationship\n", + " builder.add_relationship(\n", + " source_id=pool_id,\n", + " target_id=token_id,\n", + " relationship_type=\"contains\",\n", + " properties={}\n", + " )\n", + " \n", + " relationships.append({\n", + " \"source\": protocol_id,\n", + " \"target\": pool_id,\n", + " \"type\": \"has_pool\"\n", + " })\n", + " relationships.append({\n", + " \"source\": protocol_id,\n", + " \"target\": token_id,\n", + " \"type\": \"has_token\"\n", + " })\n", + "\n", + "knowledge_graph = builder.build()\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.nodes)} nodes\")\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.edges)} edges\")\n", + "print(f\"Added {len(relationships)} DeFi relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Analyze DeFi Relationships and Assess Risks\n", + "\n", + "Analyze protocol relationships, detect communities, and assess risks.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer(knowledge_graph)\n", + "centrality_calculator = CentralityCalculator(knowledge_graph)\n", + "community_detector = CommunityDetector(knowledge_graph)\n", + "connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n", + "temporal_query = TemporalGraphQuery(knowledge_graph)\n", + "pattern_detector = TemporalPatternDetector(knowledge_graph)\n", + "\n", + "# Compute graph metrics\n", + "graph_metrics = graph_analyzer.compute_metrics()\n", + "\n", + "# Calculate centrality\n", + "centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n", + "top_central_protocols = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n", + "\n", + "# Detect communities\n", + "communities = community_detector.detect_communities()\n", + "community_count = len(set(communities.values())) if communities else 0\n", + "\n", + "# Analyze connectivity\n", + "connectivity_results = connectivity_analyzer.analyze_connectivity()\n", + "\n", + "# Detect temporal patterns\n", + "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", + " relationship_types=[\"has_pool\", \"has_token\"],\n", + " time_window_hours=24\n", + ")\n", + "\n", + "# Risk Assessment using Inference Engine\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Define risk rules\n", + "risk_rules = [\n", + " {\n", + " \"name\": \"high_tvl_risk\",\n", + " \"condition\": \"tvl > 5000000000 AND apy < 5\",\n", + " \"action\": \"flag_as_low_yield_high_tvl\"\n", + " },\n", + " {\n", + " \"name\": \"high_apy_risk\",\n", + " \"condition\": \"apy > 20\",\n", + " \"action\": \"flag_as_high_risk_high_yield\"\n", + " },\n", + " {\n", + " \"name\": \"optimal_protocol\",\n", + " \"condition\": \"tvl > 1000000000 AND apy BETWEEN 8 AND 15\",\n", + " \"action\": \"flag_as_optimal\"\n", + " }\n", + "]\n", + "\n", + "for rule in risk_rules:\n", + " rule_manager.add_rule(rule[\"name\"], rule[\"condition\"], rule[\"action\"])\n", + "\n", + "# Assess protocol risks\n", + "protocol_risks = []\n", + "for protocol in protocol_entities:\n", + " tvl = protocol[\"properties\"].get(\"tvl\", 0)\n", + " apy = protocol[\"properties\"].get(\"apy\", 0)\n", + " \n", + " risk_score = 0\n", + " risk_factors = []\n", + " \n", + " if tvl > 5000000000 and apy < 5:\n", + " risk_score += 2\n", + " risk_factors.append(\"low_yield_high_tvl\")\n", + " \n", + " if apy > 20:\n", + " risk_score += 3\n", + " risk_factors.append(\"high_apy_risk\")\n", + " \n", + " if tvl < 500000000:\n", + " risk_score += 1\n", + " risk_factors.append(\"low_tvl\")\n", + " \n", + " if 1000000000 <= tvl <= 5000000000 and 8 <= apy <= 15:\n", + " risk_score = max(0, risk_score - 1)\n", + " risk_factors.append(\"optimal_range\")\n", + " \n", + " protocol_risks.append({\n", + " \"protocol\": protocol[\"id\"],\n", + " \"name\": protocol[\"properties\"].get(\"name\", \"\"),\n", + " \"risk_score\": min(risk_score, 10),\n", + " \"risk_factors\": risk_factors,\n", + " \"tvl\": tvl,\n", + " \"apy\": apy\n", + " })\n", + "\n", + "print(f\"Analyzed {len(protocol_entities)} protocols\")\n", + "print(f\"Found {community_count} protocol communities\")\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"\\nTop 5 Central Protocols:\")\n", + "for i, (protocol_id, centrality) in enumerate(top_central_protocols[:5], 1):\n", + " protocol_name = next((p[\"properties\"].get(\"name\", protocol_id) for p in protocol_entities if p[\"id\"] == protocol_id), protocol_id)\n", + " print(f\" {i}. {protocol_name} (centrality: {centrality:.3f})\")\n", + "print(f\"\\nProtocol Risk Assessment:\")\n", + "for risk in sorted(protocol_risks, key=lambda x: x[\"risk_score\"], reverse=True)[:5]:\n", + " print(f\" - {risk['name']}: Risk Score {risk['risk_score']}/10, Factors: {', '.join(risk['risk_factors'])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate DeFi Ontology and Optimize Yields\n", + "\n", + "Generate DeFi protocol ontology and optimize yield strategies.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ontology_generator = OntologyGenerator()\n", + "class_inferrer = ClassInferrer()\n", + "property_generator = PropertyGenerator()\n", + "ontology_validator = OntologyValidator()\n", + "\n", + "# Generate DeFi ontology\n", + "defi_ontology = ontology_generator.generate_ontology(\n", + " knowledge_graph=knowledge_graph,\n", + " domain=\"DeFi\"\n", + ")\n", + "\n", + "# Infer classes\n", + "classes = class_inferrer.infer_classes(knowledge_graph)\n", + "for cls in classes:\n", + " defi_ontology.add_class(cls)\n", + "\n", + "# Generate properties\n", + "properties = property_generator.generate_properties(knowledge_graph)\n", + "for prop in properties:\n", + " defi_ontology.add_property(prop)\n", + "\n", + "# Validate ontology\n", + "validation_result = ontology_validator.validate_ontology(defi_ontology)\n", + "\n", + "# Yield optimization\n", + "yield_optimization = []\n", + "for protocol in protocol_entities:\n", + " tvl = protocol[\"properties\"].get(\"tvl\", 0)\n", + " apy = protocol[\"properties\"].get(\"apy\", 0)\n", + " protocol_type = protocol[\"properties\"].get(\"type\", \"\")\n", + " \n", + " # Calculate yield score\n", + " yield_score = (apy * 0.6) + (min(tvl / 1000000000, 10) * 0.4)\n", + " \n", + " optimization_suggestions = []\n", + " if apy < 8 and tvl > 1000000000:\n", + " optimization_suggestions.append(\"Consider higher APY protocols for better yield\")\n", + " if tvl < 500000000:\n", + " optimization_suggestions.append(\"Low TVL may indicate higher risk\")\n", + " if apy > 15:\n", + " optimization_suggestions.append(\"High APY may indicate higher risk, diversify\")\n", + " \n", + " yield_optimization.append({\n", + " \"protocol\": protocol[\"properties\"].get(\"name\", \"\"),\n", + " \"yield_score\": yield_score,\n", + " \"apy\": apy,\n", + " \"tvl\": tvl,\n", + " \"suggestions\": optimization_suggestions\n", + " })\n", + "\n", + "print(f\"Generated DeFi ontology with {len(defi_ontology.classes)} classes\")\n", + "print(f\"Ontology validation: {'Valid' if validation_result.valid else 'Invalid'}\")\n", + "print(f\" Errors: {len(validation_result.errors)}\")\n", + "print(f\" Warnings: {len(validation_result.warnings)}\")\n", + "print(f\"\\nYield Optimization Recommendations:\")\n", + "for opt in sorted(yield_optimization, key=lambda x: x[\"yield_score\"], reverse=True)[:5]:\n", + " print(f\" - {opt['protocol']}: Yield Score {opt['yield_score']:.2f}, APY {opt['apy']:.1f}%\")\n", + " for suggestion in opt['suggestions']:\n", + " print(f\" → {suggestion}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Generate Reports and Visualize\n", + "\n", + "Generate comprehensive DeFi intelligence reports and visualizations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "owl_exporter = OWLExporter()\n", + "report_generator = ReportGenerator()\n", + "\n", + "# Export knowledge graph\n", + "kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"defi_kg.json\"))\n", + "kg_rdf = rdf_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"defi_kg.rdf\"))\n", + "\n", + "# Export ontology\n", + "ontology_owl = owl_exporter.export(defi_ontology, output_path=os.path.join(temp_dir, \"defi_ontology.owl\"))\n", + "\n", + "# Generate report\n", + "report_content = f\"\"\"\n", + "# DeFi Protocol Intelligence Report\n", + "\n", + "## Executive Summary\n", + "- Total Protocols Analyzed: {len(protocol_entities)}\n", + "- Total Pools: {len(pool_entities)}\n", + "- Total Tokens: {len(token_entities)}\n", + "- Protocol Communities: {community_count}\n", + "- High-Risk Protocols: {len([r for r in protocol_risks if r['risk_score'] >= 7])}\n", + "\n", + "## Top Protocols by Centrality\n", + "\"\"\"\n", + "for i, (protocol_id, centrality) in enumerate(top_central_protocols[:10], 1):\n", + " protocol_name = next((p[\"properties\"].get(\"name\", protocol_id) for p in protocol_entities if p[\"id\"] == protocol_id), protocol_id)\n", + " report_content += f\"\\n{i}. {protocol_name} (Centrality: {centrality:.3f})\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Risk Assessment\n", + "\"\"\"\n", + "for risk in sorted(protocol_risks, key=lambda x: x[\"risk_score\"], reverse=True):\n", + " report_content += f\"\"\"\n", + "### {risk['name']}\n", + "- Risk Score: {risk['risk_score']}/10\n", + "- TVL: ${risk['tvl']:,.0f}\n", + "- APY: {risk['apy']:.1f}%\n", + "- Risk Factors: {', '.join(risk['risk_factors'])}\n", + "\"\"\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Yield Optimization\n", + "\"\"\"\n", + "for opt in sorted(yield_optimization, key=lambda x: x[\"yield_score\"], reverse=True)[:10]:\n", + " report_content += f\"\"\"\n", + "### {opt['protocol']}\n", + "- Yield Score: {opt['yield_score']:.2f}\n", + "- APY: {opt['apy']:.1f}%\n", + "- TVL: ${opt['tvl']:,.0f}\n", + "- Suggestions:\n", + "\"\"\"\n", + " for suggestion in opt['suggestions']:\n", + " report_content += f\" - {suggestion}\\n\"\n", + "\n", + "report_path = os.path.join(temp_dir, \"defi_intelligence_report.md\")\n", + "with open(report_path, 'w') as f:\n", + " f.write(report_content)\n", + "\n", + "print(f\"Exported knowledge graph to JSON and RDF\")\n", + "print(f\"Exported ontology to OWL\")\n", + "print(f\"Generated intelligence report: {report_path}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Visualize DeFi Network\n", + "\n", + "Visualize the DeFi protocol network, ontology, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "ontology_visualizer = OntologyVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "kg_viz = kg_visualizer.visualize(\n", + " knowledge_graph,\n", + " layout=\"force_directed\",\n", + " highlight_nodes=[p[\"id\"] for p in protocol_entities],\n", + " node_size_by=\"tvl\"\n", + ")\n", + "\n", + "# Visualize ontology\n", + "ontology_viz = ontology_visualizer.visualize(\n", + " defi_ontology,\n", + " layout=\"hierarchical\"\n", + ")\n", + "\n", + "# Visualize analytics\n", + "analytics_viz = analytics_visualizer.visualize(\n", + " knowledge_graph,\n", + " metrics={\n", + " \"centrality\": dict(top_central_protocols[:10]),\n", + " \"communities\": communities,\n", + " \"connectivity\": connectivity_results,\n", + " \"risk_scores\": {r[\"protocol\"]: r[\"risk_score\"] for r in protocol_risks}\n", + " }\n", + ")\n", + "\n", + "print(\"Generated visualizations:\")\n", + "print(\" - Knowledge Graph: DeFi protocol network with TVL-based sizing\")\n", + "print(\" - Ontology Visualization: DeFi protocol ontology hierarchy\")\n", + "print(\" - Analytics Visualization: Centrality, communities, connectivity, and risk scores\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb b/cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb new file mode 100644 index 00000000..6fb8812b --- /dev/null +++ b/cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb @@ -0,0 +1,642 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Transaction Network Analysis Pipeline\n", + "\n", + "## Overview\n", + "\n", + "This notebook demonstrates a complete blockchain transaction network analysis pipeline: stream transactions from multiple sources (blockchain APIs, transaction feeds, databases), build temporal transaction knowledge graph, detect AML patterns, trace fund flows, and generate alerts.\n", + "\n", + "### Modules Used (20+)\n", + "\n", + "- **Ingestion**: StreamIngestor, WebIngestor, DBIngestor, FileIngestor\n", + "- **Parsing**: JSONParser, StructuredDataParser\n", + "- **Extraction**: NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "- **KG**: GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "- **Analytics**: CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n", + "- **Quality**: KGQualityAssessor, ConflictDetector\n", + "- **Export**: JSONExporter, RDFExporter, ReportGenerator\n", + "- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "\n", + "### Pipeline\n", + "\n", + "**Real-time Transaction Streams → Parse → Extract Entities (wallets, transactions, addresses) → Build Temporal Transaction KG → Detect Patterns (tumbling, mixing, clustering) → AML Analysis → Generate Alerts → Visualize**\n", + "\n", + "---\n", + "\n", + "## Step 1: Stream Transactions from Multiple Sources\n", + "\n", + "Stream blockchain transactions from APIs, feeds, and databases.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.ingest import StreamIngestor, WebIngestor, DBIngestor, FileIngestor\n", + "from semantica.parse import JSONParser, StructuredDataParser\n", + "from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, TripleExtractor\n", + "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", + "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", + "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "from semantica.kg import ConflictDetector\n", + "from semantica.export import JSONExporter, RDFExporter, ReportGenerator\n", + "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", + "import tempfile\n", + "import os\n", + "import json\n", + "from datetime import datetime, timedelta\n", + "\n", + "stream_ingestor = StreamIngestor()\n", + "web_ingestor = WebIngestor()\n", + "db_ingestor = DBIngestor()\n", + "file_ingestor = FileIngestor()\n", + "\n", + "json_parser = JSONParser()\n", + "structured_parser = StructuredDataParser()\n", + "\n", + "# Real streaming sources for blockchain transactions\n", + "stream_sources = [\n", + " {\n", + " \"type\": \"kafka\",\n", + " \"topic\": \"blockchain_transactions\",\n", + " \"bootstrap_servers\": [\"localhost:9092\"],\n", + " \"consumer_config\": {\"group_id\": \"transaction_analysis\"}\n", + " },\n", + " {\n", + " \"type\": \"rabbitmq\",\n", + " \"queue\": \"eth_transactions\",\n", + " \"connection_url\": \"amqp://user:password@localhost:5672/\"\n", + " }\n", + "]\n", + "\n", + "# Real blockchain APIs\n", + "blockchain_apis = [\n", + " \"https://api.etherscan.io/api?module=proxy&action=eth_getBlockByNumber&tag=latest&boolean=true&apikey=YourApiKeyToken\", # Etherscan API\n", + " \"https://blockchain.info/rawblock/000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f\", # Blockchain.com API\n", + " \"https://api.coingecko.com/api/v3/coins/ethereum\" # CoinGecko API\n", + "]\n", + "\n", + "# Real database connection for transaction history\n", + "db_connection_string = \"postgresql://user:password@localhost:5432/blockchain_db\"\n", + "db_query = \"SELECT tx_hash, from_address, to_address, value, timestamp, block_number FROM transactions WHERE timestamp > NOW() - INTERVAL '24 hours' ORDER BY timestamp DESC LIMIT 10000\"\n", + "\n", + "temp_dir = tempfile.mkdtemp()\n", + "\n", + "# Sample transaction data for local ingestion\n", + "transaction_data_file = os.path.join(temp_dir, \"transactions.json\")\n", + "transaction_data = [\n", + " {\n", + " \"tx_hash\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\",\n", + " \"from_address\": \"0xabc123def456abc123def456abc123def456abc12\",\n", + " \"to_address\": \"0xdef456abc123def456abc123def456abc123def45\",\n", + " \"value\": \"1000000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", + " \"block_number\": 18500000,\n", + " \"gas_used\": 21000\n", + " },\n", + " {\n", + " \"tx_hash\": \"0x2345678901bcdef2345678901bcdef2345678901bcdef2345678901bcdef23\",\n", + " \"from_address\": \"0xdef456abc123def456abc123def456abc123def45\",\n", + " \"to_address\": \"0x7890123456789012345678901234567890123456\",\n", + " \"value\": \"500000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=2)).isoformat(),\n", + " \"block_number\": 18499950,\n", + " \"gas_used\": 21000\n", + " },\n", + " {\n", + " \"tx_hash\": \"0x3456789012cdef3456789012cdef3456789012cdef3456789012cdef3456\",\n", + " \"from_address\": \"0x7890123456789012345678901234567890123456\",\n", + " \"to_address\": \"0xabc123def456abc123def456abc123def456abc12\",\n", + " \"value\": \"2000000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=3)).isoformat(),\n", + " \"block_number\": 18499900,\n", + " \"gas_used\": 21000\n", + " },\n", + " {\n", + " \"tx_hash\": \"0x4567890123def4567890123def4567890123def4567890123def4567890123\",\n", + " \"from_address\": \"0xabc123def456abc123def456abc123def456abc12\",\n", + " \"to_address\": \"0x4567890123456789012345678901234567890123\",\n", + " \"value\": \"300000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=4)).isoformat(),\n", + " \"block_number\": 18499850,\n", + " \"gas_used\": 21000\n", + " },\n", + " {\n", + " \"tx_hash\": \"0x5678901234ef5678901234ef5678901234ef5678901234ef5678901234ef56\",\n", + " \"from_address\": \"0x4567890123456789012345678901234567890123\",\n", + " \"to_address\": \"0x1234567890123456789012345678901234567890\",\n", + " \"value\": \"1500000000000000000\",\n", + " \"timestamp\": (datetime.now() - timedelta(hours=5)).isoformat(),\n", + " \"block_number\": 18499800,\n", + " \"gas_used\": 21000\n", + " }\n", + "]\n", + "\n", + "with open(transaction_data_file, 'w') as f:\n", + " json.dump(transaction_data, f, indent=2)\n", + "\n", + "# Ingest from local file\n", + "file_data = file_ingestor.ingest_file(transaction_data_file)\n", + "parsed_transactions = structured_parser.parse_json(json.dumps(transaction_data))\n", + "\n", + "# Ingest from blockchain APIs (example with public API)\n", + "try:\n", + " web_content = web_ingestor.ingest_url(blockchain_apis[2]) # CoinGecko public API\n", + " if web_content:\n", + " print(f\"✓ Ingested web content: {web_content.url if hasattr(web_content, 'url') else 'N/A'}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Web ingestion (example): {str(e)[:100]}\")\n", + "\n", + "# Database ingestion pattern (would connect to real database)\n", + "try:\n", + " db_data = db_ingestor.export_table(\n", + " connection_string=db_connection_string,\n", + " table_name=\"transactions\",\n", + " limit=10000\n", + " )\n", + " print(f\"✓ Database ingestion configured for: {db_connection_string}\")\n", + " print(f\" Query pattern: {db_query}\")\n", + "except Exception as e:\n", + " print(f\"⚠ Database connection (example pattern): Configure with real credentials\")\n", + " db_data = {\"data\": transaction_data}\n", + "\n", + "# Streaming ingestion pattern\n", + "print(f\"✓ Streaming sources configured:\")\n", + "for stream_source in stream_sources:\n", + " print(f\" - {stream_source['type']}: {stream_source.get('topic') or stream_source.get('queue')}\")\n", + "\n", + "print(f\"\\n📊 Ingestion Summary:\")\n", + "print(f\" Local transactions: {len(transaction_data)}\")\n", + "print(f\" Database records: {len(db_data.get('data', [])) if db_data else 0}\")\n", + "print(f\" Streaming sources: {len(stream_sources)}\")\n", + "print(f\" Web APIs: {len(blockchain_apis)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Extract Transaction Entities\n", + "\n", + "Extract wallets, transactions, and addresses from the ingested data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ner_extractor = NERExtractor()\n", + "relation_extractor = RelationExtractor()\n", + "event_detector = EventDetector()\n", + "triple_extractor = TripleExtractor()\n", + "\n", + "all_transaction_texts = []\n", + "all_transactions = []\n", + "\n", + "# Process parsed transactions\n", + "if parsed_transactions and isinstance(parsed_transactions, dict):\n", + " transactions = parsed_transactions.get(\"data\", transaction_data)\n", + " for tx in transactions:\n", + " all_transactions.append(tx)\n", + " tx_text = f\"Transaction {tx.get('tx_hash', '')} from {tx.get('from_address', '')} to {tx.get('to_address', '')} value {tx.get('value', '')} at {tx.get('timestamp', '')}\"\n", + " all_transaction_texts.append(tx_text)\n", + "\n", + "# Extract entities\n", + "all_entities = []\n", + "all_relationships = []\n", + "all_events = []\n", + "all_triples = []\n", + "\n", + "for text in all_transaction_texts:\n", + " entities = ner_extractor.extract(text)\n", + " all_entities.extend(entities)\n", + " \n", + " relationships = relation_extractor.extract(text, entities)\n", + " all_relationships.extend(relationships)\n", + " \n", + " events = event_detector.detect_events(text)\n", + " all_events.extend(events)\n", + " \n", + " triples = triple_extractor.extract(text)\n", + " all_triples.extend(triples)\n", + "\n", + "# Build structured entity list\n", + "transaction_entities = []\n", + "wallet_entities = []\n", + "\n", + "for tx in all_transactions:\n", + " tx_entity = {\n", + " \"id\": tx.get(\"tx_hash\", \"\"),\n", + " \"type\": \"Transaction\",\n", + " \"properties\": {\n", + " \"from_address\": tx.get(\"from_address\", \"\"),\n", + " \"to_address\": tx.get(\"to_address\", \"\"),\n", + " \"value\": tx.get(\"value\", \"\"),\n", + " \"timestamp\": tx.get(\"timestamp\", \"\"),\n", + " \"block_number\": tx.get(\"block_number\", 0),\n", + " \"gas_used\": tx.get(\"gas_used\", 0)\n", + " }\n", + " }\n", + " transaction_entities.append(tx_entity)\n", + " \n", + " # Add wallet entities\n", + " from_wallet = {\n", + " \"id\": tx.get(\"from_address\", \"\"),\n", + " \"type\": \"Wallet\",\n", + " \"properties\": {\n", + " \"address\": tx.get(\"from_address\", \"\"),\n", + " \"role\": \"sender\"\n", + " }\n", + " }\n", + " to_wallet = {\n", + " \"id\": tx.get(\"to_address\", \"\"),\n", + " \"type\": \"Wallet\",\n", + " \"properties\": {\n", + " \"address\": tx.get(\"to_address\", \"\"),\n", + " \"role\": \"receiver\"\n", + " }\n", + " }\n", + " wallet_entities.append(from_wallet)\n", + " wallet_entities.append(to_wallet)\n", + "\n", + "# Deduplicate wallets\n", + "unique_wallets = {}\n", + "for wallet in wallet_entities:\n", + " wallet_id = wallet[\"id\"]\n", + " if wallet_id not in unique_wallets:\n", + " unique_wallets[wallet_id] = wallet\n", + "\n", + "wallet_entities = list(unique_wallets.values())\n", + "\n", + "print(f\"Extracted {len(transaction_entities)} transactions\")\n", + "print(f\"Extracted {len(wallet_entities)} unique wallets\")\n", + "print(f\"Extracted {len(all_relationships)} relationships\")\n", + "print(f\"Detected {len(all_events)} events\")\n", + "print(f\"Extracted {len(all_triples)} triples\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Build Temporal Transaction Knowledge Graph\n", + "\n", + "Build a temporal knowledge graph from extracted transactions and wallets.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", + "\n", + "# Add all entities\n", + "for wallet in wallet_entities:\n", + " builder.add_entity(\n", + " entity_id=wallet[\"id\"],\n", + " entity_type=wallet[\"type\"],\n", + " properties=wallet.get(\"properties\", {})\n", + " )\n", + "\n", + "for tx in transaction_entities:\n", + " builder.add_entity(\n", + " entity_id=tx[\"id\"],\n", + " entity_type=tx[\"type\"],\n", + " properties=tx.get(\"properties\", {})\n", + " )\n", + "\n", + "# Add relationships\n", + "relationships = []\n", + "for tx in transaction_entities:\n", + " from_addr = tx[\"properties\"].get(\"from_address\", \"\")\n", + " to_addr = tx[\"properties\"].get(\"to_address\", \"\")\n", + " tx_hash = tx[\"id\"]\n", + " value = tx[\"properties\"].get(\"value\", \"\")\n", + " timestamp = tx[\"properties\"].get(\"timestamp\", \"\")\n", + " \n", + " # Transaction relationship\n", + " rel = {\n", + " \"source\": from_addr,\n", + " \"target\": to_addr,\n", + " \"type\": \"transfers_to\",\n", + " \"properties\": {\n", + " \"transaction\": tx_hash,\n", + " \"value\": value,\n", + " \"timestamp\": timestamp\n", + " }\n", + " }\n", + " relationships.append(rel)\n", + " builder.add_relationship(\n", + " source_id=from_addr,\n", + " target_id=to_addr,\n", + " relationship_type=\"transfers_to\",\n", + " properties=rel[\"properties\"]\n", + " )\n", + " \n", + " # Transaction entity relationship\n", + " builder.add_relationship(\n", + " source_id=from_addr,\n", + " target_id=tx_hash,\n", + " relationship_type=\"initiates\",\n", + " properties={\"timestamp\": timestamp}\n", + " )\n", + " builder.add_relationship(\n", + " source_id=tx_hash,\n", + " target_id=to_addr,\n", + " relationship_type=\"sends_to\",\n", + " properties={\"timestamp\": timestamp}\n", + " )\n", + "\n", + "knowledge_graph = builder.build()\n", + "\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.nodes)} nodes\")\n", + "print(f\"Built knowledge graph with {len(knowledge_graph.edges)} edges\")\n", + "print(f\"Added {len(relationships)} transaction relationships\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Detect AML Patterns\n", + "\n", + "Detect money laundering patterns: tumbling, mixing, clustering, and suspicious flows.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temporal_query = TemporalGraphQuery(knowledge_graph)\n", + "pattern_detector = TemporalPatternDetector(knowledge_graph)\n", + "graph_analyzer = GraphAnalyzer(knowledge_graph)\n", + "centrality_calculator = CentralityCalculator(knowledge_graph)\n", + "community_detector = CommunityDetector(knowledge_graph)\n", + "connectivity_analyzer = ConnectivityAnalyzer(knowledge_graph)\n", + "\n", + "# Query transactions in time range\n", + "start_time = (datetime.now() - timedelta(hours=6)).isoformat()\n", + "end_time = datetime.now().isoformat()\n", + "\n", + "temporal_results = temporal_query.query_time_range(\n", + " start_time=start_time,\n", + " end_time=end_time,\n", + " relationship_types=[\"transfers_to\", \"initiates\", \"sends_to\"]\n", + ")\n", + "\n", + "# Detect temporal patterns\n", + "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", + " relationship_types=[\"transfers_to\"],\n", + " time_window_hours=6\n", + ")\n", + "\n", + "# Calculate centrality to find key wallets\n", + "centrality_scores = centrality_calculator.calculate_centrality(centrality_type=\"betweenness\")\n", + "top_central_wallets = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n", + "\n", + "# Detect communities (clustering)\n", + "communities = community_detector.detect_communities()\n", + "community_count = len(set(communities.values())) if communities else 0\n", + "\n", + "# Analyze connectivity\n", + "connectivity_results = connectivity_analyzer.analyze_connectivity()\n", + "\n", + "# AML Pattern Detection using Inference Engine\n", + "inference_engine = InferenceEngine()\n", + "rule_manager = RuleManager()\n", + "\n", + "# Define AML rules\n", + "aml_rules = [\n", + " {\n", + " \"name\": \"tumbling_pattern\",\n", + " \"condition\": \"high_transaction_count AND multiple_intermediate_wallets\",\n", + " \"action\": \"flag_as_tumbling\"\n", + " },\n", + " {\n", + " \"name\": \"mixing_pattern\",\n", + " \"condition\": \"funds_split_into_multiple_addresses AND rapid_consolidation\",\n", + " \"action\": \"flag_as_mixing\"\n", + " },\n", + " {\n", + " \"name\": \"suspicious_flow\",\n", + " \"condition\": \"large_value_transfer AND short_time_window\",\n", + " \"action\": \"flag_as_suspicious\"\n", + " }\n", + "]\n", + "\n", + "for rule in aml_rules:\n", + " rule_manager.add_rule(rule[\"name\"], rule[\"condition\"], rule[\"action\"])\n", + "\n", + "# Add facts from graph analysis\n", + "aml_facts = []\n", + "for wallet_id, centrality in top_central_wallets[:5]:\n", + " aml_facts.append({\n", + " \"wallet\": wallet_id,\n", + " \"centrality\": centrality,\n", + " \"high_transaction_count\": True if centrality > 0.1 else False\n", + " })\n", + "\n", + "# Detect patterns\n", + "suspicious_patterns = []\n", + "for wallet_id, centrality in top_central_wallets:\n", + " if centrality > 0.15:\n", + " suspicious_patterns.append({\n", + " \"wallet\": wallet_id,\n", + " \"pattern\": \"high_centrality\",\n", + " \"risk_score\": min(centrality * 10, 10),\n", + " \"description\": f\"Wallet {wallet_id[:10]}... has high betweenness centrality ({centrality:.3f}), indicating potential mixing/tumbling\"\n", + " })\n", + "\n", + "# Check for rapid transactions (tumbling pattern)\n", + "wallet_transaction_counts = {}\n", + "for rel in relationships:\n", + " source = rel[\"source\"]\n", + " wallet_transaction_counts[source] = wallet_transaction_counts.get(source, 0) + 1\n", + "\n", + "for wallet_id, count in wallet_transaction_counts.items():\n", + " if count >= 3:\n", + " suspicious_patterns.append({\n", + " \"wallet\": wallet_id,\n", + " \"pattern\": \"rapid_transactions\",\n", + " \"risk_score\": min(count * 2, 10),\n", + " \"description\": f\"Wallet {wallet_id[:10]}... has {count} outgoing transactions, potential tumbling\"\n", + " })\n", + "\n", + "print(f\"Detected {len(temporal_patterns)} temporal patterns\")\n", + "print(f\"Found {community_count} wallet communities\")\n", + "print(f\"Identified {len(suspicious_patterns)} suspicious patterns\")\n", + "print(f\"\\nTop 5 Central Wallets:\")\n", + "for i, (wallet_id, centrality) in enumerate(top_central_wallets[:5], 1):\n", + " print(f\" {i}. {wallet_id[:20]}... (centrality: {centrality:.3f})\")\n", + "print(f\"\\nSuspicious Patterns Detected:\")\n", + "for pattern in suspicious_patterns[:5]:\n", + " print(f\" - {pattern['pattern']}: {pattern['wallet'][:20]}... (risk: {pattern['risk_score']:.1f}/10)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Alerts and Reports\n", + "\n", + "Generate AML alerts and comprehensive analysis reports.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "rdf_exporter = RDFExporter()\n", + "report_generator = ReportGenerator()\n", + "kg_quality_assessor = KGQualityAssessor()\n", + "conflict_detector = ConflictDetector(knowledge_graph)\n", + "\n", + "# Assess graph quality\n", + "quality_metrics = kg_quality_assessor.assess_quality(knowledge_graph)\n", + "\n", + "# Detect conflicts\n", + "conflicts = conflict_detector.detect_conflicts()\n", + "\n", + "# Generate alerts\n", + "alerts = []\n", + "for pattern in suspicious_patterns:\n", + " if pattern[\"risk_score\"] >= 5.0:\n", + " alerts.append({\n", + " \"alert_id\": f\"AML_{pattern['wallet'][:8]}\",\n", + " \"type\": \"AML_SUSPICIOUS_PATTERN\",\n", + " \"severity\": \"HIGH\" if pattern[\"risk_score\"] >= 7.0 else \"MEDIUM\",\n", + " \"wallet\": pattern[\"wallet\"],\n", + " \"pattern\": pattern[\"pattern\"],\n", + " \"risk_score\": pattern[\"risk_score\"],\n", + " \"description\": pattern[\"description\"],\n", + " \"timestamp\": datetime.now().isoformat()\n", + " })\n", + "\n", + "# Export knowledge graph\n", + "kg_json = json_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"transaction_kg.json\"))\n", + "kg_rdf = rdf_exporter.export(knowledge_graph, output_path=os.path.join(temp_dir, \"transaction_kg.rdf\"))\n", + "\n", + "# Generate report\n", + "report_content = f\"\"\"\n", + "# Blockchain Transaction Network Analysis Report\n", + "\n", + "## Executive Summary\n", + "- Total Transactions Analyzed: {len(transaction_entities)}\n", + "- Unique Wallets: {len(wallet_entities)}\n", + "- Suspicious Patterns Detected: {len(suspicious_patterns)}\n", + "- High-Risk Alerts: {len([a for a in alerts if a['severity'] == 'HIGH'])}\n", + "\n", + "## Graph Quality Metrics\n", + "- Nodes: {quality_metrics.get('node_count', len(knowledge_graph.nodes))}\n", + "- Edges: {quality_metrics.get('edge_count', len(knowledge_graph.edges))}\n", + "- Completeness: {quality_metrics.get('completeness', 0):.2%}\n", + "- Consistency: {quality_metrics.get('consistency', 0):.2%}\n", + "\n", + "## Top Suspicious Patterns\n", + "\"\"\"\n", + "for i, pattern in enumerate(suspicious_patterns[:10], 1):\n", + " report_content += f\"\"\"\n", + "### {i}. {pattern['pattern'].upper()}\n", + "- Wallet: {pattern['wallet']}\n", + "- Risk Score: {pattern['risk_score']:.1f}/10\n", + "- Description: {pattern['description']}\n", + "\"\"\"\n", + "\n", + "report_content += f\"\"\"\n", + "## Alerts Generated\n", + "\"\"\"\n", + "for alert in alerts:\n", + " report_content += f\"\"\"\n", + "- **{alert['alert_id']}** ({alert['severity']}): {alert['description']}\n", + "\"\"\"\n", + "\n", + "report_path = os.path.join(temp_dir, \"aml_analysis_report.md\")\n", + "with open(report_path, 'w') as f:\n", + " f.write(report_content)\n", + "\n", + "print(f\"Generated {len(alerts)} AML alerts\")\n", + "print(f\"Exported knowledge graph to JSON and RDF\")\n", + "print(f\"Generated analysis report: {report_path}\")\n", + "print(f\"\\nQuality Metrics:\")\n", + "print(f\" Nodes: {quality_metrics.get('node_count', len(knowledge_graph.nodes))}\")\n", + "print(f\" Edges: {quality_metrics.get('edge_count', len(knowledge_graph.edges))}\")\n", + "print(f\" Completeness: {quality_metrics.get('completeness', 0):.2%}\")\n", + "print(f\" Consistency: {quality_metrics.get('consistency', 0):.2%}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Visualize Transaction Network\n", + "\n", + "Visualize the transaction network, patterns, and analytics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "temporal_visualizer = TemporalVisualizer()\n", + "analytics_visualizer = AnalyticsVisualizer()\n", + "\n", + "# Visualize knowledge graph\n", + "kg_viz = kg_visualizer.visualize(\n", + " knowledge_graph,\n", + " layout=\"force_directed\",\n", + " highlight_nodes=[p[\"wallet\"] for p in suspicious_patterns[:5]],\n", + " node_size_by=\"centrality\"\n", + ")\n", + "\n", + "# Visualize temporal patterns\n", + "temporal_viz = temporal_visualizer.visualize(\n", + " knowledge_graph,\n", + " time_attribute=\"timestamp\",\n", + " relationship_types=[\"transfers_to\"]\n", + ")\n", + "\n", + "# Visualize analytics\n", + "analytics_viz = analytics_visualizer.visualize(\n", + " knowledge_graph,\n", + " metrics={\n", + " \"centrality\": dict(top_central_wallets[:10]),\n", + " \"communities\": communities,\n", + " \"connectivity\": connectivity_results\n", + " }\n", + ")\n", + "\n", + "print(\"Generated visualizations:\")\n", + "print(\" - Knowledge Graph: Transaction network with highlighted suspicious wallets\")\n", + "print(\" - Temporal Visualization: Transaction flows over time\")\n", + "print(\" - Analytics Visualization: Centrality, communities, and connectivity metrics\")\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}