mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-12 04:01:35 +00:00
34 KiB
34 KiB
In [ ]:
# Import all Semantica modules for intelligence analysis with Orchestrator-Worker pattern
from semantica.pipeline import PipelineBuilder, ExecutionEngine, ParallelismManager
from semantica.ingest import FileIngestor, DBIngestor, WebIngestor, StreamIngestor, FeedIngestor
from semantica.parse import JSONParser, XMLParser, CSVParser, DocumentParser, StructuredDataParser
from semantica.normalize import TextNormalizer, DataNormalizer
from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, EventDetector
from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OWLGenerator, OntologyValidator
from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphAnalyzer, ConnectivityAnalyzer
from semantica.embeddings import EmbeddingGenerator, TextEmbedder
from semantica.vector_store import VectorStore, HybridSearch, MetadataFilter
from semantica.context import AgentMemory, ContextRetriever, ContextGraphBuilder
from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator
from semantica.export import ReportGenerator, JSONExporter
from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer
import tempfile
import os
import json
from datetime import datetime, timedelta
import numpy as np
In [ ]:
# Initialize orchestrator
orchestrator = ExecutionEngine()
pipeline_builder = PipelineBuilder()
parallelism_manager = ParallelismManager(max_workers=7)
# Initialize vector store and agent memory
vector_store = VectorStore(backend="faiss", dimension=768)
agent_memory = AgentMemory(
vector_store=vector_store,
retention_policy="unlimited",
max_memory_size=10000
)
# Create temporary directory for data
temp_dir = tempfile.mkdtemp()
In [ ]:
# Worker 1: Data Ingestion Worker
def data_ingestion_worker(sources, memory):
"""Multi-source data ingestion worker."""
file_ingestor = FileIngestor()
web_ingestor = WebIngestor()
feed_ingestor = FeedIngestor()
ingested_data = []
# Sample OSINT and threat intelligence data
osint_data = {
"threat_actors": [
{"name": "Threat Actor A", "type": "APT", "location": "Country X"},
{"name": "Threat Actor B", "type": "Cybercriminal", "location": "Country Y"}
],
"indicators": [
{"type": "IP", "value": "192.168.1.100", "threat_level": "high"},
{"type": "Domain", "value": "malicious.com", "threat_level": "medium"}
]
}
# Save sample data
osint_file = os.path.join(temp_dir, "osint_data.json")
with open(osint_file, 'w') as f:
json.dump(osint_data, f, indent=2)
ingested_data.append(file_ingestor.ingest_file(osint_file, read_content=True))
# Store in memory
memory.store(
f"Data ingestion worker processed {len(ingested_data)} sources",
metadata={"worker": "data_ingestion", "sources_count": len(ingested_data)}
)
return {"ingested_data": ingested_data, "osint_data": osint_data}
# Test worker
osint_data_result = data_ingestion_worker([], agent_memory)
print(f" - Sources processed: {len(osint_data_result.get('ingested_data', []))}")
print(f" - OSINT data ingested: {len(osint_data_result.get('osint_data', {}).get('threat_actors', []))} threat actors")
In [ ]:
# Worker 2: Ontology Building Worker
def ontology_building_worker(entities, relationships, memory):
"""Complete 6-stage ontology generation worker."""
ontology_gen = OntologyGenerator(base_uri="https://example.org/intelligence/ontology/")
class_inferrer = ClassInferrer()
property_generator = PropertyGenerator()
owl_generator = OWLGenerator()
ontology_validator = OntologyValidator()
# Stage 1-6: Complete ontology generation pipeline
ontology = ontology_gen.generate_ontology({
"entities": entities,
"relationships": relationships
})
# Generate OWL/Turtle
owl_content = owl_generator.generate_owl(ontology, format="turtle")
# Validate ontology
validation_result = ontology_validator.validate_ontology(ontology, method="hermit")
# Store in memory
memory.store(
f"Ontology building worker generated ontology with {len(ontology.get('classes', [])) if isinstance(ontology, dict) else 0} classes",
metadata={
"worker": "ontology_building",
"classes_count": len(ontology.get('classes', [])) if isinstance(ontology, dict) else 0,
"validation_passed": validation_result.get('is_valid', False) if isinstance(validation_result, dict) else False
}
)
return {
"ontology": ontology,
"owl_content": owl_content,
"validation": validation_result
}
# Prepare sample entities and relationships for ontology
sample_entities = [
{"id": "E1", "type": "ThreatActor", "name": "Threat Actor A"},
{"id": "E2", "type": "Indicator", "name": "Malicious IP"},
{"id": "E3", "type": "Location", "name": "Country X"}
]
sample_relationships = [
{"source": "E1", "target": "E2", "type": "uses"},
{"source": "E1", "target": "E3", "type": "located_in"}
]
ontology_result = ontology_building_worker(sample_entities, sample_relationships, agent_memory)
print(f" - Ontology generated: {bool(ontology_result.get('ontology'))}")
print(f" - OWL content generated: {len(ontology_result.get('owl_content', ''))} characters")
print(f" - Validation passed: {ontology_result.get('validation', {}).get('is_valid', False) if isinstance(ontology_result.get('validation'), dict) else False}")
In [ ]:
# Worker 3: Graph Construction Worker
def graph_construction_worker(entities, relationships, memory):
"""Build knowledge graph worker."""
graph_builder = GraphBuilder()
temporal_query = TemporalGraphQuery()
# Build knowledge graph
kg = graph_builder.build(entities, relationships)
# Build temporal graph query capability
temporal_kg = temporal_query.build_temporal_graph(kg)
# Store in memory
memory.store(
f"Graph construction worker built KG with {len(entities)} entities and {len(relationships)} relationships",
metadata={
"worker": "graph_construction",
"entity_count": len(entities),
"relationship_count": len(relationships)
},
entities=entities,
relationships=relationships
)
return {"knowledge_graph": kg, "temporal_graph": temporal_kg}
# Test worker with sample data
graph_result = graph_construction_worker(sample_entities, sample_relationships, agent_memory)
print(f" - Knowledge graph built: {bool(graph_result.get('knowledge_graph'))}")
print(f" - Temporal graph built: {bool(graph_result.get('temporal_graph'))}")
In [ ]:
# Worker 4: Graph Analytics Worker
def graph_analytics_worker(kg, memory):
"""Comprehensive graph analytics worker."""
graph_analyzer = GraphAnalyzer()
connectivity_analyzer = ConnectivityAnalyzer()
# All centrality measures
pagerank = graph_analyzer.compute_centrality(kg, method="pagerank")
betweenness = graph_analyzer.compute_centrality(kg, method="betweenness")
closeness = graph_analyzer.compute_centrality(kg, method="closeness")
eigenvector = graph_analyzer.compute_centrality(kg, method="eigenvector")
# Community detection
communities = graph_analyzer.detect_communities(kg, method="louvain")
# Connectivity analysis
connectivity = connectivity_analyzer.analyze_connectivity(kg)
# Graph metrics
metrics = graph_analyzer.compute_metrics(kg)
# Store in memory
memory.store(
f"Graph analytics worker completed: {len(pagerank)} entities analyzed",
metadata={
"worker": "graph_analytics",
"centrality_measures": 4,
"communities": communities.get('num_communities', 0) if isinstance(communities, dict) else 0
}
)
return {
"pagerank": pagerank,
"betweenness": betweenness,
"closeness": closeness,
"eigenvector": eigenvector,
"communities": communities,
"connectivity": connectivity,
"metrics": metrics
}
# Test worker
analytics_result = graph_analytics_worker(graph_result.get('knowledge_graph'), agent_memory)
print(f" - PageRank computed: {len(analytics_result.get('pagerank', {}))}")
print(f" - Betweenness computed: {len(analytics_result.get('betweenness', {}))}")
print(f" - Closeness computed: {len(analytics_result.get('closeness', {}))}")
print(f" - Eigenvector computed: {len(analytics_result.get('eigenvector', {}))}")
print(f" - Communities detected: {analytics_result.get('communities', {}).get('num_communities', 0) if isinstance(analytics_result.get('communities'), dict) else 0}")
In [ ]:
# Worker 5: Hybrid RAG Worker
def hybrid_rag_worker(kg, vector_store, entities, memory):
"""Complete hybrid RAG worker."""
embedding_generator = EmbeddingGenerator()
text_embedder = TextEmbedder()
hybrid_search = HybridSearch(vector_store=vector_store, knowledge_graph=kg)
context_retriever = ContextRetriever()
# Generate embeddings for entities
entity_embeddings = {}
for entity in entities:
entity_text = f"{entity.get('name', '')} {entity.get('type', '')}"
embedding = text_embedder.embed(entity_text)
entity_embeddings[entity.get('id', '')] = embedding
# Store embeddings in vector store
entity_metadata = [
{"entity_id": eid, "name": next((e.get('name', '') for e in entities if e.get('id') == eid), '')}
for eid in entity_embeddings.keys()
]
vector_store.store_vectors(
vectors=list(entity_embeddings.values()),
metadata=entity_metadata
)
# Example hybrid search query
query = "What are the key threat actors?"
hybrid_results = hybrid_search.search(
query=query,
top_k=5,
use_graph=True,
use_vectors=True
)
# Context retrieval
context = context_retriever.retrieve(
query=query,
knowledge_graph=kg,
top_k=10
)
# Store in memory
memory.store(
f"Hybrid RAG worker setup: {len(entity_embeddings)} embeddings, hybrid search ready",
metadata={
"worker": "hybrid_rag",
"embeddings_count": len(entity_embeddings),
"hybrid_search": True
}
)
return {
"embeddings": entity_embeddings,
"hybrid_search": hybrid_search,
"query_results": hybrid_results,
"context": context
}
# Test worker
rag_result = hybrid_rag_worker(
graph_result.get('knowledge_graph'),
vector_store,
sample_entities,
agent_memory
)
print(f" - Entity embeddings: {len(rag_result.get('embeddings', {}))}")
print(f" - Hybrid search ready: {bool(rag_result.get('hybrid_search'))}")
print(f" - Query results: {len(rag_result.get('query_results', [])) if isinstance(rag_result.get('query_results'), list) else 1}")
In [ ]:
# Worker 6: Intelligence Analysis Worker
def intelligence_analysis_worker(kg, analytics_results, memory):
"""Threat assessment and geospatial analysis worker."""
inference_engine = InferenceEngine()
rule_manager = RuleManager()
# Threat assessment rules
threat_rules = [
{
"rule_id": "high_centrality_threat",
"condition": "IF entity has high_pagerank AND entity is threat_actor THEN entity is high_priority_threat",
"action": "flag_high_priority"
},
{
"rule_id": "geospatial_cluster",
"condition": "IF entities in same_location AND high_connectivity THEN entities form_geospatial_cluster",
"action": "identify_cluster"
}
]
# Pattern detection
patterns = inference_engine.infer(knowledge_graph=kg, rules=threat_rules)
# Threat scoring based on centrality
threat_scores = {}
pagerank = analytics_results.get('pagerank', {})
for entity_id, score in pagerank.items():
threat_scores[entity_id] = score * 100 # Scale to 0-100
# Store in memory
memory.store(
f"Intelligence analysis worker: {len(patterns) if isinstance(patterns, list) else 1} patterns detected",
metadata={
"worker": "intelligence_analysis",
"patterns_count": len(patterns) if isinstance(patterns, list) else 1,
"threat_scores": len(threat_scores)
}
)
return {
"patterns": patterns,
"threat_scores": threat_scores,
"rules": threat_rules
}
# Test worker
intelligence_result = intelligence_analysis_worker(
graph_result.get('knowledge_graph'),
analytics_result,
agent_memory
)
print(f" - Patterns detected: {len(intelligence_result.get('patterns', [])) if isinstance(intelligence_result.get('patterns'), list) else 1}")
print(f" - Threat scores computed: {len(intelligence_result.get('threat_scores', {}))}")
In [ ]:
# Worker 7: Report Generation Worker
def report_generation_worker(all_results, memory):
"""Compile comprehensive intelligence report worker."""
report_generator = ReportGenerator()
# Retrieve all context from memory
context = memory.retrieve("intelligence analysis", max_results=20)
# Compile report data
report_data = {
"title": "Intelligence Analysis Report",
"executive_summary": "Comprehensive intelligence analysis using Orchestrator-Worker pattern",
"ontology": all_results.get('ontology', {}),
"knowledge_graph": {
"entities": len(all_results.get('graph', {}).get('knowledge_graph', {}).get('entities', [])),
"relationships": len(all_results.get('graph', {}).get('knowledge_graph', {}).get('relationships', []))
},
"graph_analytics": all_results.get('analytics', {}),
"hybrid_rag": {
"embeddings_count": len(all_results.get('rag', {}).get('embeddings', {}))
},
"intelligence_analysis": all_results.get('intelligence', {}),
"agent_memory_stats": memory.get_statistics(),
"context_items": len(context)
}
# Store in memory
memory.store(
"Report generation worker compiled comprehensive intelligence report",
metadata={"worker": "report_generation", "context_items": len(context)}
)
return {"report_data": report_data, "context": context}
# Prepare all results for report generation
all_worker_results = {
"ontology": ontology_result,
"graph": graph_result,
"analytics": analytics_result,
"rag": rag_result,
"intelligence": intelligence_result
}
report_result = report_generation_worker(all_worker_results, agent_memory)
print(f" - Report data compiled: {bool(report_result.get('report_data'))}")
print(f" - Context items: {len(report_result.get('context', []))}")
In [ ]:
def data_ingestion_handler(data, **config):
sources = data.get("sources", [])
memory = data.get("memory")
r = data_ingestion_worker(sources, memory)
return {**data, "osint": r}
def ontology_handler(data, **config):
entities = data.get("entities", [])
relationships = data.get("relationships", [])
memory = data.get("memory")
r = ontology_building_worker(entities, relationships, memory)
return {**data, "ontology_result": r}
def graph_handler(data, **config):
entities = data.get("entities", [])
relationships = data.get("relationships", [])
memory = data.get("memory")
r = graph_construction_worker(entities, relationships, memory)
return {**data, "graph_result": r}
def analytics_handler(data, **config):
kg = data.get("graph_result", {}).get("knowledge_graph")
memory = data.get("memory")
r = graph_analytics_worker(kg, memory)
return {**data, "analytics_result": r}
def rag_handler(data, **config):
kg = data.get("graph_result", {}).get("knowledge_graph")
vector_store = data.get("vector_store")
entities = data.get("entities", [])
memory = data.get("memory")
r = hybrid_rag_worker(kg, vector_store, entities, memory)
return {**data, "rag_result": r}
def intel_handler(data, **config):
kg = data.get("graph_result", {}).get("knowledge_graph")
analytics_results = data.get("analytics_result")
memory = data.get("memory")
r = intelligence_analysis_worker(kg, analytics_results, memory)
return {**data, "intelligence_result": r}
def report_handler(data, **config):
memory = data.get("memory")
all_results = {"ontology": data.get("ontology_result"), "graph": data.get("graph_result"), "analytics": data.get("analytics_result"), "rag": data.get("rag_result"), "intelligence": data.get("intelligence_result")}
r = report_generation_worker(all_results, memory)
return {**data, "report_result": r}
intelligence_pipeline = (
pipeline_builder
.add_step("data_ingestion", "ingest", handler=data_ingestion_handler)
.add_step("ontology_building", "ontology", dependencies=["data_ingestion"], handler=ontology_handler)
.add_step("graph_construction", "build_graph", dependencies=["ontology_building"], handler=graph_handler)
.add_step("graph_analytics", "analyze_graph", dependencies=["graph_construction"], handler=analytics_handler)
.add_step("hybrid_rag", "rag", dependencies=["graph_construction"], handler=rag_handler)
.add_step("intelligence_analysis", "analysis", dependencies=["graph_analytics", "hybrid_rag"], handler=intel_handler)
.add_step("report_generation", "report", dependencies=["intelligence_analysis"], handler=report_handler)
)
.build()
input_data = {"sources": [], "entities": sample_entities, "relationships": sample_relationships, "vector_store": vector_store, "memory": agent_memory}
pipeline_result = orchestrator.execute_pipeline(intelligence_pipeline, data=input_data, parallel=True, max_workers=7)
print(f" - Pipeline steps: {len(intelligence_pipeline.steps)}")
print(f" - Execution status: {getattr(pipeline_result, 'success', True)}")
In [ ]:
# Generate final intelligence report
report_generator = ReportGenerator()
intelligence_report_file = os.path.join(temp_dir, "intelligence_analysis_report.html")
report_generator.generate_report(
report_result.get('report_data'),
intelligence_report_file,
format="html"
)
print(f" - Report file: {intelligence_report_file}")
print(f" - Report includes: Ontology, KG, Graph Analytics, Hybrid RAG, Intelligence Analysis, Agent Memory Stats")