mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-13 04:04:09 +00:00
20 KiB
20 KiB
In [ ]:
from semantica.ingest import DBIngestor, FileIngestor, WebIngestor, FeedIngestor
from semantica.parse import JSONParser, CSVParser, StructuredDataParser
from semantica.semantic_extract import NERExtractor, RelationExtractor, EventDetector, SemanticAnalyzer
from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer
from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer
from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator
from semantica.kg_qa import KGQualityAssessor
from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator
from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer
import tempfile
import os
import json
from datetime import datetime, timedelta
db_ingestor = DBIngestor()
file_ingestor = FileIngestor()
web_ingestor = WebIngestor()
feed_ingestor = FeedIngestor()
json_parser = JSONParser()
csv_parser = CSVParser()
structured_parser = StructuredDataParser()
# Real historical market data APIs
historical_market_apis = [
"https://api.polygon.io/v2/aggs/ticker/AAPL/range/1/day/2023-01-01/2024-01-01", # Polygon.io historical
"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=AAPL&apikey=demo", # Alpha Vantage historical
"https://api.github.com/repos/ranaroussi/yfinance" # Yahoo Finance historical data
]
# Real financial news feeds for historical context
historical_feeds = [
"https://feeds.reuters.com/reuters/businessNews",
"https://rss.cnn.com/rss/money_latest.rss",
"https://feeds.bloomberg.com/markets/news.rss"
]
# Real database connection for historical market data
db_connection_string = "postgresql://user:password@localhost:5432/historical_market_db"
db_query = "SELECT symbol, date, open, high, low, close, volume FROM historical_prices WHERE date >= '2023-01-01' AND date <= '2024-01-01' ORDER BY date DESC"
temp_dir = tempfile.mkdtemp()
# Sample historical market data (simulating real historical data structure)
historical_data_file = os.path.join(temp_dir, "historical_data.json")
historical_data = [
{"symbol": "AAPL", "date": "2023-01-15", "open": 150.00, "high": 152.00, "low": 149.50, "close": 151.50, "volume": 50000000},
{"symbol": "AAPL", "date": "2023-01-16", "open": 151.50, "high": 153.00, "low": 151.00, "close": 152.75, "volume": 52000000},
{"symbol": "MSFT", "date": "2023-01-15", "open": 350.00, "high": 352.00, "low": 349.50, "close": 351.25, "volume": 30000000},
{"symbol": "MSFT", "date": "2023-01-16", "open": 351.25, "high": 353.50, "low": 350.75, "close": 352.50, "volume": 31000000}
]
with open(historical_data_file, 'w') as f:
json.dump(historical_data, f, indent=2)
file_objects = file_ingestor.ingest_file(historical_data_file, read_content=True)
parsed_data = structured_parser.parse_json(historical_data_file)
# Ingest from historical market APIs
historical_api_list = []
for api_url in historical_market_apis[:1]:
try:
api_content = web_ingestor.ingest_url(api_url)
if api_content:
historical_api_list.append(api_content)
print(f"✓ Ingested historical market API: {api_content.url if hasattr(api_content, 'url') else api_url}")
except Exception as e:
print(f"⚠ Historical API ingestion for {api_url}: {str(e)[:100]}")
# Ingest from historical news feeds
historical_feed_list = []
for feed_url in historical_feeds:
try:
feed_data = feed_ingestor.ingest_feed(feed_url)
if feed_data:
historical_feed_list.append(feed_data)
print(f"✓ Ingested historical feed: {feed_data.title if hasattr(feed_data, 'title') else feed_url}")
except Exception as e:
print(f"⚠ Feed ingestion for {feed_url}: {str(e)[:100]}")
print(f"\n📊 Historical Data Ingestion Summary:")
print(f" Historical data files: {len([file_objects]) if file_objects else 0}")
print(f" Historical market APIs: {len(historical_api_list)}")
print(f" Historical feeds: {len(historical_feed_list)}")
print(f" Database sources: 1")
In [ ]:
ner_extractor = NERExtractor()
relation_extractor = RelationExtractor()
event_detector = EventDetector()
semantic_analyzer = SemanticAnalyzer()
historical_entities = []
historical_relationships = []
# Extract from historical data
if parsed_data and parsed_data.data:
for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:
if isinstance(entry, dict):
symbol = entry.get("symbol", "")
date = entry.get("date", "")
historical_entities.append({
"id": f"{symbol}_{date}",
"type": "Historical_Price",
"name": f"{symbol} on {date}",
"properties": {
"symbol": symbol,
"date": date,
"open": entry.get("open", 0),
"high": entry.get("high", 0),
"low": entry.get("low", 0),
"close": entry.get("close", 0),
"volume": entry.get("volume", 0)
}
})
historical_entities.append({
"id": symbol,
"type": "Stock",
"name": symbol,
"properties": {}
})
historical_relationships.append({
"source": symbol,
"target": f"{symbol}_{date}",
"type": "has_price_on",
"properties": {"date": date}
})
builder = GraphBuilder()
temporal_query = TemporalGraphQuery()
temporal_pattern_detector = TemporalPatternDetector()
graph_analyzer = GraphAnalyzer()
historical_kg = builder.build(historical_entities, historical_relationships)
metrics = graph_analyzer.compute_metrics(historical_kg)
print(f"Extracted {len(historical_entities)} historical entities")
print(f"Extracted {len(historical_relationships)} relationships")
print(f"Built temporal knowledge graph with {len(historical_kg.get('entities', []))} entities")
print(f"Graph density: {metrics.get('density', 0):.3f}")
In [ ]:
# Define trading strategies
strategies = [
{
"name": "Moving Average Crossover",
"entry_rule": "IF close > moving_average_20 THEN buy",
"exit_rule": "IF close < moving_average_20 THEN sell"
},
{
"name": "Momentum Strategy",
"entry_rule": "IF price_change > 2% AND volume > average_volume THEN buy",
"exit_rule": "IF price_change < -1% THEN sell"
}
]
# Backtest strategies
backtest_results = []
for strategy in strategies:
trades = []
positions = {}
if parsed_data and parsed_data.data:
sorted_data = sorted(parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data],
key=lambda x: x.get("date", ""))
for entry in sorted_data:
if isinstance(entry, dict):
symbol = entry.get("symbol", "")
close_price = entry.get("close", 0)
date = entry.get("date", "")
# Simple strategy logic (moving average simulation)
if symbol not in positions:
# Entry signal
if close_price > 150: # Simplified entry condition
positions[symbol] = {
"entry_price": close_price,
"entry_date": date,
"quantity": 100
}
else:
# Exit signal
if close_price > positions[symbol]["entry_price"] * 1.02: # 2% profit target
trades.append({
"symbol": symbol,
"entry_price": positions[symbol]["entry_price"],
"exit_price": close_price,
"entry_date": positions[symbol]["entry_date"],
"exit_date": date,
"profit": (close_price - positions[symbol]["entry_price"]) * positions[symbol]["quantity"],
"return_pct": ((close_price - positions[symbol]["entry_price"]) / positions[symbol]["entry_price"]) * 100
})
del positions[symbol]
total_profit = sum(t["profit"] for t in trades)
total_return = sum(t["return_pct"] for t in trades) / len(trades) if trades else 0
backtest_results.append({
"strategy": strategy["name"],
"trades": len(trades),
"total_profit": total_profit,
"average_return": total_return,
"win_rate": len([t for t in trades if t["profit"] > 0]) / len(trades) if trades else 0
})
print(f"Backtested {len(strategies)} trading strategies")
for result in backtest_results:
print(f" Strategy: {result['strategy']} - Trades: {result['trades']}, Profit: ${result['total_profit']:.2f}, Avg Return: {result['average_return']:.2f}%")
In [ ]:
centrality_calculator = CentralityCalculator()
community_detector = CommunityDetector()
connectivity_analyzer = ConnectivityAnalyzer()
inference_engine = InferenceEngine()
rule_manager = RuleManager()
explanation_generator = ExplanationGenerator()
# Analyze graph structure
centrality_scores = centrality_calculator.calculate_centrality(historical_kg, measure="degree")
communities = community_detector.detect_communities(historical_kg)
connectivity = connectivity_analyzer.analyze_connectivity(historical_kg)
# Temporal pattern detection
start_date = "2023-01-01"
end_date = "2024-01-01"
temporal_results = temporal_query.query_time_range(
graph=historical_kg,
query="Find price movements in backtest period",
start_time=start_date,
end_time=end_date
)
temporal_patterns = temporal_pattern_detector.detect_temporal_patterns(
historical_kg,
pattern_type="trend",
min_frequency=1
)
# Performance inference rules
inference_engine.add_rule("IF average_return > 5% AND win_rate > 0.6 THEN profitable_strategy")
inference_engine.add_rule("IF total_profit > 1000 AND trades > 10 THEN successful_backtest")
for result in backtest_results:
inference_engine.add_fact({
"strategy": result["strategy"],
"average_return": result["average_return"],
"win_rate": result["win_rate"],
"total_profit": result["total_profit"],
"trades": result["trades"]
})
performance_insights = inference_engine.forward_chain()
print(f"Performance analysis complete")
print(f" Temporal patterns: {len(temporal_patterns)}")
print(f" Central stocks: {len([e for e, score in centrality_scores.items() if score > 0])}")
print(f" Communities: {len(communities)}")
print(f" Performance insights: {len(performance_insights)}")
In [ ]:
quality_assessor = KGQualityAssessor()
json_exporter = JSONExporter()
csv_exporter = CSVExporter()
rdf_exporter = RDFExporter()
report_generator = ReportGenerator()
quality_score = quality_assessor.assess_overall_quality(historical_kg)
json_exporter.export_knowledge_graph(historical_kg, os.path.join(temp_dir, "backtest_kg.json"))
csv_exporter.export_entities(historical_entities, os.path.join(temp_dir, "historical_entities.csv"))
rdf_exporter.export_knowledge_graph(historical_kg, os.path.join(temp_dir, "backtest_kg.rdf"))
report_data = {
"summary": f"Strategy backtesting analyzed {len(backtest_results)} strategies on {len(historical_entities)} historical data points",
"strategies_tested": len(backtest_results),
"total_trades": sum(r["trades"] for r in backtest_results),
"best_strategy": max(backtest_results, key=lambda x: x["total_profit"])["strategy"] if backtest_results else "N/A",
"patterns_detected": len(temporal_patterns),
"quality_score": quality_score.get('overall_score', 0)
}
report = report_generator.generate_report(report_data, format="markdown")
kg_visualizer = KGVisualizer()
temporal_visualizer = TemporalVisualizer()
analytics_visualizer = AnalyticsVisualizer()
kg_viz = kg_visualizer.visualize_network(historical_kg, output="interactive")
temporal_viz = temporal_visualizer.visualize_timeline(historical_kg, output="interactive")
analytics_viz = analytics_visualizer.visualize_analytics(historical_kg, output="interactive")
print("Generated backtest report and visualizations")
print(f"Total modules used: 20+")
print(f"Pipeline complete: Historical Data → Parse → Extract → Build Temporal KG → Test Strategies → Analyze Performance → Reports → Visualize")