semantica/tests/vector_store/test_performance_benchmarks.py

"""
Performance Benchmarks for Enhanced Vector Store

This module contains comprehensive performance benchmarks for the enhanced
vector store decision tracking functionality.

Benchmarks:
    - Vector storage and retrieval performance
    - Decision processing throughput
    - Hybrid search performance
    - KG algorithm integration performance
    - Memory usage and scalability
    - Concurrent operation performance
"""

import pytest
import numpy as np
import time
import threading
import psutil
import os
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Dict, List, Any

from semantica.vector_store import VectorStore, DecisionEmbeddingPipeline, HybridSimilarityCalculator
from semantica.context import DecisionContext, ContextRetriever
from semantica.vector_store.decision_vector_methods import (
    quick_decision, find_precedents, batch_decisions
)


class TestPerformanceBenchmarks:
    """Performance benchmarks for enhanced vector store."""
    
    def setup_method(self):
        """Set up benchmark environment."""
        self.vector_store = VectorStore(backend="inmemory", dimension=384)
        self.process = psutil.Process(os.getpid())
        
        # Performance thresholds (in seconds)
        self.thresholds = {
            "single_decision": 0.1,      # 100ms per decision
            "batch_decision": 0.01,       # 10ms per decision in batch
            "search_query": 0.05,         # 50ms per search
            "precedent_search": 0.1,      # 100ms per precedent search
            "context_retrieval": 0.15,    # 150ms for context retrieval
            "memory_per_decision": 1024,   # 1KB per decision
        }
    
    def get_memory_usage(self) -> float:
        """Get current memory usage in MB."""
        return self.process.memory_info().rss / 1024 / 1024
    
    def benchmark_single_decision_processing(self):
        """Benchmark single decision processing performance."""
        print("\n=== Benchmark: Single Decision Processing ===")
        
        # Test decision
        test_decision = {
            "scenario": "Credit limit increase request for performance testing",
            "reasoning": "Performance benchmark test decision with comprehensive metadata",
            "outcome": "approved",
            "confidence": 0.85,
            "entities": ["customer_perf", "account_test", "benchmark"],
            "category": "performance_test"
        }
        
        # Initialize context
        context = DecisionContext(vector_store=self.vector_store, graph_store=None)
        
        # Benchmark single decision
        start_time = time.time()
        start_memory = self.get_memory_usage()
        
        decision_id = context.record_decision(**test_decision)
        
        end_time = time.time()
        end_memory = self.get_memory_usage()
        
        processing_time = end_time - start_time
        memory_used = end_memory - start_memory
        
        print(f"✅ Single decision: {processing_time:.3f}s, {memory_used:.2f}MB")
        
        # Verify performance
        assert processing_time < self.thresholds["single_decision"], \
            f"Single decision too slow: {processing_time:.3f}s > {self.thresholds['single_decision']}s"
        
        assert memory_used < self.thresholds["memory_per_decision"] / 1024, \
            f"Memory usage too high: {memory_used:.2f}MB > {self.thresholds['memory_per_decision']/1024}MB"
        
        return processing_time, memory_used
    
    def benchmark_batch_processing(self):
        """Benchmark batch processing performance."""
        print("\n=== Benchmark: Batch Processing ===")
        
        # Generate test batch
        batch_size = 100
        test_batch = []
        for i in range(batch_size):
            test_batch.append({
                "scenario": f"Batch decision {i}: Credit assessment",
                "reasoning": f"Automated decision {i} for performance testing",
                "outcome": "approved" if i % 3 != 0 else "rejected",
                "confidence": 0.7 + (i % 10) * 0.03,
                "entities": [f"entity_{i}", f"account_{i}"],
                "category": "batch_test"
            })
        
        # Benchmark batch processing
        start_time = time.time()
        start_memory = self.get_memory_usage()
        
        results = self.vector_store.process_decision_batch(test_batch, batch_size=20)
        
        end_time = time.time()
        end_memory = self.get_memory_usage()
        
        total_time = end_time - start_time
        memory_used = end_memory - start_memory
        avg_time_per_decision = total_time / batch_size
        avg_memory_per_decision = (memory_used * 1024) / batch_size  # Convert to KB
        
        print(f"✅ Batch processing: {batch_size} decisions in {total_time:.3f}s")
        print(f"✅ Average: {avg_time_per_decision:.3f}s per decision, {avg_memory_per_decision:.1f}KB per decision")
        
        # Verify performance
        assert avg_time_per_decision < self.thresholds["batch_decision"], \
            f"Batch processing too slow: {avg_time_per_decision:.3f}s > {self.thresholds['batch_decision']}s"
        
        assert len(results) == batch_size, "Should process all decisions"
        assert all("vector_id" in result for result in results), "All should have vector IDs"
        
        return total_time, memory_used
    
    def benchmark_search_performance(self):
        """Benchmark search and retrieval performance."""
        print("\n=== Benchmark: Search Performance ===")
        
        # Create test data
        num_documents = 500
        for i in range(num_documents):
            vector = np.random.rand(384)
            metadata = {
                "content": f"Document {i} about various topics for search testing",
                "category": f"category_{i % 10}",
                "importance": (i % 5) / 4.0
            }
            self.vector_store.store_vectors([vector], [metadata])
        
        print(f"✅ Created {num_documents} test documents")
        
        # Benchmark different search types
        search_tests = [
            ("Vector Search", lambda: self.vector_store.search_vectors(np.random.rand(384), k=10)),
            ("Decision Search", lambda: self.vector_store.search_decisions("test query", limit=10)),
            ("Precedent Search", lambda: find_precedents("test scenario", limit=10)),
        ]
        
        search_results = {}
        
        for search_name, search_func in search_tests:
            # Warm up
            search_func()
            
            # Benchmark
            times = []
            for _ in range(10):
                start_time = time.time()
                results = search_func()
                end_time = time.time()
                times.append(end_time - start_time)
            
            avg_time = sum(times) / len(times)
            search_results[search_name] = avg_time
            
            print(f"✅ {search_name}: {avg_time:.3f}s average")
            
            # Verify performance
            assert avg_time < self.thresholds["search_query"], \
                f"{search_name} too slow: {avg_time:.3f}s > {self.thresholds['search_query']}s"
        
        return search_results
    
    def benchmark_hybrid_search_performance(self):
        """Benchmark hybrid search with different weight configurations."""
        print("\n=== Benchmark: Hybrid Search Performance ===")
        
        # Create decision context
        context = DecisionContext(vector_store=self.vector_store, graph_store=None)
        
        # Record test decisions
        for i in range(50):
            context.record_decision(
                scenario=f"Decision {i}: Credit assessment",
                reasoning=f"Reasoning for decision {i}",
                outcome="approved" if i % 2 == 0 else "rejected",
                confidence=0.7 + (i % 10) * 0.03,
                entities=[f"entity_{i}"],
                category="hybrid_test"
            )
        
        # Test different weight configurations
        weight_configs = [
            (1.0, 0.0, "Semantic Only"),
            (0.0, 1.0, "Structural Only"),
            (0.7, 0.3, "Default"),
            (0.5, 0.5, "Balanced"),
            (0.9, 0.1, "Semantic Heavy"),
            (0.1, 0.9, "Structural Heavy"),
        ]
        
        hybrid_results = {}
        
        for sem_weight, struct_weight, config_name in weight_configs:
            times = []
            for _ in range(5):
                start_time = time.time()
                precedents = context.find_similar_decisions(
                    scenario="Credit assessment",
                    limit=10,
                    semantic_weight=sem_weight,
                    structural_weight=struct_weight
                )
                end_time = time.time()
                times.append(end_time - start_time)
                
                # Verify results
                assert len(precedents) > 0, f"Should find precedents for {config_name}"
            
            avg_time = sum(times) / len(times)
            hybrid_results[config_name] = avg_time
            
            print(f"✅ {config_name}: {avg_time:.3f}s average")
            
            # Verify performance
            assert avg_time < self.thresholds["precedent_search"], \
                f"{config_name} too slow: {avg_time:.3f}s > {self.thresholds['precedent_search']}s"
        
        return hybrid_results
    
    def benchmark_context_retrieval_performance(self):
        """Benchmark context retriever performance."""
        print("\n=== Benchmark: Context Retrieval Performance ===")
        
        # Create context retriever
        retriever = ContextRetriever(vector_store=self.vector_store, knowledge_graph=None)
        
        # Store test context data
        for i in range(100):
            vector = np.random.rand(384)
            metadata = {
                "content": f"Context document {i} with comprehensive information",
                "type": f"type_{i % 5}",
                "source": f"source_{i % 3}"
            }
            self.vector_store.store_vectors([vector], [metadata])
        
        # Benchmark context retrieval
        retrieval_tests = [
            ("Basic Retrieval", lambda: retriever.retrieve("test query", max_results=10)),
            ("With Expansion", lambda: retriever.retrieve("test query", max_results=10, use_graph_expansion=True)),
            ("Decision Precedents", lambda: retriever.retrieve_decision_precedents("test scenario", limit=10)),
        ]
        
        retrieval_results = {}
        
        for retrieval_name, retrieval_func in retrieval_tests:
            times = []
            for _ in range(5):
                start_time = time.time()
                results = retrieval_func()
                end_time = time.time()
                times.append(end_time - start_time)
            
            avg_time = sum(times) / len(times)
            retrieval_results[retrieval_name] = avg_time
            
            print(f"✅ {retrieval_name}: {avg_time:.3f}s average")
            
            # Verify performance
            assert avg_time < self.thresholds["context_retrieval"], \
                f"{retrieval_name} too slow: {avg_time:.3f}s > {self.thresholds['context_retrieval']}s"
        
        return retrieval_results
    
    def benchmark_concurrent_operations(self):
        """Benchmark concurrent operation performance."""
        print("\n=== Benchmark: Concurrent Operations ===")
        
        # Create decision context
        context = DecisionContext(vector_store=self.vector_store, graph_store=None)
        
        # Concurrent decision recording
        def record_decision_batch(batch_id, batch_size=10):
            """Record a batch of decisions concurrently."""
            results = []
            for i in range(batch_size):
                decision_id = context.record_decision(
                    scenario=f"Concurrent decision {batch_id}-{i}",
                    reasoning=f"Concurrent processing test",
                    outcome="approved",
                    confidence=0.8,
                    entities=[f"entity_{batch_id}_{i}"],
                    category="concurrent_test"
                )
                results.append(decision_id)
            return results
        
        # Test concurrent batches
        num_batches = 10
        batch_size = 5
        
        start_time = time.time()
        start_memory = self.get_memory_usage()
        
        with ThreadPoolExecutor(max_workers=4) as executor:
            futures = [
                executor.submit(record_decision_batch, i, batch_size)
                for i in range(num_batches)
            ]
            
            all_results = []
            for future in as_completed(futures):
                batch_results = future.result()
                all_results.extend(batch_results)
        
        end_time = time.time()
        end_memory = self.get_memory_usage()
        
        total_time = end_time - start_time
        memory_used = end_memory - start_memory
        total_decisions = len(all_results)
        avg_time_per_decision = total_time / total_decisions
        
        print(f"✅ Concurrent: {total_decisions} decisions in {total_time:.3f}s")
        print(f"✅ Average: {avg_time_per_decision:.3f}s per decision, {memory_used:.2f}MB total")
        
        # Verify performance
        assert avg_time_per_decision < self.thresholds["single_decision"] * 2, \
            f"Concurrent processing too slow: {avg_time_per_decision:.3f}s"
        
        assert len(all_results) == num_batches * batch_size, "Should process all decisions"
        
        return total_time, memory_used
    
    def benchmark_memory_scalability(self):
        """Benchmark memory usage scalability."""
        print("\n=== Benchmark: Memory Scalability ===")
        
        memory_snapshots = []
        
        # Test memory usage at different scales
        scales = [100, 500, 1000, 2000]
        
        for scale in scales:
            # Clear vector store
            self.vector_store = VectorStore(backend="inmemory", dimension=384)
            
            start_memory = self.get_memory_usage()
            
            # Add decisions at current scale
            context = DecisionContext(vector_store=self.vector_store, graph_store=None)
            
            for i in range(scale):
                context.record_decision(
                    scenario=f"Scalability test {i}",
                    reasoning=f"Memory usage test at scale {scale}",
                    outcome="approved",
                    confidence=0.8,
                    entities=[f"entity_{i}"],
                    category="scalability_test"
                )
            
            end_memory = self.get_memory_usage()
            memory_used = end_memory - start_memory
            memory_per_decision = (memory_used * 1024) / scale  # KB per decision
            
            memory_snapshots.append({
                "scale": scale,
                "total_memory_mb": memory_used,
                "memory_per_decision_kb": memory_per_decision
            })
            
            print(f"✅ Scale {scale}: {memory_used:.2f}MB total, {memory_per_decision:.1f}KB per decision")
            
            # Verify memory efficiency
            assert memory_per_decision < self.thresholds["memory_per_decision"], \
                f"Memory usage too high at scale {scale}: {memory_per_decision:.1f}KB"
        
        # Verify linear scalability
        if len(memory_snapshots) >= 3:
            # Check that memory growth is roughly linear
            first = memory_snapshots[0]
            last = memory_snapshots[-1]
            
            expected_growth = (last["scale"] / first["scale"]) * first["total_memory_mb"]
            actual_growth = last["total_memory_mb"]
            
            # Allow 50% overhead for indexing and metadata
            assert actual_growth < expected_growth * 1.5, \
                f"Memory growth not linear: expected ~{expected_growth:.1f}MB, got {actual_growth:.1f}MB"
        
        return memory_snapshots
    
    def benchmark_kg_algorithm_performance(self):
        """Benchmark KG algorithm integration performance."""
        print("\n=== Benchmark: KG Algorithm Performance ===")
        
        from unittest.mock import Mock
        from semantica.kg.path_finder import PathFinder
        from semantica.kg.community_detector import CommunityDetector
        from semantica.kg.centrality_calculator import CentralityCalculator
        
        # Mock KG algorithms
        mock_kg = Mock()
        mock_kg.get_neighbors.return_value = [f"neighbor_{i}" for i in range(5)]
        
        # Create retriever with KG algorithms
        retriever = ContextRetriever(vector_store=self.vector_store, knowledge_graph=mock_kg)
        
        # Store test data
        for i in range(50):
            vector = np.random.rand(384)
            metadata = {"content": f"KG test document {i}", "entities": [f"entity_{i}"]}
            self.vector_store.store_vectors([vector], [metadata])
        
        # Benchmark KG-enhanced context expansion
        entities = [{"name": f"entity_{i}", "type": "entity"} for i in range(10)]
        
        kg_times = []
        for _ in range(10):
            start_time = time.time()
            expanded = retriever._expand_decision_context(entities, max_hops=2)
            end_time = time.time()
            kg_times.append(end_time - start_time)
            
            assert len(expanded) > len(entities), "Should expand context"
        
        avg_kg_time = sum(kg_times) / len(kg_times)
        print(f"✅ KG Context Expansion: {avg_kg_time:.3f}s average")
        
        # Verify KG algorithm performance
        assert avg_kg_time < 0.5, f"KG processing too slow: {avg_kg_time:.3f}s"
        
        return avg_kg_time
    
    def run_comprehensive_benchmark(self):
        """Run comprehensive performance benchmark."""
        print("\n" + "="*60)
        print("COMPREHENSIVE PERFORMANCE BENCHMARK")
        print("="*60)
        
        results = {}
        
        # Run all benchmarks
        results["single_decision"] = self.benchmark_single_decision_processing()
        results["batch_processing"] = self.benchmark_batch_processing()
        results["search_performance"] = self.benchmark_search_performance()
        results["hybrid_search"] = self.benchmark_hybrid_search_performance()
        results["context_retrieval"] = self.benchmark_context_retrieval_performance()
        results["concurrent_operations"] = self.benchmark_concurrent_operations()
        results["memory_scalability"] = self.benchmark_memory_scalability()
        results["kg_algorithms"] = self.benchmark_kg_algorithm_performance()
        
        # Summary
        print("\n" + "="*60)
        print("BENCHMARK SUMMARY")
        print("="*60)
        
        print(f"✅ Single Decision: {results['single_decision'][0]:.3f}s")
        print(f"✅ Batch Processing: {results['batch_processing'][0]:.3f}s for 100 decisions")
        print(f"✅ Search Performance: {len(results['search_performance'])} search types tested")
        print(f"✅ Hybrid Search: {len(results['hybrid_search'])} weight configs tested")
        print(f"✅ Context Retrieval: {len(results['context_retrieval'])} retrieval types tested")
        print(f"✅ Concurrent Operations: {results['concurrent_operations'][0]:.3f}s for 50 decisions")
        print(f"✅ Memory Scalability: Tested up to {results['memory_scalability'][-1]['scale']} decisions")
        print(f"✅ KG Algorithms: {results['kg_algorithms']:.3f}s average")
        
        # Performance summary
        all_thresholds_met = True
        for benchmark_name, threshold_key in [
            ("single_decision", "single_decision"),
            ("batch_processing", "batch_decision"),
            ("kg_algorithms", "precedent_search"),
        ]:
            if benchmark_name in results:
                actual_time = results[benchmark_name][0] if isinstance(results[benchmark_name], tuple) else results[benchmark_name]
                if actual_time > self.thresholds[threshold_key]:
                    all_thresholds_met = False
                    print(f"❌ {benchmark_name}: {actual_time:.3f}s > {self.thresholds[threshold_key]}s")
        
        if all_thresholds_met:
            print("\n🎉 ALL PERFORMANCE BENCHMARKS PASSED!")
        else:
            print("\n⚠️  Some performance benchmarks exceeded thresholds")
        
        return results


class TestStressTests:
    """Stress tests for enhanced vector store."""
    
    def setup_method(self):
        """Set up stress test environment."""
        self.vector_store = VectorStore(backend="inmemory", dimension=384)
    
    def stress_test_high_volume_decisions(self):
        """Stress test with high volume of decisions."""
        print("\n=== Stress Test: High Volume Decisions ===")
        
        # Test with large number of decisions
        num_decisions = 1000
        context = DecisionContext(vector_store=self.vector_store, graph_store=None)
        
        start_time = time.time()
        
        # Process in batches to avoid memory issues
        batch_size = 50
        for batch_start in range(0, num_decisions, batch_size):
            batch_end = min(batch_start + batch_size, num_decisions)
            
            for i in range(batch_start, batch_end):
                context.record_decision(
                    scenario=f"Stress test decision {i}",
                    reasoning=f"High volume test decision {i}",
                    outcome="approved" if i % 2 == 0 else "rejected",
                    confidence=0.8,
                    entities=[f"entity_{i}"],
                    category="stress_test"
                )
            
            if batch_start % 100 == 0:
                print(f"✅ Processed {batch_end} decisions...")
        
        end_time = time.time()
        total_time = end_time - start_time
        avg_time = total_time / num_decisions
        
        print(f"✅ Stress test: {num_decisions} decisions in {total_time:.1f}s")
        print(f"✅ Average: {avg_time:.3f}s per decision")
        
        # Verify all decisions processed
        total_vectors = len(self.vector_store.vectors)
        assert total_vectors >= num_decisions, f"Should have at least {num_decisions} vectors, got {total_vectors}"
        
        # Test search performance under load
        search_start = time.time()
        precedents = context.find_similar_decisions("Stress test", limit=20)
        search_time = time.time() - search_start
        
        print(f"✅ Search under load: {search_time:.3f}s for {len(precedents)} results")
        
        assert len(precedents) > 0, "Should find precedents even under load"
        assert search_time < 1.0, "Search should remain fast under load"
    
    def stress_test_memory_pressure(self):
        """Stress test under memory pressure."""
        print("\n=== Stress Test: Memory Pressure ===")
        
        import psutil
        process = psutil.Process()
        initial_memory = process.memory_info().rss / 1024 / 1024  # MB
        
        # Create decisions with large metadata
        context = DecisionContext(vector_store=self.vector_store, graph_store=None)
        
        for i in range(200):
            # Large metadata to stress memory
            large_metadata = {
                "scenario": f"Memory stress test {i}",
                "reasoning": "A" * 1000,  # Large reasoning string
                "outcome": "approved",
                "confidence": 0.8,
                "entities": [f"entity_{j}" for j in range(10)],  # Many entities
                "category": "memory_stress",
                "large_field": "X" * 5000,  # Very large field
            }
            
            context.record_decision(**large_metadata)
            
            if i % 50 == 0:
                current_memory = process.memory_info().rss / 1024 / 1024
                memory_growth = current_memory - initial_memory
                print(f"✅ {i} decisions: {memory_growth:.1f}MB memory growth")
        
        # Final memory check
        final_memory = process.memory_info().rss / 1024 / 1024
        total_memory_growth = final_memory - initial_memory
        
        print(f"✅ Total memory growth: {total_memory_growth:.1f}MB")
        
        # Should not grow excessively (allow 100MB for 200 decisions)
        assert total_memory_growth < 100, f"Memory growth too high: {total_memory_growth:.1f}MB"
        
        # Verify functionality still works
        precedents = context.find_similar_decisions("Memory stress", limit=5)
        assert len(precedents) > 0, "Should still work under memory pressure"


if __name__ == "__main__":
    # Run performance benchmarks
    pytest.main([__file__, "-v", "-s"])