import argparse import os import subprocess import sys from datetime import datetime def run_benchmarks(): """ Master Runner for Semantica Benchmarks. """ parser = argparse.ArgumentParser(description="Run Semantica Benchmarks") parser.add_argument( "--strict", action="store_true", help="Fail script if performance regresses" ) args = parser.parse_args() print("Starting Semantica Benchmark Suite...") timestamp = datetime.now().strftime("%Y%m%d_%H_%M_%S") os.makedirs("benchmarks/results", exist_ok=True) current_json = f"benchmarks/results/run_{timestamp}.json" baseline_json = "benchmarks/results/baseline.json" # Run Benchmarks cmd = [ sys.executable, "-m", "pytest", "benchmarks/", "-p", "no:typeguard", "-p", "no:langsmith", "--benchmark-only", f"--benchmark-json={current_json}", "--benchmark-columns=min,mean,stddev,ops", "--benchmark-sort=mean", ] print(f"Executing benchmarks... (saving to {current_json})") result = subprocess.run(cmd) if result.returncode != 0: print("Benchmarks failed to execute (runtime errors).") sys.exit(result.returncode) print("Benchmarks completed execution.") # Compare against Baseline if os.path.exists(baseline_json): print(f"Comparing against Baseline ({baseline_json})...") if os.path.exists("benchmarks/infrastructure/compare.py"): compare_cmd = [ sys.executable, "benchmarks/infrastructure/compare.py", baseline_json, current_json, ] compare_result = subprocess.run(compare_cmd) if compare_result.returncode != 0: print("\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") print(" PERFORMANCE REGRESSION DETECTED") print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n") if args.strict: sys.exit(1) else: print("Performance is within acceptable limits.") else: print( "Comparison script not found (benchmarks/infrastructure/compare.py). Skipping comparison." ) else: print("No baseline found. This run effectively sets the new baseline.") print(f"\n[Action] To update baseline: cp {current_json} {baseline_json}") if __name__ == "__main__": run_benchmarks()