mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
103 lines
2.8 KiB
Python
103 lines
2.8 KiB
Python
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List
|
|
|
|
|
|
def load_results(filepath: str) -> Dict[str, Any]:
|
|
with open(filepath, "r") as f:
|
|
return json.load(f)
|
|
|
|
|
|
def calc_z_score(current_mean, base_mean, base_stddev):
|
|
"""
|
|
Z-Score indicates how many standard deviations
|
|
away current run is from baseline
|
|
"""
|
|
|
|
if base_stddev == 0:
|
|
return 0 if current_mean == base_mean else 100.0
|
|
|
|
return (current_mean - base_mean) / base_stddev
|
|
|
|
|
|
def compare_benchmarks(
|
|
baseline: Dict[str, Any], current: Dict[str, Any], threshold_pct: float = 10.0
|
|
):
|
|
"""
|
|
Uses Mean for % change and Z-score for noise detection.
|
|
"""
|
|
|
|
# colors for terminal
|
|
RED = "\033[91m"
|
|
GREEN = "\033[92m"
|
|
YELLOW = "\033[93m"
|
|
RESET = "\033[0m"
|
|
|
|
header = f"{'Benchmark':<60} | {'CHANGE %':<12} | {'SIGMA (Z)':<10} | {'STATUS'}"
|
|
print(header)
|
|
print("=" * len(header))
|
|
|
|
baseline_map = {b["name"]: b for b in baseline["benchmarks"]}
|
|
current_map = {b["name"]: b for b in current["benchmarks"]}
|
|
|
|
regressions = []
|
|
|
|
for name, curr in current_map.items():
|
|
base = baseline_map.get(name)
|
|
if not base:
|
|
print(f"{name:<60} | {'NEW':<12} | {'N/A':<10} | NEW")
|
|
continue
|
|
|
|
m1 = base["stats"]["mean"]
|
|
s1 = base["stats"]["stddev"]
|
|
m2 = curr["stats"]["mean"]
|
|
|
|
if m1 == 0:
|
|
delta_pct = 0.0
|
|
else:
|
|
delta_pct = ((m2 - m1) / m1) * 100
|
|
|
|
z_score = calc_z_score(m2, m1, s1)
|
|
|
|
status = f"{GREEN} OK{RESET}"
|
|
|
|
if delta_pct > threshold_pct:
|
|
if abs(z_score) > 2.0:
|
|
status = f"{RED} REGRESSION{RESET}"
|
|
regressions.append(name)
|
|
else:
|
|
status = f"{YELLOW} NOISE{RESET}"
|
|
elif delta_pct < -threshold_pct and abs(z_score) > 2.0:
|
|
status = f"{GREEN} IMPROVED{RESET}"
|
|
|
|
print(f"{name:<60} | {delta_pct:>+10.2f}% | {z_score:>9.2f} | {status}")
|
|
|
|
if regressions:
|
|
print(
|
|
f"\n{RED}FAILURE: Performance regression detected in {len(regressions)} tests.{RESET}"
|
|
)
|
|
return True
|
|
print(f"\n{GREEN}SUCCESS: No significant regressions.{RESET}")
|
|
return False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("baseline", help="Gold standard JSON")
|
|
parser.add_argument("current", help="NEW RUN JSON")
|
|
parser.add_argument(
|
|
"--threshold", type=float, default=10.0, help="FAIL if slower by %"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
failed = compare_benchmarks(
|
|
load_results(args.baseline), load_results(args.current), args.threshold
|
|
)
|
|
sys.exit(1 if failed else 0)
|
|
except FileNotFoundError as e:
|
|
print(f"Error loading files: {e}")
|
|
sys.exit(0)
|