semantica/benchmarks/benchmarks_runner.py

import argparse
import os
import subprocess
import sys
from datetime import datetime

from rich.console import Console
from rich.rule import Rule

console = Console()


def run_benchmarks():
    """
    Master Runner for Semantica Benchmarks.
    """
    parser = argparse.ArgumentParser(description="Run Semantica Benchmarks")
    parser.add_argument(
        "--strict", action="store_true", help="Fail script if performance regresses"
    )
    args = parser.parse_args()

    console.print(Rule("[bold cyan]Semantica Benchmark Suite[/bold cyan]", style="cyan"))

    timestamp = datetime.now().strftime("%Y%m%d_%H_%M_%S")
    os.makedirs("benchmarks/results", exist_ok=True)

    current_json = f"benchmarks/results/run_{timestamp}.json"
    baseline_json = "benchmarks/results/baseline.json"

    cmd = [
        sys.executable,
        "-m",
        "pytest",
        "benchmarks/",
        "-p", "no:typeguard",
        "-p", "no:langsmith",
        "--benchmark-only",
        f"--benchmark-json={current_json}",
        "--benchmark-columns=min,mean,stddev,ops",
        "--benchmark-sort=mean",
    ]

    console.print(f"[dim]Saving results to[/dim] {current_json}")
    result = subprocess.run(cmd)

    if result.returncode != 0:
        console.print("[bold red] ✗[/bold red] Benchmarks failed to execute (runtime errors).")
        sys.exit(result.returncode)

    console.print("[bold green] ✓[/bold green] Benchmarks completed execution.")

    if os.path.exists(baseline_json):
        console.print(f"[dim]Comparing against baseline:[/dim] {baseline_json}")

        if os.path.exists("benchmarks/infrastructure/compare.py"):
            compare_cmd = [
                sys.executable,
                "benchmarks/infrastructure/compare.py",
                baseline_json,
                current_json,
            ]

            compare_result = subprocess.run(compare_cmd)

            if compare_result.returncode != 0:
                console.print(Rule(style="red"))
                console.print("[bold red]  PERFORMANCE REGRESSION DETECTED[/bold red]")
                console.print(Rule(style="red"))
                if args.strict:
                    sys.exit(1)
            else:
                console.print(
                    "[bold green] ✓[/bold green] Performance is within acceptable limits."
                )
        else:
            console.print(
                "[bold yellow] ⚠[/bold yellow] Comparison script not found "
                "(benchmarks/infrastructure/compare.py). Skipping comparison."
            )
    else:
        console.print(
            "[bold yellow] ⚠[/bold yellow] No baseline found. "
            "This run effectively sets the new baseline."
        )

    console.print(
        f"\n[dim]To update baseline:[/dim]  cp {current_json} {baseline_json}"
    )


if __name__ == "__main__":
    run_benchmarks()