fix(benchmarks): replace hardcoded module choices with dynamic discovery

- Extract _discover_modules() to scan benchmarks/ at runtime so the
  --module choices list stays accurate as directories are added or
  removed; eliminates the stale context_graph_effectiveness entry and
  the missing infrastructure entry from the original implementation
- Add an existence guard before passing the resolved path to pytest so
  a valid-looking choice that maps to a missing directory fails fast
  with a clear error instead of silently collecting 0 tests and exiting 0
- Print the active module filter to the console so users can confirm
  the filtered scope in runner output
This commit is contained in:
KaifAhmad1
2026-06-06 19:05:37 +05:30
parent 027f1caa38
commit 1cf91f1621
+29 -10
View File
@@ -10,10 +10,23 @@ from rich.rule import Rule
console = Console()
def _discover_modules() -> list[str]:
benchmarks_dir = "benchmarks"
_excluded = {"results", "__pycache__"}
return sorted(
d for d in os.listdir(benchmarks_dir)
if os.path.isdir(os.path.join(benchmarks_dir, d))
and not d.startswith(("_", "."))
and d not in _excluded
)
def run_benchmarks():
"""
Master Runner for Semantica Benchmarks.
"""
available_modules = _discover_modules()
parser = argparse.ArgumentParser(description="Run Semantica Benchmarks")
parser.add_argument(
"--strict", action="store_true", help="Fail script if performance regresses"
@@ -21,13 +34,12 @@ def run_benchmarks():
parser.add_argument(
"--module",
type=str,
choices=[
"context", "context_graph_effectiveness", "context_memory",
"core_processing", "export", "input_layer", "normalize",
"ontology", "output_orchestration", "quality_assurance",
"storage", "visualization"
],
help="Run benchmarks for specific module only"
choices=available_modules,
help=(
"Run benchmarks for a specific module only "
"(default: all modules). "
f"Available: {', '.join(available_modules)}"
),
)
args = parser.parse_args()
@@ -39,9 +51,16 @@ def run_benchmarks():
current_json = f"benchmarks/results/run_{timestamp}.json"
baseline_json = "benchmarks/results/baseline.json"
# Build test path based on module selection
module_path = args.module + "/" if args.module else ""
test_path = f"benchmarks/{module_path}"
if args.module:
test_path = os.path.join("benchmarks", args.module)
if not os.path.isdir(test_path):
console.print(
f"[bold red] ✗[/bold red] Module directory not found: {test_path}"
)
sys.exit(1)
console.print(f"[dim]Module filter:[/dim] {args.module}")
else:
test_path = "benchmarks/"
cmd = [
sys.executable,