mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
fix(benchmarks): replace hardcoded module choices with dynamic discovery
- Extract _discover_modules() to scan benchmarks/ at runtime so the --module choices list stays accurate as directories are added or removed; eliminates the stale context_graph_effectiveness entry and the missing infrastructure entry from the original implementation - Add an existence guard before passing the resolved path to pytest so a valid-looking choice that maps to a missing directory fails fast with a clear error instead of silently collecting 0 tests and exiting 0 - Print the active module filter to the console so users can confirm the filtered scope in runner output
This commit is contained in:
@@ -10,10 +10,23 @@ from rich.rule import Rule
|
||||
console = Console()
|
||||
|
||||
|
||||
def _discover_modules() -> list[str]:
|
||||
benchmarks_dir = "benchmarks"
|
||||
_excluded = {"results", "__pycache__"}
|
||||
return sorted(
|
||||
d for d in os.listdir(benchmarks_dir)
|
||||
if os.path.isdir(os.path.join(benchmarks_dir, d))
|
||||
and not d.startswith(("_", "."))
|
||||
and d not in _excluded
|
||||
)
|
||||
|
||||
|
||||
def run_benchmarks():
|
||||
"""
|
||||
Master Runner for Semantica Benchmarks.
|
||||
"""
|
||||
available_modules = _discover_modules()
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run Semantica Benchmarks")
|
||||
parser.add_argument(
|
||||
"--strict", action="store_true", help="Fail script if performance regresses"
|
||||
@@ -21,13 +34,12 @@ def run_benchmarks():
|
||||
parser.add_argument(
|
||||
"--module",
|
||||
type=str,
|
||||
choices=[
|
||||
"context", "context_graph_effectiveness", "context_memory",
|
||||
"core_processing", "export", "input_layer", "normalize",
|
||||
"ontology", "output_orchestration", "quality_assurance",
|
||||
"storage", "visualization"
|
||||
],
|
||||
help="Run benchmarks for specific module only"
|
||||
choices=available_modules,
|
||||
help=(
|
||||
"Run benchmarks for a specific module only "
|
||||
"(default: all modules). "
|
||||
f"Available: {', '.join(available_modules)}"
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -39,9 +51,16 @@ def run_benchmarks():
|
||||
current_json = f"benchmarks/results/run_{timestamp}.json"
|
||||
baseline_json = "benchmarks/results/baseline.json"
|
||||
|
||||
# Build test path based on module selection
|
||||
module_path = args.module + "/" if args.module else ""
|
||||
test_path = f"benchmarks/{module_path}"
|
||||
if args.module:
|
||||
test_path = os.path.join("benchmarks", args.module)
|
||||
if not os.path.isdir(test_path):
|
||||
console.print(
|
||||
f"[bold red] ✗[/bold red] Module directory not found: {test_path}"
|
||||
)
|
||||
sys.exit(1)
|
||||
console.print(f"[dim]Module filter:[/dim] {args.module}")
|
||||
else:
|
||||
test_path = "benchmarks/"
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
|
||||
Reference in New Issue
Block a user