mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-10 04:00:35 +00:00
KevinandSameer Kadam
30a91a3a78
* chore: ignore .worktrees directory * feat(evals): add eval metric and result models * feat(evals): add evaluator registry * feat(evals): add exact/regex/range/length evaluators * feat(evals): add keyword/levenshtein/rouge/llm-as-judge evaluators * feat(evals): add decision_scores composite evaluator * feat(evals): add evaluation runner * feat(evals): expose public API and module proxy * fix(evals): resolve __all__ names and repair usage example * docs(evals): add usage docs and changelog entry * style(evals): tidy evaluator metadata and wiring comments * fix(evals): honor expected arg and classify error metrics * fix(evals): export get_evaluator and fix shared meta default * fix(evals): guard provenance check against non-dict metadata * docs: add objective layer design spec for semantica.evals * docs: refine objective spec for consistency with AIP Evals semantics * docs: add implementation plan for evals objective layer * docs: fix plan tests to use module-level pytest import * feat(evals): add per-metric objective support to runner * docs(evals): document per-metric objectives * docs(evals): fix minimize example threshold to demonstrate pass * fix(evals): validate objective config shape strictly * docs(evals): clarify objective examples and Boolean semantics * fix(evals): honor direction-only minimize, fail fast on objectives, deep-merge case config - minimize without threshold is now a no-op, matching maximize (issue #1091 requires thresholds to be optional for both directions) - objective config is parsed for every case before any target_fn/evaluator runs, so an invalid per-case objective rejects the run up front - per-case evaluator config deep-merges over the global config so a case that overrides one setting keeps the run-level objective - regression tests for all three, plus updated docs/CHANGELOG Addresses 3 of 4 Qodo findings on #1092 (the 4th, 'result models defined twice', is a false positive: types live in types.py) * fix: finalize eval objectives review --------- Co-authored-by: Sameer Kadam <sskadam6305@gmail.com>
37 lines
1.3 KiB
Python
37 lines
1.3 KiB
Python
"""Tests for the evaluator registry."""
|
|
import pytest
|
|
|
|
from semantica.evals import registry as reg
|
|
from semantica.evals.types import EvalMetric
|
|
|
|
# A unique name that will not collide with any production evaluator.
|
|
_TEST_EVAL_NAME = "test_registry_demo_eval"
|
|
|
|
|
|
class TestRegistry:
|
|
def teardown_method(self, method):
|
|
# Remove the test evaluator after each test that may have registered it,
|
|
# so re-runs and randomised collection cannot see stale state.
|
|
reg.EVALUATORS.pop(_TEST_EVAL_NAME, None)
|
|
|
|
def test_register_and_get(self):
|
|
@reg.register(_TEST_EVAL_NAME)
|
|
def demo(actual, expected, config=None, **kwargs):
|
|
return EvalMetric(1.0, True)
|
|
|
|
assert reg.get_evaluator(_TEST_EVAL_NAME) is demo
|
|
assert _TEST_EVAL_NAME in reg.list_evaluators()
|
|
|
|
def test_registration_is_immutable_after_commit(self):
|
|
with pytest.raises(ValueError):
|
|
reg.get_evaluator("does_not_exist")
|
|
|
|
def test_unknown_evaluator_failure_message(self):
|
|
with pytest.raises(ValueError) as exc:
|
|
reg.get_evaluator("nope")
|
|
msg = str(exc.value)
|
|
assert "nope" in msg
|
|
# The error message lists available evaluators; verify using a name
|
|
# that is always registered at import time (independent of test order).
|
|
assert "exact_match" in msg
|