mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-08 04:00:15 +00:00
KevinandSameer Kadam
30a91a3a78
* chore: ignore .worktrees directory * feat(evals): add eval metric and result models * feat(evals): add evaluator registry * feat(evals): add exact/regex/range/length evaluators * feat(evals): add keyword/levenshtein/rouge/llm-as-judge evaluators * feat(evals): add decision_scores composite evaluator * feat(evals): add evaluation runner * feat(evals): expose public API and module proxy * fix(evals): resolve __all__ names and repair usage example * docs(evals): add usage docs and changelog entry * style(evals): tidy evaluator metadata and wiring comments * fix(evals): honor expected arg and classify error metrics * fix(evals): export get_evaluator and fix shared meta default * fix(evals): guard provenance check against non-dict metadata * docs: add objective layer design spec for semantica.evals * docs: refine objective spec for consistency with AIP Evals semantics * docs: add implementation plan for evals objective layer * docs: fix plan tests to use module-level pytest import * feat(evals): add per-metric objective support to runner * docs(evals): document per-metric objectives * docs(evals): fix minimize example threshold to demonstrate pass * fix(evals): validate objective config shape strictly * docs(evals): clarify objective examples and Boolean semantics * fix(evals): honor direction-only minimize, fail fast on objectives, deep-merge case config - minimize without threshold is now a no-op, matching maximize (issue #1091 requires thresholds to be optional for both directions) - objective config is parsed for every case before any target_fn/evaluator runs, so an invalid per-case objective rejects the run up front - per-case evaluator config deep-merges over the global config so a case that overrides one setting keeps the run-level objective - regression tests for all three, plus updated docs/CHANGELOG Addresses 3 of 4 Qodo findings on #1092 (the 4th, 'result models defined twice', is a false positive: types live in types.py) * fix: finalize eval objectives review --------- Co-authored-by: Sameer Kadam <sskadam6305@gmail.com>
68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
"""Tests for generic evaluators: keyword, levenshtein, rouge, llm-as-judge."""
|
|
import pytest
|
|
|
|
from semantica.evals import registry as reg
|
|
|
|
|
|
class TestKeywordCheck:
|
|
def test_all_required_present(self):
|
|
r = reg.get_evaluator("keyword_check")(
|
|
"the loan was approved", expected=["loan", "approved"]
|
|
)
|
|
assert r.passed
|
|
|
|
def test_missing_keyword(self):
|
|
r = reg.get_evaluator("keyword_check")(
|
|
"the loan was approved", expected=["loan", "denied"]
|
|
)
|
|
assert not r.passed
|
|
assert "denied" in r.meta.get("missing", [])
|
|
|
|
def test_short_words_ignored(self):
|
|
r = reg.get_evaluator("keyword_check")("x and y", expected=["and"])
|
|
assert r.passed
|
|
|
|
|
|
class TestLevenshtein:
|
|
def test_identical(self):
|
|
r = reg.get_evaluator("levenshtein")("credit approved", "credit approved")
|
|
assert r.passed
|
|
|
|
def test_close_above_threshold(self):
|
|
r = reg.get_evaluator("levenshtein")(
|
|
"credit approved", "credit denied", config={"threshold": 0.8}
|
|
)
|
|
assert not r.passed
|
|
|
|
def test_default_threshold(self):
|
|
assert reg.get_evaluator("levenshtein")("a", "a").passed
|
|
|
|
|
|
class TestRouge:
|
|
def test_identical(self):
|
|
r = reg.get_evaluator("rouge")("loan approved by committee", "loan approved by committee")
|
|
assert r.passed
|
|
assert r.meta["f1"] == pytest.approx(1.0)
|
|
|
|
def test_no_overlap(self):
|
|
r = reg.get_evaluator("rouge")("one two three", "four five six")
|
|
assert not r.passed
|
|
|
|
def test_partial_sets_meta(self):
|
|
r = reg.get_evaluator("rouge")("a b c", "a b d", config={"threshold": 0.5})
|
|
assert "precision" in r.meta and "recall" in r.meta
|
|
|
|
|
|
class TestLlmAsJudge:
|
|
def test_uses_supplied_judge(self):
|
|
judge = lambda actual, expected: actual == expected # noqa: E731
|
|
r = reg.get_evaluator("llm_as_judge")(
|
|
"x", "x", config={"judge_fn": judge}
|
|
)
|
|
assert r.passed
|
|
|
|
def test_missing_judge_is_error(self):
|
|
r = reg.get_evaluator("llm_as_judge")("x", "y", config={})
|
|
assert not r.passed
|
|
assert r.meta.get("error")
|