mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-09 04:00:52 +00:00
KevinandSameer Kadam
30a91a3a78
* chore: ignore .worktrees directory * feat(evals): add eval metric and result models * feat(evals): add evaluator registry * feat(evals): add exact/regex/range/length evaluators * feat(evals): add keyword/levenshtein/rouge/llm-as-judge evaluators * feat(evals): add decision_scores composite evaluator * feat(evals): add evaluation runner * feat(evals): expose public API and module proxy * fix(evals): resolve __all__ names and repair usage example * docs(evals): add usage docs and changelog entry * style(evals): tidy evaluator metadata and wiring comments * fix(evals): honor expected arg and classify error metrics * fix(evals): export get_evaluator and fix shared meta default * fix(evals): guard provenance check against non-dict metadata * docs: add objective layer design spec for semantica.evals * docs: refine objective spec for consistency with AIP Evals semantics * docs: add implementation plan for evals objective layer * docs: fix plan tests to use module-level pytest import * feat(evals): add per-metric objective support to runner * docs(evals): document per-metric objectives * docs(evals): fix minimize example threshold to demonstrate pass * fix(evals): validate objective config shape strictly * docs(evals): clarify objective examples and Boolean semantics * fix(evals): honor direction-only minimize, fail fast on objectives, deep-merge case config - minimize without threshold is now a no-op, matching maximize (issue #1091 requires thresholds to be optional for both directions) - objective config is parsed for every case before any target_fn/evaluator runs, so an invalid per-case objective rejects the run up front - per-case evaluator config deep-merges over the global config so a case that overrides one setting keeps the run-level objective - regression tests for all three, plus updated docs/CHANGELOG Addresses 3 of 4 Qodo findings on #1092 (the 4th, 'result models defined twice', is a false positive: types live in types.py) * fix: finalize eval objectives review --------- Co-authored-by: Sameer Kadam <sskadam6305@gmail.com>
82 lines
2.5 KiB
Python
82 lines
2.5 KiB
Python
"""Tests for generic evaluators: exact, regex, ranges, length."""
|
|
import pytest
|
|
|
|
from semantica.evals import registry as reg
|
|
|
|
|
|
class TestExactMatch:
|
|
def test_exact_str(self):
|
|
r = reg.get_evaluator("exact_match")("approved", "approved")
|
|
assert r.passed and r.score == 1.0
|
|
|
|
def test_exact_str_negative(self):
|
|
r = reg.get_evaluator("exact_match")("approved", "denied")
|
|
assert not r.passed and r.score == 0.0
|
|
|
|
def test_exact_number(self):
|
|
r = reg.get_evaluator("exact_match")(5, 5)
|
|
assert r.passed
|
|
|
|
def test_exact_array(self):
|
|
r = reg.get_evaluator("exact_match")([1, 2], [1, 2])
|
|
assert r.passed
|
|
|
|
|
|
class TestRegexMatch:
|
|
def test_matching(self):
|
|
r = reg.get_evaluator("regex_match")("abc123", r"^[a-z]+\d+$")
|
|
assert r.passed
|
|
|
|
def test_non_matching(self):
|
|
r = reg.get_evaluator("regex_match")("ABC", r"^[a-z]+$")
|
|
assert not r.passed
|
|
assert "ABC" in r.meta.get("reason", "")
|
|
|
|
def test_invalid_regex_is_error_metric(self):
|
|
r = reg.get_evaluator("regex_match")("x", "[invalid")
|
|
assert not r.passed
|
|
assert r.meta.get("error")
|
|
|
|
|
|
class TestNumericRange:
|
|
def test_inside(self):
|
|
r = reg.get_evaluator("numeric_range")(0.9, config={"min": 0.8, "max": 1.0})
|
|
assert r.passed and r.score == 1.0
|
|
|
|
def test_outside(self):
|
|
r = reg.get_evaluator("numeric_range")(0.5, config={"min": 0.8, "max": 1.0})
|
|
assert not r.passed and r.score == 0.0
|
|
|
|
def test_bounds_inclusive(self):
|
|
assert reg.get_evaluator("numeric_range")(0.8, config={"min": 0.8, "max": 0.8}).passed
|
|
|
|
|
|
class TestTemporalRange:
|
|
def test_inside_window(self):
|
|
r = reg.get_evaluator("temporal_range")(
|
|
"2026-01-15T10:00:00",
|
|
config={"min": "2026-01-01T00:00:00", "max": "2026-02-01T00:00:00"},
|
|
)
|
|
assert r.passed
|
|
|
|
def test_outside_window(self):
|
|
r = reg.get_evaluator("temporal_range")(
|
|
"2026-03-01T00:00:00",
|
|
config={"min": "2026-01-01T00:00:00", "max": "2026-02-01T00:00:00"},
|
|
)
|
|
assert not r.passed
|
|
|
|
|
|
class TestLengthRange:
|
|
def test_ok(self):
|
|
r = reg.get_evaluator("length_range")("hello", config={"min": 3, "max": 5})
|
|
assert r.passed
|
|
|
|
def test_too_long(self):
|
|
r = reg.get_evaluator("length_range")([1, 2, 3], config={"min": 1, "max": 2})
|
|
assert not r.passed
|
|
|
|
def test_min_not_given_defaults_zero(self):
|
|
r = reg.get_evaluator("length_range")("abc", config={"max": 5})
|
|
assert r.passed
|