mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-08 04:00:15 +00:00
KevinandSameer Kadam
30a91a3a78
* chore: ignore .worktrees directory * feat(evals): add eval metric and result models * feat(evals): add evaluator registry * feat(evals): add exact/regex/range/length evaluators * feat(evals): add keyword/levenshtein/rouge/llm-as-judge evaluators * feat(evals): add decision_scores composite evaluator * feat(evals): add evaluation runner * feat(evals): expose public API and module proxy * fix(evals): resolve __all__ names and repair usage example * docs(evals): add usage docs and changelog entry * style(evals): tidy evaluator metadata and wiring comments * fix(evals): honor expected arg and classify error metrics * fix(evals): export get_evaluator and fix shared meta default * fix(evals): guard provenance check against non-dict metadata * docs: add objective layer design spec for semantica.evals * docs: refine objective spec for consistency with AIP Evals semantics * docs: add implementation plan for evals objective layer * docs: fix plan tests to use module-level pytest import * feat(evals): add per-metric objective support to runner * docs(evals): document per-metric objectives * docs(evals): fix minimize example threshold to demonstrate pass * fix(evals): validate objective config shape strictly * docs(evals): clarify objective examples and Boolean semantics * fix(evals): honor direction-only minimize, fail fast on objectives, deep-merge case config - minimize without threshold is now a no-op, matching maximize (issue #1091 requires thresholds to be optional for both directions) - objective config is parsed for every case before any target_fn/evaluator runs, so an invalid per-case objective rejects the run up front - per-case evaluator config deep-merges over the global config so a case that overrides one setting keeps the run-level objective - regression tests for all three, plus updated docs/CHANGELOG Addresses 3 of 4 Qodo findings on #1092 (the 4th, 'result models defined twice', is a false positive: types live in types.py) * fix: finalize eval objectives review --------- Co-authored-by: Sameer Kadam <sskadam6305@gmail.com>
33 lines
1.1 KiB
Python
33 lines
1.1 KiB
Python
"""Tests for the evals public package API."""
|
|
from semantica import evals
|
|
from semantica.evals import evaluate, get_evaluator, list_evaluators
|
|
|
|
|
|
class TestPublicAPI:
|
|
def test_imports(self):
|
|
assert callable(evaluate)
|
|
assert callable(list_evaluators)
|
|
assert callable(get_evaluator)
|
|
|
|
def test_version_present(self):
|
|
assert hasattr(evals, "__version__")
|
|
|
|
def test_module_proxy_via_root(self):
|
|
# semantica.evals must resolve through the lazy proxy
|
|
assert hasattr(evals, "evaluate")
|
|
|
|
def test_all_populated(self):
|
|
assert len(evals.__all__) >= 2
|
|
assert "evaluate" in evals.__all__
|
|
assert "list_evaluators" in evals.__all__
|
|
assert "get_evaluator" in evals.__all__
|
|
|
|
def test_register_discovery(self):
|
|
names = evals.list_evaluators()
|
|
for expected in (
|
|
"exact_match", "regex_match", "numeric_range", "temporal_range",
|
|
"length_range", "keyword_check", "levenshtein", "rouge",
|
|
"llm_as_judge", "decision_scores",
|
|
):
|
|
assert expected in names
|