mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-30 04:40:16 +00:00
32 lines
885 B
Python
32 lines
885 B
Python
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from semantica.normalize.encoding_handler import EncodingHandler
|
|
from semantica.normalize.language_detector import LanguageDetector
|
|
|
|
|
|
def test_language_detection_throughput(benchmark, generate_text_data):
|
|
"""Benchmarks langdetect intergration."""
|
|
detector = LanguageDetector()
|
|
texts = [generate_text_data("clean", 200) for _ in range(50)]
|
|
|
|
def run():
|
|
return detector.detect_batch(texts)
|
|
|
|
benchmark.pedantic(run, iterations=1, rounds=5)
|
|
|
|
|
|
def test_encoding_detection(benchmark):
|
|
"""Benchmarks chardet integration via EncodingHandler."""
|
|
handler = EncodingHandler()
|
|
data = (
|
|
b"Wowzaaa a simple string for encoding decoding , oh encoding detection just."
|
|
* 100
|
|
)
|
|
|
|
def run():
|
|
return handler.detect(data)
|
|
|
|
benchmark.pedantic(run, iterations=5, rounds=10)
|