mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
59 lines
2.1 KiB
Python
59 lines
2.1 KiB
Python
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
from semantica.normalize.text_normalizer import TextNormalizer
|
|
from semantica.normalize.text_cleaner import TextCleaner
|
|
|
|
class TestTextNormalizer(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.normalizer = TextNormalizer()
|
|
|
|
def test_normalize_text_case(self):
|
|
text = "Hello World"
|
|
self.assertEqual(self.normalizer.normalize_text(text, case="lower"), "hello world")
|
|
self.assertEqual(self.normalizer.normalize_text(text, case="upper"), "HELLO WORLD")
|
|
self.assertEqual(self.normalizer.normalize_text(text, case="preserve"), "Hello World")
|
|
|
|
def test_normalize_unicode(self):
|
|
# e + combining acute accent
|
|
text = "e\u0301"
|
|
normalized = self.normalizer.normalize_unicode(text, form="NFC")
|
|
# should become single character é (\u00e9)
|
|
self.assertEqual(normalized, "\u00e9")
|
|
|
|
def test_process_special_chars(self):
|
|
text = "Hello\u2013World" # En dash
|
|
processed = self.normalizer.process_special_chars(text)
|
|
self.assertEqual(processed, "Hello-World")
|
|
|
|
def test_handle_encoding(self):
|
|
text_bytes = "Hello World".encode("utf-8")
|
|
result = self.normalizer.handle_encoding(text_bytes, "utf-8")
|
|
self.assertEqual(result, "Hello World")
|
|
|
|
# Test string pass-through
|
|
self.assertEqual(self.normalizer.handle_encoding("Hello", "utf-8"), "Hello")
|
|
|
|
class TestTextCleaner(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.cleaner = TextCleaner()
|
|
|
|
def test_clean_html(self):
|
|
text = "<p>Hello <b>World</b></p>"
|
|
cleaned = self.cleaner.clean(text, remove_html=True)
|
|
self.assertEqual(cleaned.strip(), "Hello World")
|
|
|
|
def test_clean_whitespace(self):
|
|
text = "Hello World\n\n"
|
|
cleaned = self.cleaner.clean(text, normalize_whitespace=True, remove_html=False)
|
|
self.assertEqual(cleaned, "Hello World")
|
|
|
|
def test_clean_unicode(self):
|
|
text = "e\u0301"
|
|
cleaned = self.cleaner.clean(text, normalize_unicode=True)
|
|
self.assertEqual(cleaned, "\u00e9")
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|