mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Fixed max line length (88) issues and eager imports
This commit is contained in:
@@ -93,12 +93,11 @@ from ..utils.exceptions import ProcessingError
|
||||
from ..utils.helpers import safe_import
|
||||
from ..utils.logging import get_logger
|
||||
from .semantic_chunker import Chunk
|
||||
from ..semantic_extract.methods import load_spacy_model
|
||||
|
||||
logger = get_logger("split_methods")
|
||||
|
||||
# Try to import optional dependencies
|
||||
spacy, SPACY_AVAILABLE = safe_import("spacy")
|
||||
_, SPACY_AVAILABLE = safe_import("spacy")
|
||||
|
||||
nltk, NLTK_AVAILABLE = safe_import("nltk")
|
||||
tiktoken, TIKTOKEN_AVAILABLE = safe_import("tiktoken")
|
||||
@@ -337,6 +336,7 @@ def split_by_sentences(
|
||||
# Try spaCy first
|
||||
if SPACY_AVAILABLE and kwargs.get("use_spacy", True):
|
||||
try:
|
||||
from ..semantic_extract.methods import load_spacy_model
|
||||
nlp = load_spacy_model("en_core_web_sm")
|
||||
doc = nlp(text)
|
||||
sentences = [sent.text for sent in doc.sents]
|
||||
|
||||
@@ -35,10 +35,9 @@ from ..utils.exceptions import ProcessingError
|
||||
from ..utils.helpers import safe_import
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ..semantic_extract.methods import load_spacy_model
|
||||
|
||||
|
||||
spacy, SPACY_AVAILABLE = safe_import("spacy")
|
||||
_, SPACY_AVAILABLE = safe_import("spacy")
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -81,6 +80,7 @@ class SemanticChunker:
|
||||
if SPACY_AVAILABLE:
|
||||
model_name = config.get("model", "en_core_web_sm")
|
||||
try:
|
||||
from ..semantic_extract.methods import load_spacy_model
|
||||
self.nlp = load_spacy_model(model_name)
|
||||
except OSError:
|
||||
self.logger.warning(
|
||||
|
||||
@@ -26,7 +26,12 @@ def force_spacy_available(monkeypatch):
|
||||
|
||||
|
||||
def _fake_spacy(load):
|
||||
return SimpleNamespace(load=load, util=SimpleNamespace(is_package=lambda _name: True))
|
||||
return SimpleNamespace(
|
||||
load=load,
|
||||
util=SimpleNamespace(
|
||||
is_package=lambda _name: True
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _nlp_mock(sentences=("Hello world.",)):
|
||||
@@ -129,7 +134,10 @@ class TestSpacyModelCache:
|
||||
se_methods.clear_spacy_model_cache()
|
||||
semantic_chunker.SemanticChunker()
|
||||
|
||||
assert calls == [{}, {}], "neither caller should request a partial pipeline"
|
||||
assert len(calls) == 2
|
||||
assert all("disable" not in kwargs for kwargs in calls), (
|
||||
"neither caller should request a partial pipeline"
|
||||
)
|
||||
|
||||
def test_missing_model_falls_back_without_poisoning_cache(self, monkeypatch):
|
||||
attempts = []
|
||||
@@ -161,7 +169,9 @@ class TestSpacyModelCache:
|
||||
chunker2 = semantic_chunker.SemanticChunker()
|
||||
split_methods.split_by_sentences("One more sentence.")
|
||||
|
||||
assert len(attempts) == 3, "the model should load once after it becomes available"
|
||||
assert len(attempts) == 3, (
|
||||
"the model should load once after it becomes available"
|
||||
)
|
||||
assert chunker2.nlp is not None
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user