Fixed max line length (88) issues and eager imports

This commit is contained in:
Accute9
2026-08-16 21:04:57 -04:00
parent 893b6db3c3
commit 0f252ab355
3 changed files with 17 additions and 7 deletions
+2 -2
View File
@@ -93,12 +93,11 @@ from ..utils.exceptions import ProcessingError
from ..utils.helpers import safe_import
from ..utils.logging import get_logger
from .semantic_chunker import Chunk
from ..semantic_extract.methods import load_spacy_model
logger = get_logger("split_methods")
# Try to import optional dependencies
spacy, SPACY_AVAILABLE = safe_import("spacy")
_, SPACY_AVAILABLE = safe_import("spacy")
nltk, NLTK_AVAILABLE = safe_import("nltk")
tiktoken, TIKTOKEN_AVAILABLE = safe_import("tiktoken")
@@ -337,6 +336,7 @@ def split_by_sentences(
# Try spaCy first
if SPACY_AVAILABLE and kwargs.get("use_spacy", True):
try:
from ..semantic_extract.methods import load_spacy_model
nlp = load_spacy_model("en_core_web_sm")
doc = nlp(text)
sentences = [sent.text for sent in doc.sents]
+2 -2
View File
@@ -35,10 +35,9 @@ from ..utils.exceptions import ProcessingError
from ..utils.helpers import safe_import
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from ..semantic_extract.methods import load_spacy_model
spacy, SPACY_AVAILABLE = safe_import("spacy")
_, SPACY_AVAILABLE = safe_import("spacy")
@dataclass
@@ -81,6 +80,7 @@ class SemanticChunker:
if SPACY_AVAILABLE:
model_name = config.get("model", "en_core_web_sm")
try:
from ..semantic_extract.methods import load_spacy_model
self.nlp = load_spacy_model(model_name)
except OSError:
self.logger.warning(
+13 -3
View File
@@ -26,7 +26,12 @@ def force_spacy_available(monkeypatch):
def _fake_spacy(load):
return SimpleNamespace(load=load, util=SimpleNamespace(is_package=lambda _name: True))
return SimpleNamespace(
load=load,
util=SimpleNamespace(
is_package=lambda _name: True
),
)
def _nlp_mock(sentences=("Hello world.",)):
@@ -129,7 +134,10 @@ class TestSpacyModelCache:
se_methods.clear_spacy_model_cache()
semantic_chunker.SemanticChunker()
assert calls == [{}, {}], "neither caller should request a partial pipeline"
assert len(calls) == 2
assert all("disable" not in kwargs for kwargs in calls), (
"neither caller should request a partial pipeline"
)
def test_missing_model_falls_back_without_poisoning_cache(self, monkeypatch):
attempts = []
@@ -161,7 +169,9 @@ class TestSpacyModelCache:
chunker2 = semantic_chunker.SemanticChunker()
split_methods.split_by_sentences("One more sentence.")
assert len(attempts) == 3, "the model should load once after it becomes available"
assert len(attempts) == 3, (
"the model should load once after it becomes available"
)
assert chunker2.nlp is not None