fix: complete spaCy model cache integration

semantica/semantic_extract/ner_extractor.py

@@ -144,7 +144,12 @@ class NERExtractor:
         self._ml_runtime_usable = True
         if "ml" in self.method and SPACY_AVAILABLE:
             try:
-                self.nlp = spacy.load(self.model_name)
+                # Deferred import: keeps semantic_extract.methods out of the
+                # module-level import graph and routes loading through the
+                # process-level cache so repeated NERExtractor constructions
+                # never pay the ~120 ms spacy.load() cost more than once.
+                from .methods import load_spacy_model
+                self.nlp = load_spacy_model(self.model_name)
             except OSError:
                 self.logger.warning(
                     f"spaCy model {self.model_name} not found. ML method will fallback."

tests/split/test_spacy_model_cache.py

@@ -6,6 +6,8 @@ import pytest
 from semantica.semantic_extract import methods as se_methods
 from semantica.split import methods as split_methods
 from semantica.split import semantic_chunker
+from semantica.semantic_extract import ner_extractor as ner_extractor_module
+from semantica.semantic_extract.ner_extractor import NERExtractor
 
 
 @pytest.fixture(autouse=True)
@@ -17,12 +19,13 @@ def clear_cache():
 
 @pytest.fixture(autouse=True)
 def force_spacy_available(monkeypatch):
-    # split.methods and split.semantic_chunker each compute their own
-    # SPACY_AVAILABLE flag from the real environment at import time; force
-    # both true so these tests exercise the spaCy branch regardless of
+    # split.methods, split.semantic_chunker, and ner_extractor each compute
+    # their own SPACY_AVAILABLE flag from the real environment at import time;
+    # force all true so these tests exercise the spaCy branch regardless of
     # whether spaCy is actually installed where they run.
     monkeypatch.setattr(split_methods, "SPACY_AVAILABLE", True)
     monkeypatch.setattr(semantic_chunker, "SPACY_AVAILABLE", True)
+    monkeypatch.setattr(ner_extractor_module, "SPACY_AVAILABLE", True)
 
 
 def _fake_spacy(load):
@@ -133,9 +136,11 @@ class TestSpacyModelCache:
         semantic_chunker.SemanticChunker()
 
         assert len(calls) == 2
-        assert all("disable" not in kwargs for kwargs in calls), (
-            "neither caller should request a partial pipeline"
-        )   
+        assert all(kwargs == {} for kwargs in calls), (
+            "neither caller should pass any pipeline-configuration kwargs; "
+            "the name-only cache key in load_spacy_model cannot distinguish "
+            "models loaded with different component configs"
+        )
 
     def test_missing_model_falls_back_without_poisoning_cache(self, monkeypatch):
         attempts = []
@@ -173,5 +178,126 @@ class TestSpacyModelCache:
         assert chunker2.nlp is not None
 
 
+class TestNERExtractorSpacyModelCache:
+    """NERExtractor(method="ml") must reuse the centralized cache in
+    semantic_extract.methods, not call spacy.load() on every construction.
+
+    These tests mirror TestSpacyModelCache but focus on the NERExtractor path,
+    confirming that all three callers (split_by_sentences, SemanticChunker, and
+    NERExtractor) draw from the same process-level cache.
+    """
+
+    def test_ner_extractor_reuses_cached_model_across_instances(self, monkeypatch):
+        """Two NERExtractor(method='ml') constructions with the same model name
+        must cause exactly one underlying spacy.load() call."""
+        calls = []
+
+        def fake_load(name, **kwargs):
+            calls.append(name)
+            return _nlp_mock()
+
+        monkeypatch.setattr(se_methods, "spacy", _fake_spacy(fake_load))
+
+        e1 = NERExtractor(method="ml")
+        e2 = NERExtractor(method="ml")
+        e3 = NERExtractor(method="ml", model="en_core_web_sm")
+
+        assert len(calls) == 1, (
+            "repeated NERExtractor constructions should not reload the model"
+        )
+        assert e1.nlp is e2.nlp is e3.nlp
+
+    def test_ner_extractor_and_split_callers_share_one_cached_model(self, monkeypatch):
+        """NERExtractor, SemanticChunker, and split_by_sentences must all use
+        the same cached Language object for the same model name."""
+        calls = []
+
+        def fake_load(name, **kwargs):
+            calls.append(name)
+            return _nlp_mock()
+
+        monkeypatch.setattr(se_methods, "spacy", _fake_spacy(fake_load))
+
+        split_methods.split_by_sentences("First sentence.")
+        semantic_chunker.SemanticChunker()
+        NERExtractor(method="ml")
+
+        assert len(calls) == 1, (
+            "split_by_sentences, SemanticChunker, and NERExtractor must share "
+            "one cached model instead of each loading their own"
+        )
+
+    def test_ner_extractor_distinct_model_names_load_separately(self, monkeypatch):
+        """Different model names must produce separate cache entries."""
+        calls = []
+
+        def fake_load(name, **kwargs):
+            calls.append(name)
+            return _nlp_mock()
+
+        monkeypatch.setattr(se_methods, "spacy", _fake_spacy(fake_load))
+
+        sm = NERExtractor(method="ml", model="en_core_web_sm")
+        lg = NERExtractor(method="ml", model="en_core_web_lg")
+        sm_again = NERExtractor(method="ml", model="en_core_web_sm")
+
+        assert calls == ["en_core_web_sm", "en_core_web_lg"]
+        assert sm.nlp is sm_again.nlp
+        assert sm.nlp is not lg.nlp
+
+    def test_ner_extractor_failed_load_not_cached_and_retried(self, monkeypatch):
+        """A missing model must not poison the cache.  A subsequent construction
+        after the model becomes available must succeed and share the loaded model."""
+        attempts = []
+
+        def failing_load(name, **_kwargs):
+            attempts.append(name)
+            raise OSError(f"Can't find model '{name}'")
+
+        monkeypatch.setattr(se_methods, "spacy", _fake_spacy(failing_load))
+
+        # Construction with missing model: nlp must remain None, no crash
+        extractor1 = NERExtractor(method="ml")
+        assert extractor1.nlp is None
+        assert len(attempts) == 1, "one load attempt expected for the missing model"
+
+        # Second construction: must retry (cache must not hold the failure)
+        extractor2 = NERExtractor(method="ml")
+        assert extractor2.nlp is None
+        assert len(attempts) == 2, "a failed load must not be cached"
+
+        # Now install a working model and verify recovery
+        def working_load(name, **_kwargs):
+            attempts.append(name)
+            return _nlp_mock()
+
+        monkeypatch.setattr(se_methods, "spacy", _fake_spacy(working_load))
+
+        extractor3 = NERExtractor(method="ml")
+        extractor4 = NERExtractor(method="ml")
+
+        assert extractor3.nlp is not None
+        assert extractor3.nlp is extractor4.nlp
+        assert len(attempts) == 3, (
+            "exactly one successful load expected after the model becomes available"
+        )
+
+    def test_ner_extractor_non_ml_method_does_not_load_model(self, monkeypatch):
+        """NERExtractor with a non-ml method must not touch the spaCy cache."""
+        calls = []
+
+        def fake_load(name, **kwargs):
+            calls.append(name)
+            return _nlp_mock()
+
+        monkeypatch.setattr(se_methods, "spacy", _fake_spacy(fake_load))
+
+        NERExtractor(method="pattern")
+        NERExtractor(method="llm")
+        NERExtractor(method="regex")
+
+        assert calls == [], "non-ml methods must not trigger any spacy.load()"
+
+
 if __name__ == "__main__":
     pytest.main([__file__])

tests/split/test_splitter.py

@@ -30,18 +30,16 @@ class TestSplitter(unittest.TestCase):
         splitter = TextSplitter(method=["recursive", "token"])
         self.assertEqual(splitter.methods, ["recursive", "token"])
 
-    @patch('semantica.split.semantic_chunker.spacy')
+    @patch('semantica.semantic_extract.methods.spacy')
     def test_semantic_chunker_initialization(self, mock_spacy):
-        # Mock spacy.load to return a mock nlp object
+        # SemanticChunker now loads spaCy through the centralized
+        # load_spacy_model() in semantic_extract.methods, so we patch
+        # methods.spacy rather than the removed semantic_chunker.spacy binding.
         mock_nlp = MagicMock()
         mock_spacy.load.return_value = mock_nlp
-        
-        # We need to ensure SPACY_AVAILABLE is True for this test context if possible, 
-        # but it is imported at module level. 
-        # If spacy is not installed, it sets SPACY_AVAILABLE = False.
-        # We might need to patch the module attribute or just test fallback if spacy missing.
-        
-        chunker = SemanticChunker(chunk_size=100)
+
+        with patch('semantica.split.semantic_chunker.SPACY_AVAILABLE', True):
+            chunker = SemanticChunker(chunk_size=100)
         self.assertEqual(chunker.chunk_size, 100)
 
     def test_chunk_dataclass(self):

tests/test_ner_configurations.py

@@ -101,9 +101,15 @@ class TestNERConfigurations(unittest.TestCase):
             self.assertEqual(entities[0].metadata["extraction_method"], "ml")
             self.assertEqual(entities[0].metadata["model"], "en_core_web_trf")
 
-    @patch('semantica.semantic_extract.ner_extractor.spacy')
+    @patch('semantica.semantic_extract.methods.spacy')
     def test_ner_ml_init_falls_back_when_spacy_runtime_is_broken(self, mock_spacy):
-        """Test NER init does not crash when spaCy is installed but unusable at runtime."""
+        """Test NER init does not crash when spaCy is installed but unusable at runtime.
+
+        The model load now goes through load_spacy_model() in semantic_extract.methods,
+        so we patch methods.spacy (not ner_extractor.spacy) to inject the failure.
+        """
+        from semantica.semantic_extract.methods import clear_spacy_model_cache
+        clear_spacy_model_cache()
         mock_spacy.load.side_effect = RuntimeError("ConfigSchemaNlp is not fully defined")
 
         with patch('semantica.semantic_extract.ner_extractor.SPACY_AVAILABLE', True):
@@ -112,17 +118,23 @@ class TestNERConfigurations(unittest.TestCase):
         self.assertIsNone(extractor.nlp)
         self.assertFalse(extractor._ml_runtime_usable)
 
-    @patch('semantica.semantic_extract.ner_extractor.spacy')
     @patch('semantica.semantic_extract.methods.get_entity_method')
     @patch('semantica.semantic_extract.methods.spacy')
     def test_ner_ml_runtime_failure_disables_repeated_ml_load_attempts(
         self,
         mock_methods_spacy,
         mock_get_method,
-        mock_init_spacy,
     ):
-        """Test degraded ML mode skips repeated spaCy load attempts after init failure."""
-        mock_init_spacy.load.side_effect = RuntimeError("ConfigSchemaNlp is not fully defined")
+        """Test degraded ML mode skips repeated spaCy load attempts after init failure.
+
+        The model load at construction time now goes through load_spacy_model() in
+        semantic_extract.methods, so methods.spacy is the single mock target for the
+        init-time failure.  After the RuntimeError is raised, _ml_runtime_usable is
+        False and no further spacy.load (or extract_entities_ml) calls are made.
+        """
+        from semantica.semantic_extract.methods import clear_spacy_model_cache
+        clear_spacy_model_cache()
+        mock_methods_spacy.load.side_effect = RuntimeError("ConfigSchemaNlp is not fully defined")
         mock_ml_method = MagicMock(return_value=[])
         mock_get_method.side_effect = lambda name: mock_ml_method if name == "ml" else (lambda *_args, **_kwargs: [])
 
@@ -132,8 +144,9 @@ class TestNERConfigurations(unittest.TestCase):
         entities = extractor.extract_entities(self.text)
 
         self.assertFalse(extractor._ml_runtime_usable)
-        self.assertEqual(mock_init_spacy.load.call_count, 1)
-        self.assertEqual(mock_methods_spacy.load.call_count, 0)
+        # methods.spacy.load called once during __init__ (the RuntimeError); not again
+        # during extract_entities because _filter_unusable_methods removes "ml".
+        self.assertEqual(mock_methods_spacy.load.call_count, 1)
         self.assertEqual(mock_ml_method.call_count, 0)
         self.assertIsInstance(entities, list)