fix(ingest): lazy-load optional ingestion backends

semantica/ingest/__init__.py

@@ -112,22 +112,18 @@ Example Usage:
     >>> content = ingest_web("https://example.com", method="url")
 """
 
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from __future__ import annotations
+
+import importlib
+from typing import Any, Dict, Tuple
 
 from .config import IngestConfig, ingest_config
-from .db_ingestor import DatabaseConnector, DataExporter, DBIngestor, TableData
-from .email_ingestor import AttachmentProcessor, EmailData, EmailIngestor
-from .email_ingestor import EmailParser as EmailIngestorParser
-from .feed_ingestor import FeedData, FeedIngestor, FeedItem, FeedMonitor, FeedParser
 from .file_ingestor import (
     CloudStorageIngestor,
     FileIngestor,
     FileObject,
     FileTypeDetector,
 )
-from .mcp_client import MCPClient, MCPResource, MCPTool
-from .mcp_ingestor import MCPData, MCPIngestor
 from .methods import (
     get_ingest_method,
     ingest,
@@ -143,34 +139,103 @@ from .methods import (
     list_available_methods,
 )
 from .registry import MethodRegistry, method_registry
-from .repo_ingestor import (
-    CodeExtractor,
-    CodeFile,
-    CommitInfo,
-    GitAnalyzer,
-    RepoIngestor,
-)
-from .stream_ingestor import (
-    KafkaProcessor,
-    KinesisProcessor,
-    PulsarProcessor,
-    RabbitMQProcessor,
-    StreamIngestor,
-    StreamMessage,
-    StreamMonitor,
-    StreamProcessor,
-)
-from .web_ingestor import (
-    ContentExtractor,
-    RateLimiter,
-    RobotsChecker,
-    SitemapCrawler,
-    WebContent,
-    WebIngestor,
-)
 
-from .ontology_ingestor import OntologyData, OntologyIngestor
-from .snowflake_ingestor import SnowflakeConnector, SnowflakeData, SnowflakeIngestor
+_LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
+    # Web ingestion
+    "WebIngestor": (".web_ingestor", "WebIngestor"),
+    "WebContent": (".web_ingestor", "WebContent"),
+    "RateLimiter": (".web_ingestor", "RateLimiter"),
+    "RobotsChecker": (".web_ingestor", "RobotsChecker"),
+    "ContentExtractor": (".web_ingestor", "ContentExtractor"),
+    "SitemapCrawler": (".web_ingestor", "SitemapCrawler"),
+    # Feed ingestion
+    "FeedIngestor": (".feed_ingestor", "FeedIngestor"),
+    "FeedItem": (".feed_ingestor", "FeedItem"),
+    "FeedData": (".feed_ingestor", "FeedData"),
+    "FeedParser": (".feed_ingestor", "FeedParser"),
+    "FeedMonitor": (".feed_ingestor", "FeedMonitor"),
+    # Stream ingestion
+    "StreamIngestor": (".stream_ingestor", "StreamIngestor"),
+    "StreamMessage": (".stream_ingestor", "StreamMessage"),
+    "StreamProcessor": (".stream_ingestor", "StreamProcessor"),
+    "KafkaProcessor": (".stream_ingestor", "KafkaProcessor"),
+    "RabbitMQProcessor": (".stream_ingestor", "RabbitMQProcessor"),
+    "KinesisProcessor": (".stream_ingestor", "KinesisProcessor"),
+    "PulsarProcessor": (".stream_ingestor", "PulsarProcessor"),
+    "StreamMonitor": (".stream_ingestor", "StreamMonitor"),
+    # Repository ingestion
+    "RepoIngestor": (".repo_ingestor", "RepoIngestor"),
+    "CodeFile": (".repo_ingestor", "CodeFile"),
+    "CommitInfo": (".repo_ingestor", "CommitInfo"),
+    "CodeExtractor": (".repo_ingestor", "CodeExtractor"),
+    "GitAnalyzer": (".repo_ingestor", "GitAnalyzer"),
+    # Email ingestion
+    "EmailIngestor": (".email_ingestor", "EmailIngestor"),
+    "EmailData": (".email_ingestor", "EmailData"),
+    "AttachmentProcessor": (".email_ingestor", "AttachmentProcessor"),
+    "EmailIngestorParser": (".email_ingestor", "EmailParser"),
+    # Database ingestion
+    "DBIngestor": (".db_ingestor", "DBIngestor"),
+    "TableData": (".db_ingestor", "TableData"),
+    "DatabaseConnector": (".db_ingestor", "DatabaseConnector"),
+    "DataExporter": (".db_ingestor", "DataExporter"),
+    # MCP ingestion
+    "MCPIngestor": (".mcp_ingestor", "MCPIngestor"),
+    "MCPData": (".mcp_ingestor", "MCPData"),
+    "MCPClient": (".mcp_client", "MCPClient"),
+    "MCPResource": (".mcp_client", "MCPResource"),
+    "MCPTool": (".mcp_client", "MCPTool"),
+    # Ontology ingestion
+    "OntologyIngestor": (".ontology_ingestor", "OntologyIngestor"),
+    "OntologyData": (".ontology_ingestor", "OntologyData"),
+    # Snowflake ingestion
+    "SnowflakeIngestor": (".snowflake_ingestor", "SnowflakeIngestor"),
+    "SnowflakeData": (".snowflake_ingestor", "SnowflakeData"),
+    "SnowflakeConnector": (".snowflake_ingestor", "SnowflakeConnector"),
+}
+
+_OPTIONAL_DEPENDENCY_MESSAGES = {
+    ".repo_ingestor": (
+        "Repository ingestion requires optional dependency 'GitPython'. "
+        "Install it before importing RepoIngestor or using ingest_repository()."
+    ),
+    ".web_ingestor": (
+        "Web ingestion requires optional dependency 'beautifulsoup4'. "
+        "Install it before importing WebIngestor or using ingest_web()."
+    ),
+    ".feed_ingestor": (
+        "Feed ingestion requires optional dependency 'beautifulsoup4'. "
+        "Install it before importing FeedIngestor or using ingest_feed()."
+    ),
+    ".email_ingestor": (
+        "Email ingestion requires optional dependency 'beautifulsoup4'. "
+        "Install it before importing EmailIngestor or using ingest_email()."
+    ),
+}
+
+
+def __getattr__(name: str) -> Any:
+    """Load optional ingestion backends only when callers request them."""
+    if name not in _LAZY_EXPORTS:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+    module_name, attr_name = _LAZY_EXPORTS[name]
+    try:
+        module = importlib.import_module(module_name, __name__)
+    except ImportError as exc:
+        message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+        missing_name = getattr(exc, "name", None)
+        missing_dependency = missing_name in {"git", "bs4"} or any(
+            f"No module named '{dependency}'" in str(exc)
+            for dependency in ("git", "bs4")
+        )
+        if message and missing_dependency:
+            raise ImportError(message) from exc
+        raise
+
+    value = getattr(module, attr_name)
+    globals()[name] = value
+    return value
 
 __all__ = [
     # File ingestion

semantica/ingest/methods.py

@@ -139,26 +139,35 @@ Example Usage:
     >>> content = ingest_web("https://example.com", method="url")
 """
 
+from __future__ import annotations
+
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Union
 
 from ..utils.exceptions import ConfigurationError, ProcessingError
 from ..utils.logging import get_logger
 from .config import ingest_config
-from .db_ingestor import DBIngestor, TableData
-from .email_ingestor import EmailData, EmailIngestor
-from .feed_ingestor import FeedData, FeedIngestor
 from .file_ingestor import FileIngestor, FileObject
-from .mcp_ingestor import MCPData, MCPIngestor
-from .ontology_ingestor import OntologyData, OntologyIngestor
 from .registry import method_registry
-from .repo_ingestor import RepoIngestor
-from .stream_ingestor import StreamIngestor, StreamProcessor
-from .web_ingestor import WebContent, WebIngestor
 
 logger = get_logger("ingest_methods")
 
 
+def _missing_optional_dependency(feature: str, package: str) -> ConfigurationError:
+    return ConfigurationError(
+        f"{feature} requires optional dependency '{package}'. "
+        f"Install it before using this ingestion backend."
+    )
+
+
+def _is_missing_dependency(exc: ImportError, *dependency_names: str) -> bool:
+    missing_name = getattr(exc, "name", None)
+    return missing_name in dependency_names or any(
+        f"No module named '{dependency}'" in str(exc)
+        for dependency in dependency_names
+    )
+
+
 def ingest_file(
     source: Union[str, Path, List[Union[str, Path]]], method: str = "file", **kwargs
 ) -> Union[FileObject, List[FileObject], Dict[str, Any]]:
@@ -259,6 +268,16 @@ def ingest_web(
             )
 
     try:
+        try:
+            from .web_ingestor import WebIngestor
+        except ImportError as exc:
+            if _is_missing_dependency(exc, "bs4"):
+                raise _missing_optional_dependency(
+                    "Web ingestion",
+                    "beautifulsoup4",
+                ) from exc
+            raise
+
         # Get config
         config = ingest_config.get_method_config("web")
         config.update(kwargs)
@@ -318,6 +337,16 @@ def ingest_feed(
             )
 
     try:
+        try:
+            from .feed_ingestor import FeedIngestor
+        except ImportError as exc:
+            if _is_missing_dependency(exc, "bs4"):
+                raise _missing_optional_dependency(
+                    "Feed ingestion",
+                    "beautifulsoup4",
+                ) from exc
+            raise
+
         # Get config
         config = ingest_config.get_method_config("feed")
         config.update(kwargs)
@@ -375,6 +404,8 @@ def ingest_stream(
             )
 
     try:
+        from .stream_ingestor import StreamIngestor
+
         # Get config
         config = ingest_config.get_method_config("stream")
         config.update(kwargs)
@@ -447,6 +478,13 @@ def ingest_repository(
             )
 
     try:
+        try:
+            from .repo_ingestor import RepoIngestor
+        except ImportError as exc:
+            if _is_missing_dependency(exc, "git"):
+                raise _missing_optional_dependency("Repository ingestion", "GitPython") from exc
+            raise
+
         # Get config
         config = ingest_config.get_method_config("repo")
         config.update(kwargs)
@@ -505,6 +543,16 @@ def ingest_email(
             )
 
     try:
+        try:
+            from .email_ingestor import EmailIngestor
+        except ImportError as exc:
+            if _is_missing_dependency(exc, "bs4"):
+                raise _missing_optional_dependency(
+                    "Email ingestion",
+                    "beautifulsoup4",
+                ) from exc
+            raise
+
         # Get config
         config = ingest_config.get_method_config("email")
         config.update(kwargs)
@@ -572,6 +620,8 @@ def ingest_ontology(
             )
 
     try:
+        from .ontology_ingestor import OntologyIngestor
+
         # Get config
         config = ingest_config.get_method_config("ontology")
         config.update(kwargs)
@@ -636,6 +686,8 @@ def ingest_database(
                 )
 
     try:
+        from .db_ingestor import DBIngestor
+
         # Get config
         config = ingest_config.get_method_config("db")
         config.update(kwargs)
@@ -728,6 +780,8 @@ def ingest_mcp(
             )
 
     try:
+        from .mcp_ingestor import MCPIngestor
+
         # Get config
         config = ingest_config.get_method_config("mcp")
         config.update(kwargs)

tests/ingest/test_optional_imports.py

@@ -0,0 +1,76 @@
+import os
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def _run_python_with_blocked_modules(
+    code: str,
+    blocked_modules: tuple[str, ...],
+) -> subprocess.CompletedProcess[str]:
+    blocker = f"""
+import importlib.abc
+import sys
+
+BLOCKED_MODULES = {blocked_modules!r}
+
+
+class OptionalDependencyBlocker(importlib.abc.MetaPathFinder):
+    def find_spec(self, fullname, path=None, target=None):
+        root_name = fullname.split(".", 1)[0]
+        if root_name in BLOCKED_MODULES:
+            raise ModuleNotFoundError(f"No module named '{{root_name}}'")
+        return None
+
+
+sys.meta_path.insert(0, OptionalDependencyBlocker())
+"""
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(REPO_ROOT)
+    env["PYTHONDONTWRITEBYTECODE"] = "1"
+    return subprocess.run(
+        [sys.executable, "-c", textwrap.dedent(blocker + "\n" + code)],
+        cwd=REPO_ROOT,
+        env=env,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+
+
+def test_file_ingestion_imports_without_optional_backends() -> None:
+    result = _run_python_with_blocked_modules(
+        """
+from semantica.ingest import FileIngestor, ingest_file
+print(FileIngestor.__name__, callable(ingest_file))
+""",
+        ("git", "bs4"),
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert "FileIngestor True" in result.stdout
+
+
+def test_repository_ingestion_reports_missing_gitpython_when_used() -> None:
+    result = _run_python_with_blocked_modules(
+        """
+from semantica.ingest import ingest_repository
+
+try:
+    ingest_repository("https://example.com/repo.git")
+except Exception as exc:
+    print(type(exc).__name__, exc)
+else:
+    raise SystemExit("expected repository ingestion to fail without GitPython")
+""",
+        ("git",),
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert "ConfigurationError" in result.stdout
+    assert "Repository ingestion" in result.stdout
+    assert "GitPython" in result.stdout