diff --git a/semantica/ingest/__init__.py b/semantica/ingest/__init__.py index 02ea1a2e..95b252d4 100644 --- a/semantica/ingest/__init__.py +++ b/semantica/ingest/__init__.py @@ -112,22 +112,18 @@ Example Usage: >>> content = ingest_web("https://example.com", method="url") """ -from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from __future__ import annotations + +import importlib +from typing import Any, Dict, Tuple from .config import IngestConfig, ingest_config -from .db_ingestor import DatabaseConnector, DataExporter, DBIngestor, TableData -from .email_ingestor import AttachmentProcessor, EmailData, EmailIngestor -from .email_ingestor import EmailParser as EmailIngestorParser -from .feed_ingestor import FeedData, FeedIngestor, FeedItem, FeedMonitor, FeedParser from .file_ingestor import ( CloudStorageIngestor, FileIngestor, FileObject, FileTypeDetector, ) -from .mcp_client import MCPClient, MCPResource, MCPTool -from .mcp_ingestor import MCPData, MCPIngestor from .methods import ( get_ingest_method, ingest, @@ -143,34 +139,103 @@ from .methods import ( list_available_methods, ) from .registry import MethodRegistry, method_registry -from .repo_ingestor import ( - CodeExtractor, - CodeFile, - CommitInfo, - GitAnalyzer, - RepoIngestor, -) -from .stream_ingestor import ( - KafkaProcessor, - KinesisProcessor, - PulsarProcessor, - RabbitMQProcessor, - StreamIngestor, - StreamMessage, - StreamMonitor, - StreamProcessor, -) -from .web_ingestor import ( - ContentExtractor, - RateLimiter, - RobotsChecker, - SitemapCrawler, - WebContent, - WebIngestor, -) -from .ontology_ingestor import OntologyData, OntologyIngestor -from .snowflake_ingestor import SnowflakeConnector, SnowflakeData, SnowflakeIngestor +_LAZY_EXPORTS: Dict[str, Tuple[str, str]] = { + # Web ingestion + "WebIngestor": (".web_ingestor", "WebIngestor"), + "WebContent": (".web_ingestor", "WebContent"), + "RateLimiter": (".web_ingestor", "RateLimiter"), + "RobotsChecker": (".web_ingestor", "RobotsChecker"), + "ContentExtractor": (".web_ingestor", "ContentExtractor"), + "SitemapCrawler": (".web_ingestor", "SitemapCrawler"), + # Feed ingestion + "FeedIngestor": (".feed_ingestor", "FeedIngestor"), + "FeedItem": (".feed_ingestor", "FeedItem"), + "FeedData": (".feed_ingestor", "FeedData"), + "FeedParser": (".feed_ingestor", "FeedParser"), + "FeedMonitor": (".feed_ingestor", "FeedMonitor"), + # Stream ingestion + "StreamIngestor": (".stream_ingestor", "StreamIngestor"), + "StreamMessage": (".stream_ingestor", "StreamMessage"), + "StreamProcessor": (".stream_ingestor", "StreamProcessor"), + "KafkaProcessor": (".stream_ingestor", "KafkaProcessor"), + "RabbitMQProcessor": (".stream_ingestor", "RabbitMQProcessor"), + "KinesisProcessor": (".stream_ingestor", "KinesisProcessor"), + "PulsarProcessor": (".stream_ingestor", "PulsarProcessor"), + "StreamMonitor": (".stream_ingestor", "StreamMonitor"), + # Repository ingestion + "RepoIngestor": (".repo_ingestor", "RepoIngestor"), + "CodeFile": (".repo_ingestor", "CodeFile"), + "CommitInfo": (".repo_ingestor", "CommitInfo"), + "CodeExtractor": (".repo_ingestor", "CodeExtractor"), + "GitAnalyzer": (".repo_ingestor", "GitAnalyzer"), + # Email ingestion + "EmailIngestor": (".email_ingestor", "EmailIngestor"), + "EmailData": (".email_ingestor", "EmailData"), + "AttachmentProcessor": (".email_ingestor", "AttachmentProcessor"), + "EmailIngestorParser": (".email_ingestor", "EmailParser"), + # Database ingestion + "DBIngestor": (".db_ingestor", "DBIngestor"), + "TableData": (".db_ingestor", "TableData"), + "DatabaseConnector": (".db_ingestor", "DatabaseConnector"), + "DataExporter": (".db_ingestor", "DataExporter"), + # MCP ingestion + "MCPIngestor": (".mcp_ingestor", "MCPIngestor"), + "MCPData": (".mcp_ingestor", "MCPData"), + "MCPClient": (".mcp_client", "MCPClient"), + "MCPResource": (".mcp_client", "MCPResource"), + "MCPTool": (".mcp_client", "MCPTool"), + # Ontology ingestion + "OntologyIngestor": (".ontology_ingestor", "OntologyIngestor"), + "OntologyData": (".ontology_ingestor", "OntologyData"), + # Snowflake ingestion + "SnowflakeIngestor": (".snowflake_ingestor", "SnowflakeIngestor"), + "SnowflakeData": (".snowflake_ingestor", "SnowflakeData"), + "SnowflakeConnector": (".snowflake_ingestor", "SnowflakeConnector"), +} + +_OPTIONAL_DEPENDENCY_MESSAGES = { + ".repo_ingestor": ( + "Repository ingestion requires optional dependency 'GitPython'. " + "Install it before importing RepoIngestor or using ingest_repository()." + ), + ".web_ingestor": ( + "Web ingestion requires optional dependency 'beautifulsoup4'. " + "Install it before importing WebIngestor or using ingest_web()." + ), + ".feed_ingestor": ( + "Feed ingestion requires optional dependency 'beautifulsoup4'. " + "Install it before importing FeedIngestor or using ingest_feed()." + ), + ".email_ingestor": ( + "Email ingestion requires optional dependency 'beautifulsoup4'. " + "Install it before importing EmailIngestor or using ingest_email()." + ), +} + + +def __getattr__(name: str) -> Any: + """Load optional ingestion backends only when callers request them.""" + if name not in _LAZY_EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + module_name, attr_name = _LAZY_EXPORTS[name] + try: + module = importlib.import_module(module_name, __name__) + except ImportError as exc: + message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name) + missing_name = getattr(exc, "name", None) + missing_dependency = missing_name in {"git", "bs4"} or any( + f"No module named '{dependency}'" in str(exc) + for dependency in ("git", "bs4") + ) + if message and missing_dependency: + raise ImportError(message) from exc + raise + + value = getattr(module, attr_name) + globals()[name] = value + return value __all__ = [ # File ingestion diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.py index ecc1a90a..e0d56408 100644 --- a/semantica/ingest/methods.py +++ b/semantica/ingest/methods.py @@ -139,26 +139,35 @@ Example Usage: >>> content = ingest_web("https://example.com", method="url") """ +from __future__ import annotations + from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Union from ..utils.exceptions import ConfigurationError, ProcessingError from ..utils.logging import get_logger from .config import ingest_config -from .db_ingestor import DBIngestor, TableData -from .email_ingestor import EmailData, EmailIngestor -from .feed_ingestor import FeedData, FeedIngestor from .file_ingestor import FileIngestor, FileObject -from .mcp_ingestor import MCPData, MCPIngestor -from .ontology_ingestor import OntologyData, OntologyIngestor from .registry import method_registry -from .repo_ingestor import RepoIngestor -from .stream_ingestor import StreamIngestor, StreamProcessor -from .web_ingestor import WebContent, WebIngestor logger = get_logger("ingest_methods") +def _missing_optional_dependency(feature: str, package: str) -> ConfigurationError: + return ConfigurationError( + f"{feature} requires optional dependency '{package}'. " + f"Install it before using this ingestion backend." + ) + + +def _is_missing_dependency(exc: ImportError, *dependency_names: str) -> bool: + missing_name = getattr(exc, "name", None) + return missing_name in dependency_names or any( + f"No module named '{dependency}'" in str(exc) + for dependency in dependency_names + ) + + def ingest_file( source: Union[str, Path, List[Union[str, Path]]], method: str = "file", **kwargs ) -> Union[FileObject, List[FileObject], Dict[str, Any]]: @@ -259,6 +268,16 @@ def ingest_web( ) try: + try: + from .web_ingestor import WebIngestor + except ImportError as exc: + if _is_missing_dependency(exc, "bs4"): + raise _missing_optional_dependency( + "Web ingestion", + "beautifulsoup4", + ) from exc + raise + # Get config config = ingest_config.get_method_config("web") config.update(kwargs) @@ -318,6 +337,16 @@ def ingest_feed( ) try: + try: + from .feed_ingestor import FeedIngestor + except ImportError as exc: + if _is_missing_dependency(exc, "bs4"): + raise _missing_optional_dependency( + "Feed ingestion", + "beautifulsoup4", + ) from exc + raise + # Get config config = ingest_config.get_method_config("feed") config.update(kwargs) @@ -375,6 +404,8 @@ def ingest_stream( ) try: + from .stream_ingestor import StreamIngestor + # Get config config = ingest_config.get_method_config("stream") config.update(kwargs) @@ -447,6 +478,13 @@ def ingest_repository( ) try: + try: + from .repo_ingestor import RepoIngestor + except ImportError as exc: + if _is_missing_dependency(exc, "git"): + raise _missing_optional_dependency("Repository ingestion", "GitPython") from exc + raise + # Get config config = ingest_config.get_method_config("repo") config.update(kwargs) @@ -505,6 +543,16 @@ def ingest_email( ) try: + try: + from .email_ingestor import EmailIngestor + except ImportError as exc: + if _is_missing_dependency(exc, "bs4"): + raise _missing_optional_dependency( + "Email ingestion", + "beautifulsoup4", + ) from exc + raise + # Get config config = ingest_config.get_method_config("email") config.update(kwargs) @@ -572,6 +620,8 @@ def ingest_ontology( ) try: + from .ontology_ingestor import OntologyIngestor + # Get config config = ingest_config.get_method_config("ontology") config.update(kwargs) @@ -636,6 +686,8 @@ def ingest_database( ) try: + from .db_ingestor import DBIngestor + # Get config config = ingest_config.get_method_config("db") config.update(kwargs) @@ -728,6 +780,8 @@ def ingest_mcp( ) try: + from .mcp_ingestor import MCPIngestor + # Get config config = ingest_config.get_method_config("mcp") config.update(kwargs) diff --git a/tests/ingest/test_optional_imports.py b/tests/ingest/test_optional_imports.py new file mode 100644 index 00000000..9f629edb --- /dev/null +++ b/tests/ingest/test_optional_imports.py @@ -0,0 +1,76 @@ +import os +import subprocess +import sys +import textwrap +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _run_python_with_blocked_modules( + code: str, + blocked_modules: tuple[str, ...], +) -> subprocess.CompletedProcess[str]: + blocker = f""" +import importlib.abc +import sys + +BLOCKED_MODULES = {blocked_modules!r} + + +class OptionalDependencyBlocker(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + root_name = fullname.split(".", 1)[0] + if root_name in BLOCKED_MODULES: + raise ModuleNotFoundError(f"No module named '{{root_name}}'") + return None + + +sys.meta_path.insert(0, OptionalDependencyBlocker()) +""" + env = os.environ.copy() + env["PYTHONPATH"] = str(REPO_ROOT) + env["PYTHONDONTWRITEBYTECODE"] = "1" + return subprocess.run( + [sys.executable, "-c", textwrap.dedent(blocker + "\n" + code)], + cwd=REPO_ROOT, + env=env, + text=True, + capture_output=True, + check=False, + ) + + +def test_file_ingestion_imports_without_optional_backends() -> None: + result = _run_python_with_blocked_modules( + """ +from semantica.ingest import FileIngestor, ingest_file +print(FileIngestor.__name__, callable(ingest_file)) +""", + ("git", "bs4"), + ) + + assert result.returncode == 0, result.stderr + assert "FileIngestor True" in result.stdout + + +def test_repository_ingestion_reports_missing_gitpython_when_used() -> None: + result = _run_python_with_blocked_modules( + """ +from semantica.ingest import ingest_repository + +try: + ingest_repository("https://example.com/repo.git") +except Exception as exc: + print(type(exc).__name__, exc) +else: + raise SystemExit("expected repository ingestion to fail without GitPython") +""", + ("git",), + ) + + assert result.returncode == 0, result.stderr + assert "ConfigurationError" in result.stdout + assert "Repository ingestion" in result.stdout + assert "GitPython" in result.stdout