mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-11 04:01:32 +00:00
Luffy2208andKaifAhmad1
15d58f2b88
* Added Parquet ingest support (#234) * docs: Add Parquet ingestion support to CHANGELOG - Add comprehensive changelog entry for PR #548 - Document ParquetIngestor class and key features - Include author credit (@Luffy2208) and PR reference - Follow existing changelog format and structure --------- Co-authored-by: KaifAhmad1 <kaifahmad087@gmail.com>
99 lines
2.6 KiB
Python
99 lines
2.6 KiB
Python
import os
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _run_python_with_blocked_modules(
|
|
code: str,
|
|
blocked_modules: tuple[str, ...],
|
|
) -> subprocess.CompletedProcess[str]:
|
|
blocker = f"""
|
|
import importlib.abc
|
|
import sys
|
|
|
|
BLOCKED_MODULES = {blocked_modules!r}
|
|
|
|
|
|
class OptionalDependencyBlocker(importlib.abc.MetaPathFinder):
|
|
def find_spec(self, fullname, path=None, target=None):
|
|
root_name = fullname.split(".", 1)[0]
|
|
if root_name in BLOCKED_MODULES:
|
|
err = ModuleNotFoundError(f"No module named '{{root_name}}'")
|
|
err.name = root_name
|
|
raise err
|
|
return None
|
|
|
|
|
|
sys.meta_path.insert(0, OptionalDependencyBlocker())
|
|
"""
|
|
env = os.environ.copy()
|
|
env["PYTHONPATH"] = str(REPO_ROOT)
|
|
env["PYTHONDONTWRITEBYTECODE"] = "1"
|
|
return subprocess.run(
|
|
[sys.executable, "-c", textwrap.dedent(blocker + "\n" + code)],
|
|
cwd=REPO_ROOT,
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def test_file_ingestion_imports_without_optional_backends() -> None:
|
|
result = _run_python_with_blocked_modules(
|
|
"""
|
|
from semantica.ingest import FileIngestor, ingest_file
|
|
print(FileIngestor.__name__, callable(ingest_file))
|
|
""",
|
|
("git", "bs4", "pyarrow"),
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "FileIngestor True" in result.stdout
|
|
|
|
|
|
def test_repository_ingestion_reports_missing_gitpython_when_used() -> None:
|
|
result = _run_python_with_blocked_modules(
|
|
"""
|
|
from semantica.ingest import ingest_repository
|
|
|
|
try:
|
|
ingest_repository("https://example.com/repo.git")
|
|
except Exception as exc:
|
|
print(type(exc).__name__, exc)
|
|
else:
|
|
raise SystemExit("expected repository ingestion to fail without GitPython")
|
|
""",
|
|
("git",),
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "ConfigurationError" in result.stdout
|
|
assert "Repository ingestion" in result.stdout
|
|
assert "GitPython" in result.stdout
|
|
|
|
|
|
def test_parquet_ingestion_reports_missing_pyarrow_when_used() -> None:
|
|
result = _run_python_with_blocked_modules(
|
|
"""
|
|
from semantica.ingest import ingest_parquet
|
|
|
|
try:
|
|
ingest_parquet("events.parquet")
|
|
except Exception as exc:
|
|
print(type(exc).__name__, exc)
|
|
else:
|
|
raise SystemExit("expected parquet ingestion to fail without pyarrow")
|
|
""",
|
|
("pyarrow",),
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "ConfigurationError" in result.stdout
|
|
assert "Parquet ingestion" in result.stdout
|
|
assert "pyarrow" in result.stdout
|