diff --git a/README.md b/README.md index a2646bde..f61d3845 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,8 @@ semantica doctor # Config file pass ~/.semantica/config.yaml ``` +**Running in a script or CI?** Progress bars are written only when stdout is an interactive terminal (or a Jupyter notebook), so piping and redirecting stay clean by default. Override with `SEMANTICA_DISABLE_PROGRESS=1` to silence progress everywhere, or `SEMANTICA_FORCE_PROGRESS=1` to keep it when stdout is redirected. `SEMANTICA_DISABLE_PROGRESS` takes precedence. +
If Semantica solves a real problem for you, a star helps others find it. @@ -1561,11 +1563,11 @@ On-premises deployment · Private cloud · Custom domain implementations · SLA- ## Star History - + - - - Star History Chart + + + Star History Chart diff --git a/docs/reference/triplet_store.md b/docs/reference/triplet_store.md index ad7a0645..ee5c24e6 100644 --- a/docs/reference/triplet_store.md +++ b/docs/reference/triplet_store.md @@ -182,7 +182,7 @@ for row in result.bindings: store = TripletStore( backend="rdf4j", endpoint="http://localhost:8080/rdf4j-server", - repository_id="semantica", # passed through **config + repository_id="semantica", # selects the remote repository ) ``` diff --git a/docs/reference/utils.md b/docs/reference/utils.md index 83f9cc97..f7e97789 100644 --- a/docs/reference/utils.md +++ b/docs/reference/utils.md @@ -77,7 +77,17 @@ Most users won't call utils directly: it's the **shared foundation** for all mod export SEMANTICA_LOG_LEVEL=DEBUG export SEMANTICA_LOG_FORMAT=json # "json" | "text" export SEMANTICA_DISABLE_PROGRESS=true + export SEMANTICA_FORCE_PROGRESS=true ``` + + + **Progress bars follow your terminal.** Console progress is written only when + stdout is an interactive terminal (or a Jupyter notebook), so piping or + redirecting output no longer fills logs with progress bars and escape + sequences. Set `SEMANTICA_DISABLE_PROGRESS` to silence progress even in a + terminal, or `SEMANTICA_FORCE_PROGRESS` to keep it when stdout is redirected. + `SEMANTICA_DISABLE_PROGRESS` wins if both are set. + diff --git a/docs/storage-backends.md b/docs/storage-backends.md index bb05e115..dc211ef0 100644 --- a/docs/storage-backends.md +++ b/docs/storage-backends.md @@ -35,7 +35,7 @@ This page is intentionally conservative: it distinguishes between an adapter exi | FalkorDB | LPG | Yes | Yes | Partial | Partial | Redis-based; provenance depends on node/edge properties, and multi-graph isolation depends on the selected graph name. | | Amazon Neptune | LPG | Yes | Yes | Partial | Partial | Use the property-graph endpoint; AWS auth, VPC, and endpoint configuration can affect local tests. Provenance depends on node/edge properties. | | Apache AGE | LPG | Yes | Yes | Partial | Partial | Runs through PostgreSQL/AGE; Cypher compatibility and property handling can differ from standalone LPG engines. | -| RDF4J | RDF | Yes | Partial | Partial | Partial | Context separation relies on named graphs; triple-level provenance may require reification or graph-level metadata. `RDF4JStore(repository_id=...)` currently has no effect — the constructor always connects to the `"default"` repository regardless of the value passed; track a fix separately. | +| RDF4J | RDF | Yes | Partial | Partial | Partial | Context separation relies on named graphs; triple-level provenance may require reification or graph-level metadata. | | Apache Jena | RDF | Yes | Partial | Partial | Partial | Named graphs are needed for context separation; backend configuration and transaction behavior matter. | | Blazegraph | RDF | Yes | Partial | Partial | Partial | Use quads/named graphs for context; IRI stability and graph naming matter for provenance. | | Anzo | RDF | Yes | Partial | Partial | Partial | Anzo deployments are environment-specific; validate `dataset_uri`/graphmart naming, named-graph support, and provenance mapping. | @@ -107,7 +107,7 @@ from semantica.triplet_store import RDF4JStore store = RDF4JStore( endpoint='http://localhost:8080/rdf4j-server', - repository_id='semantica' # currently has no effect; connects to "default" (see Known limitations) + repository_id='semantica' ) ``` diff --git a/semantica/triplet_store/rdf4j_store.py b/semantica/triplet_store/rdf4j_store.py index 8dad6997..d788ab7b 100644 --- a/semantica/triplet_store/rdf4j_store.py +++ b/semantica/triplet_store/rdf4j_store.py @@ -28,7 +28,7 @@ License: MIT import re from typing import Any, Dict, List, Optional -from urllib.parse import urlparse +from urllib.parse import quote, urlparse import requests from rdflib import Graph, Literal @@ -67,7 +67,8 @@ class RDF4JStore: self.progress_tracker.enabled = True self.endpoint = endpoint.rstrip("/") - self.repository_id = config.get("repository_id", "default") + self.repository_id = repository_id or config.get("repository_id", "default") + self._encoded_repository_id = quote(self.repository_id, safe="") self.username = config.get("username") self.password = config.get("password") self.timeout = config.get("timeout", 30) @@ -79,7 +80,7 @@ class RDF4JStore: """Connect to RDF4J server.""" try: # Test connection - test_url = f"{self.endpoint}/repositories/{self.repository_id}" + test_url = f"{self.endpoint}/repositories/{self._encoded_repository_id}" response = requests.get( test_url, timeout=self.timeout, @@ -100,11 +101,11 @@ class RDF4JStore: def _get_sparql_endpoint(self) -> str: """Get SPARQL query endpoint.""" - return f"{self.endpoint}/repositories/{self.repository_id}" + return f"{self.endpoint}/repositories/{self._encoded_repository_id}" def _get_update_endpoint(self) -> str: """Get SPARQL Update endpoint.""" - return f"{self.endpoint}/repositories/{self.repository_id}/statements" + return f"{self.endpoint}/repositories/{self._encoded_repository_id}/statements" def _is_construct_query(self, query: str) -> bool: """ @@ -163,7 +164,7 @@ class RDF4JStore: """ # RDF4J transaction support transaction_url = ( - f"{self.endpoint}/repositories/{self.repository_id}/transactions" + f"{self.endpoint}/repositories/{self._encoded_repository_id}/transactions" ) try: diff --git a/semantica/utils/progress_tracker.py b/semantica/utils/progress_tracker.py index febfb4b9..f27768d4 100644 --- a/semantica/utils/progress_tracker.py +++ b/semantica/utils/progress_tracker.py @@ -64,6 +64,29 @@ def _progress_disabled_from_env() -> bool: "on", ) + +def _progress_forced_from_env() -> bool: + """Return whether console progress is forced on despite a non-interactive stdout.""" + return os.getenv("SEMANTICA_FORCE_PROGRESS", "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + + +def _stdout_is_tty() -> bool: + """Return whether stdout is an interactive terminal. + + Replacement streams do not always implement ``isatty`` and closed streams can + raise, so both cases are treated as non-interactive. + """ + try: + return bool(sys.stdout is not None and sys.stdout.isatty()) + except (AttributeError, ValueError): + return False + + # Try to import IPython for Jupyter support try: from IPython import get_ipython @@ -1040,18 +1063,24 @@ class ProgressTracker: # Create displays self.displays: List[ProgressDisplay] = [] + # Console output only suits an interactive stdout. When output is piped or + # redirected (scripts, CI logs) the progress bars and their escape + # sequences would otherwise drown the program's own output. + console_ok = _stdout_is_tty() or self.is_jupyter or _progress_forced_from_env() + # Always try Jupyter first if available, fallback to console if IPYTHON_AVAILABLE: # Try to detect Jupyter - if available, use it if self.is_jupyter and not self.disable_jupyter_progress: self.displays.append(JupyterProgressDisplay(use_emoji=use_emoji)) # Also add console as fallback for immediate feedback - self.displays.append( - ConsoleProgressDisplay( - use_emoji=use_emoji, update_interval=update_interval + if console_ok: + self.displays.append( + ConsoleProgressDisplay( + use_emoji=use_emoji, update_interval=update_interval + ) ) - ) - else: + elif console_ok: self.displays.append( ConsoleProgressDisplay( use_emoji=use_emoji, update_interval=update_interval diff --git a/tests/test_progress_tracker_regressions.py b/tests/test_progress_tracker_regressions.py index ac4c09da..b42a885b 100644 --- a/tests/test_progress_tracker_regressions.py +++ b/tests/test_progress_tracker_regressions.py @@ -12,6 +12,7 @@ import semantica.utils.progress_tracker as progress_module @pytest.fixture(autouse=True) def reset_progress_singletons(monkeypatch): monkeypatch.delenv("SEMANTICA_DISABLE_PROGRESS", raising=False) + monkeypatch.delenv("SEMANTICA_FORCE_PROGRESS", raising=False) progress_module.ProgressTracker._instance = None progress_module._global_tracker = None yield @@ -19,6 +20,40 @@ def reset_progress_singletons(monkeypatch): progress_module._global_tracker = None +class _FakeStdout: + """Minimal stdout stand-in with controllable TTY reporting.""" + + encoding = "utf-8" + + def __init__(self, tty): + self._tty = tty + self.written = [] + + def isatty(self): + return self._tty + + def write(self, text): + self.written.append(text) + return len(text) + + def flush(self): + pass + + +def _use_stdout(monkeypatch, tty): + """Point sys.stdout at a fake with the given TTY behaviour, outside Jupyter.""" + stream = _FakeStdout(tty=tty) + monkeypatch.setattr(sys, "stdout", stream) + monkeypatch.setattr( + progress_module.ProgressTracker, "_detect_jupyter", lambda *_: False + ) + return stream + + +def _displays_of(tracker, display_cls): + return [d for d in tracker.displays if isinstance(d, display_cls)] + + def _install_tracker_as_singleton(tracker: progress_module.ProgressTracker) -> None: progress_module.ProgressTracker._instance = tracker progress_module._global_tracker = tracker @@ -100,6 +135,67 @@ def test_disable_progress_env_prevents_reenable(monkeypatch): assert tracker.start_tracking(module="core", submodule="test") == "" +def test_console_display_omitted_when_stdout_is_not_a_tty(monkeypatch): + _use_stdout(monkeypatch, tty=False) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + + assert _displays_of(tracker, progress_module.ConsoleProgressDisplay) == [] + + +def test_console_display_present_when_stdout_is_a_tty(monkeypatch): + _use_stdout(monkeypatch, tty=True) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + + assert _displays_of(tracker, progress_module.ConsoleProgressDisplay) + + +def test_file_display_survives_non_tty_stdout(monkeypatch): + _use_stdout(monkeypatch, tty=False) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + + assert _displays_of(tracker, progress_module.FileProgressDisplay) + + +def test_force_progress_env_restores_console_display_on_non_tty(monkeypatch): + monkeypatch.setenv("SEMANTICA_FORCE_PROGRESS", "1") + _use_stdout(monkeypatch, tty=False) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + + assert _displays_of(tracker, progress_module.ConsoleProgressDisplay) + + +def test_disable_progress_env_beats_force_progress_env(monkeypatch): + monkeypatch.setenv("SEMANTICA_DISABLE_PROGRESS", "1") + monkeypatch.setenv("SEMANTICA_FORCE_PROGRESS", "1") + stream = _use_stdout(monkeypatch, tty=False) + + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + _install_tracker_as_singleton(tracker) + + assert tracker.enabled is False + assert tracker.start_tracking(module="core", submodule="test") == "" + assert stream.written == [] + + +def test_non_tty_stdout_stays_silent_after_module_reenables_tracker(monkeypatch): + stream = _use_stdout(monkeypatch, tty=False) + tracker = progress_module.ProgressTracker(use_emoji=False, update_interval=0) + _install_tracker_as_singleton(tracker) + + # Mirrors the ~20 modules that do `self.progress_tracker.enabled = True`. + tracker.enabled = True + tracking_id = tracker.start_tracking( + module="core", submodule="Semantica", message="Building" + ) + tracker.update_progress(tracking_id, processed=1, total=1, message="Processing") + + assert stream.written == [] + + def test_build_knowledge_base_subprocess_does_not_deadlock(): root = Path(__file__).resolve().parents[1] runtime_dir = root / "test_data" / "runtime" / f"build-regression-{os.getpid()}" diff --git a/tests/triplet_store/test_rdf4j_store.py b/tests/triplet_store/test_rdf4j_store.py index 4ef4f16b..03a9b90b 100644 --- a/tests/triplet_store/test_rdf4j_store.py +++ b/tests/triplet_store/test_rdf4j_store.py @@ -22,6 +22,71 @@ def _make_connected_store(): CONSTRUCT_QUERY = "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }" +class TestRDF4JStoreInitialization(unittest.TestCase): + + def test_explicit_repository_id_selects_repository(self): + response = MagicMock(status_code=200) + + with patch( + "semantica.triplet_store.rdf4j_store.requests.get", + return_value=response, + ) as mock_get: + store = RDF4JStore( + endpoint="http://localhost:8080/rdf4j-server/", + repository_id="semantica", + ) + + self.assertEqual(store.repository_id, "semantica") + mock_get.assert_called_once_with( + "http://localhost:8080/rdf4j-server/repositories/semantica", + timeout=30, + auth=None, + ) + + def test_repository_id_is_encoded_as_a_single_url_path_segment(self): + response = MagicMock(status_code=200) + + with patch( + "semantica.triplet_store.rdf4j_store.requests.get", + return_value=response, + ) as mock_get: + store = RDF4JStore( + endpoint="http://localhost:8080/rdf4j-server", + repository_id="team/repo ?#", + ) + + self.assertEqual(store.repository_id, "team/repo ?#") + mock_get.assert_called_once_with( + "http://localhost:8080/rdf4j-server/repositories/team%2Frepo%20%3F%23", + timeout=30, + auth=None, + ) + self.assertEqual( + store._get_sparql_endpoint(), + "http://localhost:8080/rdf4j-server/repositories/team%2Frepo%20%3F%23", + ) + self.assertEqual( + store._get_update_endpoint(), + "http://localhost:8080/rdf4j-server/repositories/" + "team%2Frepo%20%3F%23/statements", + ) + + transaction_response = MagicMock() + transaction_response.headers = {"Location": "/transactions/tx-1"} + with patch( + "semantica.triplet_store.rdf4j_store.requests.post", + return_value=transaction_response, + ) as mock_post: + self.assertEqual(store.begin_transaction(), "tx-1") + + mock_post.assert_called_once_with( + "http://localhost:8080/rdf4j-server/repositories/" + "team%2Frepo%20%3F%23/transactions", + timeout=30, + auth=None, + ) + + class TestRDF4JStoreIsConstructQuery(unittest.TestCase): def test_detects_uppercase(self): self.assertTrue(_make_connected_store()._is_construct_query(